From 7c88e3e1a2b5d36281b3981e2144c2dd16c7596c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 10 Sep 2010 18:42:59 -0500 Subject: Rename generate mirrorlist view Signed-off-by: Dan McGee --- mirrors/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index ddc42cbb..b4c2c2c7 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -21,7 +21,7 @@ def __init__(self, *args, **kwargs): self.fields['protocol'].initial = [t[0] for t in protos] @csrf_exempt -def generate(request): +def generate_mirrorlist(request): if request.REQUEST.get('country', ''): form = MirrorlistForm(data=request.REQUEST) if form.is_valid(): -- cgit v1.2.3-54-g00ecf From 40ac4818aa7812a5399a0d4c176137984d5cfd30 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 12 Sep 2010 10:14:34 -0500 Subject: Add mirror status view Signed-off-by: Dan McGee --- mirrors/templatetags/__init__.py | 0 mirrors/templatetags/mirror_status.py | 15 ++++++++++ mirrors/views.py | 50 +++++++++++++++++++++++++++++++++ templates/mirrors/status.html | 53 +++++++++++++++++++++++++++++++++++ templates/mirrors/status_table.html | 29 +++++++++++++++++++ urls.py | 1 + 6 files changed, 148 insertions(+) create mode 100644 mirrors/templatetags/__init__.py create mode 100644 mirrors/templatetags/mirror_status.py create mode 100644 templates/mirrors/status.html create mode 100644 templates/mirrors/status_table.html (limited to 'mirrors/views.py') diff --git a/mirrors/templatetags/__init__.py b/mirrors/templatetags/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mirrors/templatetags/mirror_status.py b/mirrors/templatetags/mirror_status.py new file mode 100644 index 00000000..09c5b331 --- /dev/null +++ b/mirrors/templatetags/mirror_status.py @@ -0,0 +1,15 @@ +from django import template + +register = template.Library() + +@register.filter +def duration(value): + if not value: + return u'\u221e' + # does not take microseconds into account + total_secs = value.seconds + value.days * 24 * 3600 + mins, secs = divmod(total_secs, 60) + hrs, mins = divmod(mins, 60) + return '%d:%02d' % (hrs, mins) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index b4c2c2c7..a31c1371 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,9 +1,14 @@ from django import forms +from django.db.models import Avg, Count, Max, Min, StdDev +from django.db.models import Q from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template from main.utils import make_choice from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import MirrorLog + +import datetime class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False) @@ -49,4 +54,49 @@ def find_mirrors(request, countries=None, protocols=None): }, mimetype='text/plain') +def status(request): + cutoff_time = datetime.datetime.utcnow() - datetime.timedelta(hours=24) + bad_timedelta = datetime.timedelta(days=3) + + protocols = MirrorProtocol.objects.exclude(protocol__iexact='rsync') + # I swear, this actually has decent performance... + urls = MirrorUrl.objects.select_related( + 'mirror', 'protocol').filter( + mirror__active=True, mirror__public=True, + protocol__in=protocols).filter( + logs__check_time__gte=cutoff_time).annotate( + check_count=Count('logs'), last_sync=Max('logs__last_sync'), + last_check=Max('logs__check_time'), + duration_avg=Avg('logs__duration'), duration_min=Min('logs__duration'), + duration_max=Max('logs__duration'), duration_stddev=StdDev('logs__duration') + ).order_by('mirror__country', 'url') + # errors during check process go in another table + error_logs = MirrorLog.objects.filter( + is_success=False, check_time__gte=cutoff_time).values( + 'url__url', 'url__protocol__protocol', 'url__mirror__country', + 'error').annotate(Count('error'), Max('check_time')) + + good_urls = [] + bad_urls = [] + for url in urls: + if url.last_check and url.last_sync: + d = url.last_check - url.last_sync + url.delay = d + url.score = d.days * 24 + d.seconds / 3600 + url.duration_avg + url.duration_stddev + else: + url.delay = None + url.score = None + # split them into good and bad lists based on delay + if not url.delay or url.delay > bad_timedelta: + bad_urls.append(url) + else: + good_urls.append(url) + + context = { + 'good_urls': good_urls, + 'bad_urls': bad_urls, + 'error_logs': error_logs, + } + return direct_to_template(request, 'mirrors/status.html', context) + # vim: set ts=4 sw=4 et: diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html new file mode 100644 index 00000000..8dd6fa11 --- /dev/null +++ b/templates/mirrors/status.html @@ -0,0 +1,53 @@ +{% extends "base.html" %} + +{% block title %}Arch Linux - Mirror Status{% endblock %} + +{% block content %} +
+

Mirror Status

+ +

Out of Sync Mirrors

+ {% with bad_urls as urls %} + {% include "mirrors/status_table.html" %} + {% endwith %} + +

Successfully Syncing Mirrors

+ {% with good_urls as urls %} + {% include "mirrors/status_table.html" %} + {% endwith %} + +

Mirror Syncing Error Log

+ + + + + + + + + + + + + {% for log in error_logs %} + + + + + + + + + {% endfor %} + +
Mirror URLProtocolCountryError MessageLast OccurredOccurrences (last 24 hours)
{{ log.url__url }}{{ log.url__protocol__protocol }}{{ log.url__mirror__country }}{{ log.error }}{{ log.check_time__max|date:'Y-m-d H:i' }}{{ log.error__count }}
+ +
+{% load cdn %}{% jquery %} + + +{% endblock %} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html new file mode 100644 index 00000000..d7f5b0ec --- /dev/null +++ b/templates/mirrors/status_table.html @@ -0,0 +1,29 @@ +{% load mirror_status %} + + + + + + + + + + + + + + + {% for m_url in urls %} + + + + + + + + + + + {% endfor %} + +
Mirror URLProtocolCountryLast SyncDelay (hh:mm)μ Duration (secs)σ Duration (secs)Mirror Score
{{ m_url.url }}{{ m_url.protocol }}{{ m_url.mirror.country }}{{ m_url.last_sync|date:'Y-m-d H:i'|default:'unknown' }}{{ m_url.delay|duration|default:'unknown' }}{{ m_url.duration_avg|floatformat:2|default:'unknown' }}{{ m_url.duration_stddev|floatformat:2|default:'unknown' }}{{ m_url.score|floatformat:1|default:'unknown' }}
diff --git a/urls.py b/urls.py index f2ef3418..7cc60cfe 100644 --- a/urls.py +++ b/urls.py @@ -72,6 +72,7 @@ (r'^news/$', 'news.views.news_list', {}, 'news-list'), (r'^mirrors/$', 'devel.views.mirrorlist', {}, 'mirrors-list'), + (r'^mirrors/status/$', 'mirrors.views.status', {}, 'mirrors-status'), (r'^mirrorlist/$', 'mirrors.views.generate_mirrorlist', {}, 'mirrorlist'), (r'^mirrorlist/all/$', 'mirrors.views.find_mirrors', {'countries': ['all']}), -- cgit v1.2.3-54-g00ecf From 2a296af10d34c65e0f94d1a5b70c84ba31596ba4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Sep 2010 17:30:14 -0500 Subject: Add ordering, sorting, and a lot more info to mirror status page This should get this to the point where it is releasable to the general public for their use and pleasure. Still not sure on how often the check should be run, and we probably want to incorporate this mined data into some other things like the mirror list generator. Signed-off-by: Dan McGee --- mirrors/views.py | 9 ++++-- templates/mirrors/status.html | 56 ++++++++++++++++++++++++++++++++++--- templates/mirrors/status_table.html | 2 +- 3 files changed, 60 insertions(+), 7 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index a31c1371..59d6337b 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -69,12 +69,16 @@ def status(request): last_check=Max('logs__check_time'), duration_avg=Avg('logs__duration'), duration_min=Min('logs__duration'), duration_max=Max('logs__duration'), duration_stddev=StdDev('logs__duration') - ).order_by('mirror__country', 'url') + ).order_by('-last_sync', '-duration_avg') # errors during check process go in another table error_logs = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time).values( 'url__url', 'url__protocol__protocol', 'url__mirror__country', - 'error').annotate(Count('error'), Max('check_time')) + 'error').annotate( + error_count=Count('error'), last_occurred=Max('check_time') + ).order_by('-last_occurred', '-error_count') + + last_check = max([u.last_check for u in urls]) good_urls = [] bad_urls = [] @@ -93,6 +97,7 @@ def status(request): good_urls.append(url) context = { + 'last_check': last_check, 'good_urls': good_urls, 'bad_urls': bad_urls, 'error_logs': error_logs, diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 8dd6fa11..5743e47b 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -5,19 +5,64 @@ {% block content %}

Mirror Status

+

This page reports the status of all known, public, and active Arch Linux + mirrors. All data on this page reflects the status of the mirrors within + the last 24 hours. All listed times are UTC. The check script runs + on a regular basis and polls for the lastsync file in the root of + our repository layout. This file is regularly updated on the central + repository, so checking the value within allows one to see if the mirror + has synced recently. This page contains several pieces of information about + each mirror.

+
    +
  • Mirror URL: Mirrors are checked on a per-URL basis. If + both FTP and HTTP access are provided, both will be listed here.
    • +
  • Last Sync: The timestamp retrieved from the + lastsync file on the mirror. If this file could not be + retrieved or contained data we didn't recognize, this column will show + 'unknown'.
    • +
  • Delay: The calculated mirroring delay; e.g. last + check − last sync.
    • +
  • μ Duration: The average (mean) time it took to connect and + retrieve the lastsync file from the given URL. Note that this + connection time is from the location of the Arch server; your geography + may product different results.
    • +
  • σ Duration: The standard deviation of the connect and + retrieval time. A high standard deviation can indicate an unstable or + overloaded mirror.
    • +
  • Mirror Score: A very rough calculation for ranking + mirrors. It is currently calculated as hours delay + average + duration + standard deviation. Lower is better.
    • +
+

The final table on this page is an error log, which shows any errors + that occurred while contacting mirrors. This only shows errors that + occurred within the last 24 hours.

+ +

The last mirror check ran at {{ last_check|date:'Y-m-d H:i' }} UTC.

+ +

Out of Sync Mirrors

{% with bad_urls as urls %} + {% with 'outofsync_mirrors' as table_id %} {% include "mirrors/status_table.html" %} {% endwith %} + {% endwith %} +

Successfully Syncing Mirrors

{% with good_urls as urls %} + {% with 'successful_mirrors' as table_id %} {% include "mirrors/status_table.html" %} {% endwith %} + {% endwith %} +

Mirror Syncing Error Log

- +
@@ -35,8 +80,8 @@

Mirror Syncing Error Log

- - + + {% endfor %} @@ -47,7 +92,10 @@

Mirror Syncing Error Log

{% endblock %} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html index d7f5b0ec..90bbf6b6 100644 --- a/templates/mirrors/status_table.html +++ b/templates/mirrors/status_table.html @@ -1,5 +1,5 @@ {% load mirror_status %} -
Mirror URL {{ log.url__protocol__protocol }} {{ log.url__mirror__country }} {{ log.error }}{{ log.check_time__max|date:'Y-m-d H:i' }}{{ log.error__count }}{{ log.last_occurred|date:'Y-m-d H:i' }}{{ log.error_count }}
+
-- cgit v1.2.3-54-g00ecf
Mirror URL