diff options
Diffstat (limited to 'mirrors')
| -rw-r--r-- | mirrors/admin.py | 80 | ||||
| -rw-r--r-- | mirrors/fields.py | 45 | ||||
| -rw-r--r-- | mirrors/fixtures/mirrorprotocols.json | 29 | ||||
| -rw-r--r-- | mirrors/management/commands/mirrorcheck.py | 286 | ||||
| -rw-r--r-- | mirrors/management/commands/mirrorresolv.py | 65 | ||||
| -rw-r--r-- | mirrors/migrations/0001_initial.py | 130 | ||||
| -rw-r--r-- | mirrors/migrations/0002_mirrorurl_bandwidth.py | 20 | ||||
| -rw-r--r-- | mirrors/migrations/0002_rename_model_tables.py | 61 | ||||
| -rw-r--r-- | mirrors/migrations/0003_auto__add_mirrorlog.py | 72 | ||||
| -rw-r--r-- | mirrors/models.py | 165 | ||||
| -rw-r--r-- | mirrors/static/mirror_status.js | 193 | ||||
| -rw-r--r-- | mirrors/templatetags/jinja2.py | 53 | ||||
| -rw-r--r-- | mirrors/templatetags/mirror_status.py | 12 | ||||
| -rw-r--r-- | mirrors/urls.py | 18 | ||||
| -rw-r--r-- | mirrors/urls_mirrorlist.py | 11 | ||||
| -rw-r--r-- | mirrors/utils.py | 247 | ||||
| -rw-r--r-- | mirrors/views.py | 113 | ||||
| -rw-r--r-- | mirrors/views/__init__.py | 149 | ||||
| -rw-r--r-- | mirrors/views/api.py | 108 | ||||
| -rw-r--r-- | mirrors/views/mirrorlist.py | 129 |
20 files changed, 1525 insertions, 461 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py index 394b3508..16a97ea2 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -1,61 +1,99 @@ -import re +from datetime import datetime +from urlparse import urlparse, urlunsplit from django import forms from django.contrib import admin -from .models import Mirror, MirrorProtocol, MirrorUrl, MirrorRsync +from .models import (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, + CheckLocation) + class MirrorUrlForm(forms.ModelForm): class Meta: model = MirrorUrl + fields = ('url', 'country', 'bandwidth', 'active') + def clean_url(self): + # is this a valid-looking URL? + url_parts = urlparse(self.cleaned_data["url"]) + if not url_parts.scheme: + raise forms.ValidationError("No URL scheme (protocol) provided.") + if not url_parts.netloc: + raise forms.ValidationError("No URL host provided.") + if url_parts.params or url_parts.query or url_parts.fragment: + raise forms.ValidationError( + "URL parameters, query, and fragment elements are not supported.") # ensure we always save the URL with a trailing slash - url = self.cleaned_data["url"].strip() - if url[-1] == '/': - return url - return url + '/' + path = url_parts.path + if not path.endswith('/'): + path += '/' + url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', '')) + return url + class MirrorUrlInlineAdmin(admin.TabularInline): model = MirrorUrl form = MirrorUrlForm + readonly_fields = ('protocol', 'has_ipv4', 'has_ipv6') extra = 3 -# ripped off from django.forms.fields, adding netmask ability -ipv4nm_re = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$') -class IPAddressNetmaskField(forms.fields.RegexField): - default_error_messages = { - 'invalid': u'Enter a valid IPv4 address, possibly including netmask.', - } - - def __init__(self, *args, **kwargs): - super(IPAddressNetmaskField, self).__init__(ipv4nm_re, *args, **kwargs) class MirrorRsyncForm(forms.ModelForm): class Meta: model = MirrorRsync - ip = IPAddressNetmaskField(label='IP') + fields = ('ip',) + class MirrorRsyncInlineAdmin(admin.TabularInline): model = MirrorRsync form = MirrorRsyncForm extra = 2 + class MirrorAdminForm(forms.ModelForm): class Meta: model = Mirror - upstream = forms.ModelChoiceField(queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), required=False) + fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email', + 'public', 'active', 'isos', 'rsync_user', 'rsync_password', + 'bug', 'notes') + + upstream = forms.ModelChoiceField( + queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), + required=False) + class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm - list_display = ('name', 'tier', 'country', 'active', 'public', 'isos', 'admin_email', 'supported_protocols') - list_filter = ('tier', 'country', 'active', 'public') - search_fields = ('name',) + list_display = ('name', 'tier', 'active', 'public', + 'isos', 'admin_email', 'alternate_email') + list_filter = ('tier', 'active', 'public') + search_fields = ('name', 'admin_email', 'alternate_email') + readonly_fields = ('created', 'last_modified') inlines = [ MirrorUrlInlineAdmin, MirrorRsyncInlineAdmin, ] + def save_model(self, request, obj, form, change): + if '~~~' in obj.notes: + date = datetime.utcnow().strftime('%Y-%m-%d') + usertext = request.user.get_full_name() + obj.notes = obj.notes.replace('~~~', '%s (%s)' % (date, usertext)) + obj.save() + + +class MirrorProtocolAdmin(admin.ModelAdmin): + list_display = ('protocol', 'is_download', 'default') + list_filter = ('is_download', 'default') + + +class CheckLocationAdmin(admin.ModelAdmin): + list_display = ('hostname', 'source_ip', 'country', 'created') + search_fields = ('hostname', 'source_ip') + + admin.site.register(Mirror, MirrorAdmin) -admin.site.register(MirrorProtocol) +admin.site.register(MirrorProtocol, MirrorProtocolAdmin) +admin.site.register(CheckLocation, CheckLocationAdmin) # vim: set ts=4 sw=4 et: diff --git a/mirrors/fields.py b/mirrors/fields.py new file mode 100644 index 00000000..e8963edf --- /dev/null +++ b/mirrors/fields.py @@ -0,0 +1,45 @@ +from IPy import IP + +from django import forms +from django.core import validators +from django.core.exceptions import ValidationError +from django.db import models + + +class IPNetworkFormField(forms.Field): + def to_python(self, value): + if value in validators.EMPTY_VALUES: + return None + try: + value = IP(value) + except ValueError as e: + raise ValidationError(str(e)) + return value + + +class IPNetworkField(models.Field): + __metaclass__ = models.SubfieldBase + description = "IPv4 or IPv6 address or subnet" + + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 44 + super(IPNetworkField, self).__init__(*args, **kwargs) + + def get_internal_type(self): + return "IPAddressField" + + def to_python(self, value): + if not value: + return None + return IP(value) + + def get_prep_value(self, value): + value = self.to_python(value) + if not value: + return None + return str(value) + + def formfield(self, **kwargs): + defaults = {'form_class': IPNetworkFormField} + defaults.update(kwargs) + return super(IPNetworkField, self).formfield(**defaults) diff --git a/mirrors/fixtures/mirrorprotocols.json b/mirrors/fixtures/mirrorprotocols.json new file mode 100644 index 00000000..1a07510b --- /dev/null +++ b/mirrors/fixtures/mirrorprotocols.json @@ -0,0 +1,29 @@ +[ + { + "pk": 1, + "model": "mirrors.mirrorprotocol", + "fields": { + "is_download": true, + "default": true, + "protocol": "http" + } + }, + { + "pk": 3, + "model": "mirrors.mirrorprotocol", + "fields": { + "is_download": false, + "default": false, + "protocol": "rsync" + } + }, + { + "pk": 5, + "model": "mirrors.mirrorprotocol", + "fields": { + "is_download": true, + "default": false, + "protocol": "https" + } + } +] diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 1662b15c..1a33073a 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -9,145 +9,255 @@ we encounter errors, record those as well. Usage: ./manage.py mirrorcheck """ -from django.core.management.base import NoArgsCommand -from django.db.models import Q - -from datetime import datetime, timedelta +from collections import deque +from datetime import datetime +from httplib import HTTPException import logging +import os +from optparse import make_option +from pytz import utc import re import socket +import ssl +import subprocess import sys import time -import thread +import tempfile from threading import Thread +import types from Queue import Queue, Empty import urllib2 -from logging import ERROR, WARNING, INFO, DEBUG +from django.core.management.base import NoArgsCommand +from django.db import transaction +from django.utils.timezone import now + +from mirrors.models import MirrorUrl, MirrorLog, CheckLocation -from mirrors.models import Mirror, MirrorUrl, MirrorLog logging.basicConfig( - level=WARNING, + level=logging.WARNING, format='%(asctime)s -> %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', stream=sys.stderr) logger = logging.getLogger() + class Command(NoArgsCommand): + option_list = NoArgsCommand.option_list + ( + make_option('-t', '--timeout', dest='timeout', type='float', default=10.0, + help='Timeout value for connecting to URL'), + make_option('-l', '--location', dest='location', type='int', + help='ID of CheckLocation object to use for this run'), + ) help = "Runs a check on all known mirror URLs to determine their up-to-date status." def handle_noargs(self, **options): v = int(options.get('verbosity', 0)) if v == 0: - logger.level = ERROR + logger.level = logging.ERROR elif v == 1: - logger.level = WARNING - elif v == 2: - logger.level = DEBUG - - import signal, traceback - handler = lambda sig, stack: traceback.print_stack(stack) - signal.signal(signal.SIGQUIT, handler) - signal.signal(signal.SIGUSR1, handler) - - return check_current_mirrors() - -def parse_rfc3339_datetime(time): - # '2010-09-02 11:05:06+02:00' - m = re.match('^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time) - if m: - vals = m.groups() - parsed = datetime(int(vals[0]), int(vals[1]), int(vals[2]), - int(vals[3]), int(vals[4]), int(vals[5])) - # now account for time zone offset - sign = vals[6] - offset = timedelta(hours=int(sign + vals[7]), - minutes=int(sign + vals[8])) - # subtract the offset, e.g. '-04:00' should be moved up 4 hours - return parsed - offset - return None - -def check_mirror_url(mirror_url): + logger.level = logging.WARNING + elif v >= 2: + logger.level = logging.DEBUG + + timeout = options.get('timeout') + + urls = MirrorUrl.objects.select_related('protocol').filter( + active=True, mirror__active=True, mirror__public=True) + + location = options.get('location', None) + if location: + location = CheckLocation.objects.get(id=location) + family = location.family + monkeypatch_getaddrinfo(family) + if family == socket.AF_INET6: + urls = urls.filter(has_ipv6=True) + elif family == socket.AF_INET: + urls = urls.filter(has_ipv4=True) + + pool = MirrorCheckPool(urls, location, timeout) + pool.run() + return 0 + + +def monkeypatch_getaddrinfo(force_family=socket.AF_INET): + '''Force the Python socket module to connect over the designated family; + e.g. socket.AF_INET or socket.AF_INET6.''' + orig = socket.getaddrinfo + + def wrapper(host, port, family=0, socktype=0, proto=0, flags=0): + return orig(host, port, force_family, socktype, proto, flags) + + socket.getaddrinfo = wrapper + + +def parse_lastsync(log, data): + '''lastsync file should be an epoch value created by us.''' + try: + parsed_time = datetime.utcfromtimestamp(int(data)) + log.last_sync = parsed_time.replace(tzinfo=utc) + except (TypeError, ValueError): + # it is bad news to try logging the lastsync value; + # sometimes we get a crazy-encoded web page. + # if we couldn't parse a time, this is a failure. + log.last_sync = None + log.error = "Could not parse time from lastsync" + log.is_success = False + + +def check_mirror_url(mirror_url, location, timeout): url = mirror_url.url + 'lastsync' - logger.info("checking URL %s" % url) - log = MirrorLog(url=mirror_url, check_time=datetime.utcnow()) + logger.info("checking URL %s", url) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) + headers = {'User-Agent': 'archweb/1.0'} + req = urllib2.Request(url, None, headers) + start = time.time() try: - start = time.time() - result = urllib2.urlopen(url, timeout=10) + result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() end = time.time() - # lastsync should be an epoch value, but some mirrors - # are creating their own in RFC-3339 format: - # '2010-09-02 11:05:06+02:00' - try: - parsed_time = datetime.utcfromtimestamp(int(data)) - except ValueError: - # it is bad news to try logging the lastsync value; - # sometimes we get a crazy-encoded web page. - logger.info("attempting to parse generated lastsync file" - " from mirror %s" % url) - parsed_time = parse_rfc3339_datetime(data) - - log.last_sync = parsed_time + parse_lastsync(log, data) log.duration = end - start - logger.debug("success: %s, %.2f" % (url, log.duration)) - except urllib2.HTTPError, e: + logger.debug("success: %s, %.2f", url, log.duration) + except urllib2.HTTPError as e: + if e.code == 404: + # we have a duration, just not a success + end = time.time() + log.duration = end - start + log.is_success = False + log.error = str(e) + logger.debug("failed: %s, %s", url, log.error) + except urllib2.URLError as e: log.is_success = False - log.error =str(e) - logger.debug("failed: %s, %s" % (url, log.error)) - except urllib2.URLError, e: - log.is_success=False log.error = e.reason + if isinstance(e.reason, types.StringTypes) and \ + re.search(r'550.*No such file', e.reason): + # similar to 404 case above, still record duration + end = time.time() + log.duration = end - start if isinstance(e.reason, socket.timeout): log.error = "Connection timed out." elif isinstance(e.reason, socket.error): - log.error = e.reason.args[1] - logger.debug("failed: %s, %s" % (url, log.error)) + log.error = e.reason.args[-1] + logger.debug("failed: %s, %s", url, log.error) + except HTTPException: + # e.g., BadStatusLine + log.is_success = False + log.error = "Exception in processing HTTP request." + logger.debug("failed: %s, %s", url, log.error) + except ssl.CertificateError as e: + log.is_success = False + log.error = str(e) + logger.debug("failed: %s, %s", url, log.error) + except socket.timeout: + log.is_success = False + log.error = "Connection timed out." + logger.debug("failed: %s, %s", url, log.error) + except socket.error as e: + log.is_success = False + log.error = str(e) + logger.debug("failed: %s, %s", url, log.error) - log.save() return log -def mirror_url_worker(queue): + +def check_rsync_url(mirror_url, location, timeout): + url = mirror_url.url + 'lastsync' + logger.info("checking URL %s", url) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) + + tempdir = tempfile.mkdtemp() + ipopt = '' + if location: + if location.family == socket.AF_INET6: + ipopt = '--ipv6' + elif location.family == socket.AF_INET: + ipopt = '--ipv4' + lastsync_path = os.path.join(tempdir, 'lastsync') + rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout, + "--timeout=%d" % timeout] + if ipopt: + rsync_cmd.append(ipopt) + rsync_cmd.append(url) + rsync_cmd.append(lastsync_path) + try: + with open(os.devnull, 'w') as devnull: + if logger.isEnabledFor(logging.DEBUG): + logger.debug("rsync cmd: %s", ' '.join(rsync_cmd)) + start = time.time() + proc = subprocess.Popen(rsync_cmd, stdout=devnull, + stderr=subprocess.PIPE) + _, errdata = proc.communicate() + end = time.time() + log.duration = end - start + if proc.returncode != 0: + logger.debug("error: %s, %s", url, errdata) + log.is_success = False + log.error = errdata.strip() + # look at rsync error code- if we had a command error or timed out, + # don't record a duration as it is misleading + if proc.returncode in (1, 30, 35): + log.duration = None + else: + logger.debug("success: %s, %.2f", url, log.duration) + if os.path.exists(lastsync_path): + with open(lastsync_path, 'r') as lastsync: + parse_lastsync(log, lastsync.read()) + else: + parse_lastsync(log, None) + finally: + if os.path.exists(lastsync_path): + os.unlink(lastsync_path) + os.rmdir(tempdir) + + return log + + +def mirror_url_worker(work, output, location, timeout): while True: try: - item = queue.get(block=False) - check_mirror_url(item) - queue.task_done() + url = work.get(block=False) + try: + if url.protocol.protocol == 'rsync': + log = check_rsync_url(url, location, timeout) + elif (url.protocol.protocol == 'ftp' and location and + location.family == socket.AF_INET6): + # IPv6 + FTP don't work; skip checking completely + log = None + else: + log = check_mirror_url(url, location, timeout) + if log: + output.append(log) + finally: + work.task_done() except Empty: return 0 + class MirrorCheckPool(object): - def __init__(self, work, num_threads=10): + def __init__(self, urls, location, timeout=10, num_threads=10): self.tasks = Queue() - for i in work: - self.tasks.put(i) + self.logs = deque() + for url in list(urls): + self.tasks.put(url) self.threads = [] - for i in range(num_threads): - thread = Thread(target=mirror_url_worker, args=(self.tasks,)) + for _ in range(num_threads): + thread = Thread(target=mirror_url_worker, + args=(self.tasks, self.logs, location, timeout)) thread.daemon = True self.threads.append(thread) - def run_and_join(self): + @transaction.atomic + def run(self): logger.debug("starting threads") - for t in self.threads: - t.start() + for thread in self.threads: + thread.start() logger.debug("joining on all threads") self.tasks.join() - -def check_current_mirrors(): - urls = MirrorUrl.objects.filter( - Q(protocol__protocol__iexact='HTTP') | - Q(protocol__protocol__iexact='FTP'), - mirror__active=True, mirror__public=True) - - pool = MirrorCheckPool(urls) - pool.run_and_join() - return 0 - -# For lack of a better place to put it, here is a query to get latest check -# result joined with mirror details: -# SELECT mu.*, m.*, ml.* FROM mirrors_mirrorurl mu JOIN mirrors_mirror m ON mu.mirror_id = m.id JOIN mirrors_mirrorlog ml ON mu.id = ml.url_id LEFT JOIN mirrors_mirrorlog ml2 ON ml.url_id = ml2.url_id AND ml.id < ml2.id WHERE ml2.id IS NULL AND m.active = 1 AND m.public = 1; + logger.debug("processing %d log entries", len(self.logs)) + MirrorLog.objects.bulk_create(self.logs) + logger.debug("log entries saved") # vim: set ts=4 sw=4 et: diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py new file mode 100644 index 00000000..0e71894b --- /dev/null +++ b/mirrors/management/commands/mirrorresolv.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +""" +mirrorresolv command + +Poll all mirror URLs and determine whether they have IPv4 and/or IPv6 addresses +available. + +Usage: ./manage.py mirrorresolv +""" + +from django.core.management.base import NoArgsCommand + +import sys +import logging +import socket + +from mirrors.models import MirrorUrl + +logging.basicConfig( + level=logging.WARNING, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(NoArgsCommand): + help = "Runs a check on all active mirror URLs to determine if they are reachable via IPv4 and/or v6." + + def handle_noargs(self, **options): + v = int(options.get('verbosity', 0)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.WARNING + elif v >= 2: + logger.level = logging.DEBUG + + return resolve_mirrors() + +def resolve_mirrors(): + logger.debug("requesting list of mirror URLs") + for mirrorurl in MirrorUrl.objects.filter(active=True, mirror__active=True): + try: + # save old values, we can skip no-op updates this way + oldvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) + logger.debug("resolving %3i (%s)", mirrorurl.id, mirrorurl.hostname) + families = mirrorurl.address_families() + mirrorurl.has_ipv4 = socket.AF_INET in families + mirrorurl.has_ipv6 = socket.AF_INET6 in families + logger.debug("%s: v4: %s v6: %s", mirrorurl.hostname, + mirrorurl.has_ipv4, mirrorurl.has_ipv6) + # now check new values, only update if new != old + newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) + if newvals != oldvals: + logger.debug("values changed for %s", mirrorurl) + mirrorurl.save(update_fields=('has_ipv4', 'has_ipv6')) + except socket.gaierror, e: + if e.errno == socket.EAI_NONAME: + logger.debug("gaierror resolving %s: %s", mirrorurl.hostname, e) + else: + logger.warn("gaierror resolving %s: %s", mirrorurl.hostname, e) + except socket.error, e: + logger.warn("error resolving %s: %s", mirrorurl.hostname, e) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/migrations/0001_initial.py b/mirrors/migrations/0001_initial.py index 4a3173c5..6f36c9eb 100644 --- a/mirrors/migrations/0001_initial.py +++ b/mirrors/migrations/0001_initial.py @@ -1,17 +1,123 @@ -# encoding: utf-8 -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models +# -*- coding: utf-8 -*- +from __future__ import unicode_literals -class Migration(SchemaMigration): +from django.db import models, migrations +import django_countries.fields +import django.db.models.deletion +import mirrors.fields - def forwards(self, orm): - pass - def backwards(self, orm): - pass +class Migration(migrations.Migration): - models = {} + dependencies = [ + ] - complete_apps = ['mirrors'] + operations = [ + migrations.CreateModel( + name='CheckLocation', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('hostname', models.CharField(max_length=255)), + ('source_ip', models.GenericIPAddressField(unique=True, verbose_name=b'source IP', unpack_ipv4=True)), + ('country', django_countries.fields.CountryField(max_length=2)), + ('created', models.DateTimeField(editable=False)), + ], + options={ + 'ordering': ('hostname', 'source_ip'), + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Mirror', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(unique=True, max_length=255)), + ('tier', models.SmallIntegerField(default=2, choices=[(0, b'Tier 0'), (1, b'Tier 1'), (2, b'Tier 2'), (-1, b'Untiered')])), + ('admin_email', models.EmailField(max_length=255, blank=True)), + ('alternate_email', models.EmailField(max_length=255, blank=True)), + ('public', models.BooleanField(default=True)), + ('active', models.BooleanField(default=True)), + ('isos', models.BooleanField(default=True, verbose_name=b'ISOs')), + ('rsync_user', models.CharField(default=b'', max_length=50, blank=True)), + ('rsync_password', models.CharField(default=b'', max_length=50, blank=True)), + ('bug', models.PositiveIntegerField(null=True, verbose_name=b'Flyspray bug', blank=True)), + ('notes', models.TextField(blank=True)), + ('created', models.DateTimeField(editable=False)), + ('last_modified', models.DateTimeField(editable=False)), + ('upstream', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to='mirrors.Mirror', null=True)), + ], + options={ + 'ordering': ('name',), + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='MirrorLog', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('check_time', models.DateTimeField(db_index=True)), + ('last_sync', models.DateTimeField(null=True)), + ('duration', models.FloatField(null=True)), + ('is_success', models.BooleanField(default=True)), + ('error', models.TextField(default=b'', blank=True)), + ('location', models.ForeignKey(related_name=b'logs', to='mirrors.CheckLocation', null=True)), + ], + options={ + 'get_latest_by': 'check_time', + 'verbose_name': 'mirror check log', + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='MirrorProtocol', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('protocol', models.CharField(unique=True, max_length=10)), + ('is_download', models.BooleanField(default=True, help_text=b'Is protocol useful for end-users, e.g. HTTP')), + ('default', models.BooleanField(default=True, help_text=b'Included by default when building mirror list?')), + ('created', models.DateTimeField(editable=False)), + ], + options={ + 'ordering': ('protocol',), + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='MirrorRsync', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('ip', mirrors.fields.IPNetworkField(max_length=44, verbose_name=b'IP')), + ('created', models.DateTimeField(editable=False)), + ('mirror', models.ForeignKey(related_name=b'rsync_ips', to='mirrors.Mirror')), + ], + options={ + 'ordering': ('ip',), + 'verbose_name': 'mirror rsync IP', + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='MirrorUrl', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('url', models.CharField(unique=True, max_length=255, verbose_name=b'URL')), + ('country', django_countries.fields.CountryField(blank=True, max_length=2, db_index=True)), + ('has_ipv4', models.BooleanField(default=True, verbose_name=b'IPv4 capable', editable=False)), + ('has_ipv6', models.BooleanField(default=False, verbose_name=b'IPv6 capable', editable=False)), + ('created', models.DateTimeField(editable=False)), + ('active', models.BooleanField(default=True)), + ('mirror', models.ForeignKey(related_name=b'urls', to='mirrors.Mirror')), + ('protocol', models.ForeignKey(related_name=b'urls', on_delete=django.db.models.deletion.PROTECT, editable=False, to='mirrors.MirrorProtocol')), + ], + options={ + 'verbose_name': 'mirror URL', + }, + bases=(models.Model,), + ), + migrations.AddField( + model_name='mirrorlog', + name='url', + field=models.ForeignKey(related_name=b'logs', to='mirrors.MirrorUrl'), + preserve_default=True, + ), + ] diff --git a/mirrors/migrations/0002_mirrorurl_bandwidth.py b/mirrors/migrations/0002_mirrorurl_bandwidth.py new file mode 100644 index 00000000..f0118199 --- /dev/null +++ b/mirrors/migrations/0002_mirrorurl_bandwidth.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('mirrors', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='mirrorurl', + name='bandwidth', + field=models.FloatField(null=True, verbose_name=b'bandwidth (mbits)', blank=True), + preserve_default=True, + ), + ] diff --git a/mirrors/migrations/0002_rename_model_tables.py b/mirrors/migrations/0002_rename_model_tables.py deleted file mode 100644 index d510bada..00000000 --- a/mirrors/migrations/0002_rename_model_tables.py +++ /dev/null @@ -1,61 +0,0 @@ -# encoding: utf-8 -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models - -class Migration(SchemaMigration): - - depends_on = ( - ('main', '0014_mirror_notes_rsync_optional'), - ) - - def forwards(self, orm): - db.rename_table('main_mirror', 'mirrors_mirror') - db.rename_table('main_mirrorurl', 'mirrors_mirrorurl') - db.rename_table('main_mirrorrsync', 'mirrors_mirrorrsync') - db.rename_table('main_mirrorprotocol', 'mirrors_mirrorprotocol') - - def backwards(self, orm): - db.rename_table('mirrors_mirror', 'main_mirror') - db.rename_table('mirrors_mirrorurl', 'main_mirrorurl') - db.rename_table('mirrors_mirrorrsync', 'main_mirrorrsync') - db.rename_table('mirrors_mirrorprotocol', 'main_mirrorprotocol') - - models = { - 'mirrors.mirror': { - 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, - 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), - 'country': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), - 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), - 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), - 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), - 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True'}) - }, - 'mirrors.mirrorprotocol': { - 'Meta': {'object_name': 'MirrorProtocol'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) - }, - 'mirrors.mirrorrsync': { - 'Meta': {'object_name': 'MirrorRsync'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), - 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) - }, - 'mirrors.mirrorurl': { - 'Meta': {'object_name': 'MirrorUrl'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), - 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.MirrorProtocol']"}), - 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - } - } - - complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0003_auto__add_mirrorlog.py b/mirrors/migrations/0003_auto__add_mirrorlog.py deleted file mode 100644 index 5b4c225b..00000000 --- a/mirrors/migrations/0003_auto__add_mirrorlog.py +++ /dev/null @@ -1,72 +0,0 @@ -# encoding: utf-8 -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models - -class Migration(SchemaMigration): - - def forwards(self, orm): - # Adding model 'MirrorLog' - db.create_table('mirrors_mirrorlog', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('url', self.gf('django.db.models.fields.related.ForeignKey')(related_name='logs', to=orm['mirrors.MirrorUrl'])), - ('check_time', self.gf('django.db.models.fields.DateTimeField')(db_index=True)), - ('last_sync', self.gf('django.db.models.fields.DateTimeField')(null=True)), - ('duration', self.gf('django.db.models.fields.FloatField')(null=True)), - ('is_success', self.gf('django.db.models.fields.BooleanField')(default=True)), - ('error', self.gf('django.db.models.fields.CharField')(default='', max_length=255, blank=True)), - )) - db.send_create_signal('mirrors', ['MirrorLog']) - - def backwards(self, orm): - # Deleting model 'MirrorLog' - db.delete_table('mirrors_mirrorlog') - - models = { - 'mirrors.mirror': { - 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, - 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), - 'country': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), - 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), - 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), - 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), - 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True'}) - }, - 'mirrors.mirrorlog': { - 'Meta': {'object_name': 'MirrorLog'}, - 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), - 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), - 'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), - 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) - }, - 'mirrors.mirrorprotocol': { - 'Meta': {'object_name': 'MirrorProtocol'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) - }, - 'mirrors.mirrorrsync': { - 'Meta': {'object_name': 'MirrorRsync'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), - 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) - }, - 'mirrors.mirrorurl': { - 'Meta': {'object_name': 'MirrorUrl'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), - 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.MirrorProtocol']"}), - 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - } - } - - complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 85423303..9743d177 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,78 +1,189 @@ +from datetime import timedelta +import socket +from urlparse import urlparse + +from django.core.exceptions import ValidationError from django.db import models +from django.db.models.signals import pre_save +from django_countries.fields import CountryField + +from .fields import IPNetworkField +from main.utils import set_created_field -TIER_CHOICES = ( - (0, 'Tier 0'), - (1, 'Tier 1'), - (2, 'Tier 2'), - (-1, 'Untiered'), -) class Mirror(models.Model): - name = models.CharField(max_length=255) + TIER_CHOICES = ( + (0, 'Tier 0'), + (1, 'Tier 1'), + (2, 'Tier 2'), + (-1, 'Untiered'), + ) + + name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) - upstream = models.ForeignKey('self', null=True) - country = models.CharField(max_length=255, db_index=True) + upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) admin_email = models.EmailField(max_length=255, blank=True) + alternate_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) active = models.BooleanField(default=True) - isos = models.BooleanField(default=True) + isos = models.BooleanField("ISOs", default=True) rsync_user = models.CharField(max_length=50, blank=True, default='') rsync_password = models.CharField(max_length=50, blank=True, default='') + bug = models.PositiveIntegerField("Flyspray bug", null=True, blank=True) notes = models.TextField(blank=True) + created = models.DateTimeField(editable=False) + last_modified = models.DateTimeField(editable=False) class Meta: - ordering = ('country', 'name') + ordering = ('name',) def __unicode__(self): return self.name - def supported_protocols(self): - protocols = MirrorProtocol.objects.filter( - urls__mirror=self).order_by('protocol').distinct() - return ", ".join([p.protocol for p in protocols]) - def downstream(self): return Mirror.objects.filter(upstream=self).order_by('name') def get_absolute_url(self): return '/mirrors/%s/' % self.name + class MirrorProtocol(models.Model): protocol = models.CharField(max_length=10, unique=True) + is_download = models.BooleanField(default=True, + help_text="Is protocol useful for end-users, e.g. HTTP") + default = models.BooleanField(default=True, + help_text="Included by default when building mirror list?") + created = models.DateTimeField(editable=False) + def __unicode__(self): return self.protocol + class Meta: - verbose_name = 'Mirror Protocol' + ordering = ('protocol',) + class MirrorUrl(models.Model): - url = models.CharField(max_length=255) - protocol = models.ForeignKey(MirrorProtocol, related_name="urls") + url = models.CharField("URL", max_length=255, unique=True) + protocol = models.ForeignKey(MirrorProtocol, related_name="urls", + editable=False, on_delete=models.PROTECT) mirror = models.ForeignKey(Mirror, related_name="urls") + country = CountryField(blank=True, db_index=True) + has_ipv4 = models.BooleanField("IPv4 capable", default=True, + editable=False) + has_ipv6 = models.BooleanField("IPv6 capable", default=False, + editable=False) + active = models.BooleanField(default=True) + bandwidth = models.FloatField("bandwidth (mbits)", null=True, blank=True) + created = models.DateTimeField(editable=False) + + class Meta: + verbose_name = 'mirror URL' + def __unicode__(self): return self.url - class Meta: - verbose_name = 'Mirror URL' + + def address_families(self): + hostname = urlparse(self.url).hostname + info = socket.getaddrinfo(hostname, None, 0, socket.SOCK_STREAM) + families = [x[0] for x in info] + return families + + @property + def hostname(self): + return urlparse(self.url).hostname + + def clean(self): + try: + # Auto-map the protocol field by looking at the URL + protocol = urlparse(self.url).scheme + self.protocol = MirrorProtocol.objects.get(protocol=protocol) + except Exception as e: + raise ValidationError(e) + try: + families = self.address_families() + self.has_ipv4 = socket.AF_INET in families + self.has_ipv6 = socket.AF_INET6 in families + except socket.error: + # We don't fail in this case; we'll just set both to False + self.has_ipv4 = False + self.has_ipv6 = False + + def get_absolute_url(self): + return '/mirrors/%s/%d/' % (self.mirror.name, self.pk) + class MirrorRsync(models.Model): - ip = models.CharField(max_length=24) + # max length is 40 chars for full-form IPv6 addr + subnet + ip = IPNetworkField("IP") mirror = models.ForeignKey(Mirror, related_name="rsync_ips") + created = models.DateTimeField(editable=False) + def __unicode__(self): - return "%s" % (self.ip) + return unicode(self.ip) + + class Meta: + verbose_name = 'mirror rsync IP' + ordering = ('ip',) + + +class CheckLocation(models.Model): + hostname = models.CharField(max_length=255) + source_ip = models.GenericIPAddressField('source IP', + unpack_ipv4=True, unique=True) + country = CountryField() + created = models.DateTimeField(editable=False) + class Meta: - verbose_name = 'Mirror Rsync IP' + ordering = ('hostname', 'source_ip') + + def __unicode__(self): + return self.hostname + + @property + def family(self): + info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0, + socket.AI_NUMERICHOST) + families = [x[0] for x in info] + return families[0] + + @property + def ip_version(self): + '''Returns integer '4' or '6'.''' + if self.family == socket.AF_INET6: + return 6 + if self.family == socket.AF_INET: + return 4 + return None + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") + location = models.ForeignKey(CheckLocation, related_name="logs", null=True) check_time = models.DateTimeField(db_index=True) last_sync = models.DateTimeField(null=True) duration = models.FloatField(null=True) is_success = models.BooleanField(default=True) - error = models.CharField(max_length=255, blank=True, default='') + error = models.TextField(blank=True, default='') + + @property + def delay(self): + if self.last_sync is None: + return None + # sanity check, this shouldn't happen + if self.check_time < self.last_sync: + return timedelta() + return self.check_time - self.last_sync def __unicode__(self): - return "Check of %s at %s" % (url.url, check_time) + return "Check of %s at %s" % (self.url.url, self.check_time) class Meta: - verbose_name = 'Mirror Check Log' + verbose_name = 'mirror check log' + get_latest_by = 'check_time' + + +for model in (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, CheckLocation): + pre_save.connect(set_created_field, sender=model, + dispatch_uid="mirrors.models") # vim: set ts=4 sw=4 et: diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js new file mode 100644 index 00000000..44c81935 --- /dev/null +++ b/mirrors/static/mirror_status.js @@ -0,0 +1,193 @@ +/* mirror_status.js + * Homepage: https://projects.archlinux.org/archweb.git/ + * Copyright: 2012-2013 The Archweb Team (Dan McGee) + * License: GPLv2 + * + * This file is part of Archweb. + * + * Archweb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Archweb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Archweb. If not, see <http://www.gnu.org/licenses/>. + */ + +function draw_graphs(location_url, log_url, container_id) { + jQuery.when(jQuery.getJSON(location_url), jQuery.getJSON(log_url)) + .then(function(loc_data, log_data) { + /* use the same color selection for a given URL in every graph */ + var color = d3.scale.category10(); + jQuery.each(loc_data[0].locations, function(i, val) { + mirror_status(container_id, val, log_data[0], color); + }); + }); +} + +function mirror_status(container_id, check_loc, log_data, color) { + + var draw_graph = function(chart_id, data) { + var jq_div = jQuery(chart_id); + var margin = {top: 20, right: 20, bottom: 30, left: 40}, + width = jq_div.width() - margin.left - margin.right, + height = jq_div.height() - margin.top - margin.bottom; + + var x = d3.time.scale.utc().range([0, width]), + y = d3.scale.linear().range([height, 0]), + x_axis = d3.svg.axis().scale(x).orient("bottom"), + y_axis = d3.svg.axis().scale(y).orient("left"); + + /* remove any existing graph first if we are redrawing after resize */ + d3.select(chart_id).select("svg").remove(); + var svg = d3.select(chart_id).append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + x.domain([ + d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.check_time; }); }), + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) + ]).nice(d3.time.hour); + y.domain([ + 0, + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) + ]).nice(); + + /* build the axis lines... */ + svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(x_axis) + .append("text") + .attr("class", "label") + .attr("x", width) + .attr("y", -6) + .style("text-anchor", "end") + .text("Check Time (UTC)"); + + svg.append("g") + .attr("class", "y axis") + .call(y_axis) + .append("text") + .attr("class", "label") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text("Duration (seconds)"); + + var line = d3.svg.line() + .interpolate("basis") + .x(function(d) { return x(d.check_time); }) + .y(function(d) { return y(d.duration); }); + + /* ...then the points and lines between them. */ + var urls = svg.selectAll(".url") + .data(data) + .enter() + .append("g") + .attr("class", "url"); + + urls.append("path") + .attr("class", "url-line") + .attr("d", function(d) { return line(d.logs); }) + .style("stroke", function(d) { return color(d.url); }); + + urls.selectAll("circle") + .data(function(u) { + return jQuery.map(u.logs, function(l, i) { + return {url: u.url, check_time: l.check_time, duration: l.duration}; + }); + }) + .enter() + .append("circle") + .attr("class", "url-dot") + .attr("r", 3.5) + .attr("cx", function(d) { return x(d.check_time); }) + .attr("cy", function(d) { return y(d.duration); }) + .style("fill", function(d) { return color(d.url); }) + .append("title") + .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); }); + + /* add a legend for good measure */ + var active = jQuery.map(data, function(item, i) { return item.url; }); + var legend = svg.selectAll(".legend") + .data(active) + .enter().append("g") + .attr("class", "legend") + .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); + + legend.append("rect") + .attr("x", width - 18) + .attr("width", 18) + .attr("height", 18) + .style("fill", color); + + legend.append("text") + .attr("x", width - 24) + .attr("y", 9) + .attr("dy", ".35em") + .style("text-anchor", "end") + .text(function(d) { return d; }); + }; + + var filter_data = function(json, location_id) { + return jQuery.map(json.urls, function(url, i) { + var logs = jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } + /* screen by location ID if we were given one */ + if (location_id && log.location_id !== location_id) { + return null; + } + return { + duration: log.duration, + check_time: new Date(log.check_time) + }; + }); + /* don't return URLs without any log info */ + if (logs.length === 0) { + return null; + } + return { + url: url.url, + logs: logs + }; + }); + }; + + var cached_data = filter_data(log_data, check_loc.id); + /* we had a check location with no log data handed to us, skip graphing */ + if (cached_data.length === 0) { + return; + } + + /* create the containers, defer the actual graph drawing */ + var chart_id = 'status-chart-' + check_loc.id; + jQuery(container_id).append('<h3><span class="fam-flag fam-flag-' + check_loc.country_code.toLowerCase() + '" title="' + check_loc.country + '"></span> ' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '</h3>'); + jQuery(container_id).append('<div id="' + chart_id + '" class="visualize-mirror visualize-chart"></div>'); + jQuery(container_id).append('<br/>'); + setTimeout(function() { + draw_graph('#' + chart_id, cached_data); + }, 0); + + /* then hook up a resize handler to redraw if necessary */ + var resize_timeout = null; + var real_resize = function() { + resize_timeout = null; + draw_graph('#' + chart_id, cached_data); + }; + jQuery(window).resize(function() { + if (resize_timeout) { + clearTimeout(resize_timeout); + } + resize_timeout = setTimeout(real_resize, 200); + }); +} diff --git a/mirrors/templatetags/jinja2.py b/mirrors/templatetags/jinja2.py new file mode 100644 index 00000000..04e50238 --- /dev/null +++ b/mirrors/templatetags/jinja2.py @@ -0,0 +1,53 @@ +from datetime import timedelta +from django_jinja import library +from markupsafe import Markup + + +@library.global_function +def country_flag(country): + if not country: + return '' + html = '<span class="fam-flag fam-flag-%s" title="%s"></span> ' % ( + unicode(country.code).lower(), unicode(country.name)) + return Markup(html) + + +@library.filter +def duration(value): + if not value and type(value) != timedelta: + return u'' + # does not take microseconds into account + total_secs = value.seconds + value.days * 24 * 3600 + mins = total_secs // 60 + hrs, mins = divmod(mins, 60) + return '%d:%02d' % (hrs, mins) + + +@library.filter +def hours(value): + if not value and type(value) != timedelta: + return u'' + # does not take microseconds into account + total_secs = value.seconds + value.days * 24 * 3600 + mins = total_secs // 60 + hrs, mins = divmod(mins, 60) + if hrs == 1: + return '%d hour' % hrs + return '%d hours' % hrs + + +@library.filter +def floatvalue(value, arg=2): + if value is None: + return u'' + return '%.*f' % (arg, value) + + +@library.filter +def percentage(value, arg=1): + if not value and type(value) != float: + return u'' + new_val = value * 100.0 + return '%.*f%%' % (arg, new_val) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/templatetags/mirror_status.py b/mirrors/templatetags/mirror_status.py index 0031d83b..c8004e4b 100644 --- a/mirrors/templatetags/mirror_status.py +++ b/mirrors/templatetags/mirror_status.py @@ -1,6 +1,5 @@ from datetime import timedelta from django import template -from django.template.defaultfilters import floatformat register = template.Library() @@ -10,7 +9,7 @@ def duration(value): return u'' # does not take microseconds into account total_secs = value.seconds + value.days * 24 * 3600 - mins, secs = divmod(total_secs, 60) + mins = total_secs // 60 hrs, mins = divmod(mins, 60) return '%d:%02d' % (hrs, mins) @@ -20,17 +19,16 @@ def hours(value): return u'' # does not take microseconds into account total_secs = value.seconds + value.days * 24 * 3600 - mins, secs = divmod(total_secs, 60) + mins = total_secs // 60 hrs, mins = divmod(mins, 60) if hrs == 1: return '%d hour' % hrs return '%d hours' % hrs @register.filter -def percentage(value, arg=-1): - if not value and type(value) != float: +def floatvalue(value, arg=2): + if value is None: return u'' - new_val = value * 100.0 - return floatformat(new_val, arg) + '%' + return '%.*f' % (arg, value) # vim: set ts=4 sw=4 et: diff --git a/mirrors/urls.py b/mirrors/urls.py new file mode 100644 index 00000000..fc510fbb --- /dev/null +++ b/mirrors/urls.py @@ -0,0 +1,18 @@ +from django.conf.urls import patterns + +from .views import mirrors, status, mirror_details, url_details +from .views.api import status_json, mirror_details_json, locations_json + +urlpatterns = patterns('', + (r'^$', mirrors, {}, 'mirror-list'), + (r'^status/$', status, {}, 'mirror-status'), + (r'^status/json/$', status_json, {}, 'mirror-status-json'), + (r'^status/tier/(?P<tier>\d+)/$', status, {}, 'mirror-status-tier'), + (r'^status/tier/(?P<tier>\d+)/json/$', status_json, {}, 'mirror-status-tier-json'), + (r'^locations/json/$', locations_json, {}, 'mirror-locations-json'), + (r'^(?P<name>[\.\-\w]+)/$', mirror_details), + (r'^(?P<name>[\.\-\w]+)/json/$', mirror_details_json), + (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', url_details), +) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py new file mode 100644 index 00000000..a64656a9 --- /dev/null +++ b/mirrors/urls_mirrorlist.py @@ -0,0 +1,11 @@ +from django.conf.urls import patterns + + +urlpatterns = patterns('mirrors.views.mirrorlist', + (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), + (r'^all/$', 'find_mirrors', {'countries': ['all']}), + (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple', + {}, 'mirrorlist_simple') +) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/utils.py b/mirrors/utils.py index 0463247a..7c2f5d17 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,64 +1,150 @@ -from django.db.models import Avg, Count, Max, Min, StdDev - -from main.utils import cache_function -from .models import MirrorLog, MirrorProtocol, MirrorUrl - -import datetime - -default_cutoff = datetime.timedelta(hours=24) - -@cache_function(300) -def get_mirror_statuses(cutoff=default_cutoff): - cutoff_time = datetime.datetime.utcnow() - cutoff - protocols = MirrorProtocol.objects.exclude(protocol__iexact='rsync') - # I swear, this actually has decent performance... - urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( - mirror__active=True, mirror__public=True, - protocol__in=protocols, - logs__check_time__gte=cutoff_time).annotate( - check_count=Count('logs'), - success_count=Count('logs__duration'), - last_sync=Max('logs__last_sync'), - last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration'), - duration_stddev=StdDev('logs__duration') - ).order_by('-last_sync', '-duration_avg') - - # The Django ORM makes it really hard to get actual average delay in the - # above query, so run a seperate query for it and we will process the - # results here. - times = MirrorLog.objects.filter(is_success=True, last_sync__isnull=False, - check_time__gte=cutoff_time) - delays = {} - for log in times: - d = log.check_time - log.last_sync - delays.setdefault(log.url_id, []).append(d) +from datetime import timedelta + +from django.db import connection +from django.db.models import Count, Max, Min +from django.utils.dateparse import parse_datetime +from django.utils.timezone import now + +from main.utils import cache_function, database_vendor +from .models import MirrorLog, MirrorUrl + + +DEFAULT_CUTOFF = timedelta(hours=24) + + +def dictfetchall(cursor): + "Returns all rows from a cursor as a dict." + desc = cursor.description + return [ + dict(zip([col[0] for col in desc], row)) + for row in cursor.fetchall() + ] + +def status_data(cutoff_time, mirror_id=None): + if mirror_id is not None: + params = [cutoff_time, mirror_id] + mirror_where = 'AND u.mirror_id = %s' + else: + params = [cutoff_time] + mirror_where = '' + + vendor = database_vendor(MirrorUrl) + if vendor == 'sqlite': + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.last_sync) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + 0.0 AS duration_stddev, + AVG(STRFTIME('%%s', check_time) - STRFTIME('%%s', last_sync)) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id +""" + else: + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.last_sync) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + STDDEV(l.duration) AS duration_stddev, + AVG(check_time - last_sync) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id +""" + + cursor = connection.cursor() + cursor.execute(sql, params) + url_data = dictfetchall(cursor) + + # sqlite loves to return less than ideal types + if vendor == 'sqlite': + for item in url_data: + if item['delay'] is not None: + item['delay'] = timedelta(seconds=item['delay']) + if item['last_sync'] is not None: + item['last_sync'] = parse_datetime(item['last_sync']) + item['last_check'] = parse_datetime(item['last_check']) + + return {item['url_id']: item for item in url_data} + + +def annotate_url(url, url_data): + '''Given a MirrorURL object, add a few more attributes to it regarding + status, including completion_pct, delay, and score.''' + # set up some sane default values in case we are missing anything + url.success_count = 0 + url.check_count = 0 + url.completion_pct = None + url.duration_avg = None + url.duration_stddev = None + url.last_check = None + url.last_sync = None + url.delay = None + url.score = None + for k, v in url_data.items(): + if k not in ('url_id', 'mirror_id'): + setattr(url, k, v) + + if url.check_count > 0: + url.completion_pct = float(url.success_count) / url.check_count + + if url.delay is not None: + hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 + + if url.completion_pct > 0.0: + divisor = url.completion_pct + else: + # arbitrary small value + divisor = 0.005 + stddev = url.duration_stddev or 0.0 + url.score = (hours + url.duration_avg + stddev) / divisor + + return url + + +@cache_function(178) +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): + cutoff_time = now() - cutoff + + urls = MirrorUrl.objects.select_related( + 'mirror', 'protocol').order_by('mirror__id', 'url') + if mirror_id: + urls = urls.filter(mirror_id=mirror_id) + if not show_all: + urls = urls.filter(active=True, mirror__active=True, + mirror__public=True) if urls: - last_check = max([u.last_check for u in urls]) - num_checks = max([u.check_count for u in urls]) - check_info = MirrorLog.objects.filter( - check_time__gte=cutoff_time).aggregate( + url_data = status_data(cutoff_time, mirror_id) + urls = [annotate_url(url, url_data.get(url.id, {})) for url in urls] + last_check = max([u.last_check for u in urls if u.last_check] or [None]) + num_checks = max(u.check_count for u in urls) + check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) + if mirror_id: + check_info = check_info.filter(url__mirror_id=mirror_id) + check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) - check_frequency = (check_info['mx'] - check_info['mn']) / num_checks + if num_checks > 1: + check_frequency = (check_info['mx'] - check_info['mn']) \ + / (num_checks - 1) + else: + check_frequency = None else: + urls = [] last_check = None num_checks = 0 check_frequency = None - for url in urls: - url.completion_pct = float(url.success_count) / num_checks - if url.id in delays: - url_delays = delays[url.id] - d = sum(url_delays, datetime.timedelta()) / len(url_delays) - url.delay = d - hours = d.days * 24.0 + d.seconds / 3600.0 - url.score = hours + url.duration_avg + url.duration_stddev - else: - url.delay = None - url.score = None - url.completion = 0.0 - return { 'cutoff': cutoff, 'last_check': last_check, @@ -67,16 +153,57 @@ def get_mirror_statuses(cutoff=default_cutoff): 'urls': urls, } -@cache_function(300) -def get_mirror_errors(cutoff=default_cutoff): - cutoff_time = datetime.datetime.utcnow() - cutoff + +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): + cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, - url__mirror__active=True, url__mirror__public=True).values( - 'url__url', 'url__protocol__protocol', 'url__mirror__country', - 'error').annotate( + url__mirror__public=True).values('url__id', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') - return list(errors) + + if mirror_id: + errors = errors.filter(url__mirror_id=mirror_id) + if not show_all: + errors = errors.filter(url__active=True, url__mirror__active=True, + url__mirror__public=True) + + errors = list(errors) + to_fetch = [err['url__id'] for err in errors] + urls = MirrorUrl.objects.select_related( + 'mirror', 'protocol').in_bulk(to_fetch) + for err in errors: + err['url'] = urls[err['url__id']] + return errors + + +@cache_function(295) +def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): + '''Find a good mirror URL to use for package downloads. If we have mirror + status data available, it is used to determine a good choice by looking at + the last batch of status rows.''' + cutoff_time = now() - cutoff + log_data = MirrorLog.objects.filter( + check_time__gte=cutoff_time).aggregate( + Max('check_time'), Max('last_sync')) + if log_data['check_time__max'] is not None: + min_check_time = log_data['check_time__max'] - timedelta(minutes=5) + min_sync_time = log_data['last_sync__max'] - timedelta(minutes=20) + best_logs = MirrorLog.objects.select_related('url').filter( + is_success=True, + check_time__gte=min_check_time, last_sync__gte=min_sync_time, + url__active=True, + url__mirror__public=True, url__mirror__active=True, + url__protocol__default=True).order_by( + 'duration')[:1] + if best_logs: + return best_logs[0].url + + mirror_urls = MirrorUrl.objects.filter(active=True, + mirror__public=True, mirror__active=True, + protocol__default=True)[:1] + if not mirror_urls: + return None + return mirror_urls[0] # vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py deleted file mode 100644 index 34385a98..00000000 --- a/mirrors/views.py +++ /dev/null @@ -1,113 +0,0 @@ -from django import forms -from django.db.models import Avg, Count, Max, Min, StdDev -from django.db.models import Q -from django.http import Http404 -from django.shortcuts import get_object_or_404 -from django.views.decorators.csrf import csrf_exempt -from django.views.generic.simple import direct_to_template - -from main.utils import make_choice -from .models import Mirror, MirrorUrl, MirrorProtocol -from .utils import get_mirror_statuses, get_mirror_errors - -import datetime -from operator import attrgetter - -class MirrorlistForm(forms.Form): - country = forms.MultipleChoiceField(required=False) - protocol = forms.MultipleChoiceField(required=False) - use_mirror_status = forms.BooleanField(required=False) - - def __init__(self, *args, **kwargs): - super(MirrorlistForm, self).__init__(*args, **kwargs) - mirrors = Mirror.objects.filter(active=True).values_list( - 'country', flat=True).distinct().order_by('country') - self.fields['country'].choices = make_choice(mirrors) - self.fields['country'].initial = ['Any'] - protos = make_choice( - MirrorProtocol.objects.exclude(protocol__iexact='rsync')) - self.fields['protocol'].choices = protos - self.fields['protocol'].initial = [t[0] for t in protos] - -@csrf_exempt -def generate_mirrorlist(request): - if request.REQUEST.get('country', ''): - form = MirrorlistForm(data=request.REQUEST) - if form.is_valid(): - countries = form.cleaned_data['country'] - protocols = form.cleaned_data['protocol'] - use_status = form.cleaned_data['use_mirror_status'] - return find_mirrors(request, countries, protocols, use_status) - else: - form = MirrorlistForm() - - return direct_to_template(request, 'mirrors/index.html', {'mirrorlist_form': form}) - -def find_mirrors(request, countries=None, protocols=None, use_status=False): - if not protocols: - protocols = MirrorProtocol.objects.exclude( - protocol__iexact='rsync').values_list('protocol', flat=True) - qset = MirrorUrl.objects.select_related().filter( - protocol__protocol__in=protocols, - mirror__public=True, mirror__active=True, mirror__isos=True - ) - if countries and 'all' not in countries: - qset = qset.filter(mirror__country__in=countries) - if not use_status: - urls = qset.order_by('mirror__country', 'mirror__name', 'url') - template = 'mirrors/mirrorlist.txt' - else: - status_info = get_mirror_statuses() - scores = dict([(u.id, u.score) for u in status_info['urls']]) - urls = [] - for u in qset: - u.score = scores[u.id] - if u.score and u.score < 100.0: - urls.append(u) - urls = sorted(urls, key=attrgetter('score')) - template = 'mirrors/mirrorlist_status.txt' - - return direct_to_template(request, template, { - 'mirror_urls': urls, - }, - mimetype='text/plain') - -def mirrors(request): - mirrors = Mirror.objects.select_related().order_by('tier', 'country') - if not request.user.is_authenticated(): - mirrors = mirrors.filter(public=True, active=True) - return direct_to_template(request, 'mirrors/mirrors.html', - {'mirror_list': mirrors}) - -def mirror_details(request, name): - mirror = get_object_or_404(Mirror, name=name) - if not request.user.is_authenticated() and \ - (not mirror.public or not mirror.active): - # TODO: maybe this should be 403? but that would leak existence - raise Http404 - return direct_to_template(request, 'mirrors/mirror_details.html', - {'mirror': mirror}) - -def status(request): - bad_timedelta = datetime.timedelta(days=3) - status_info = get_mirror_statuses() - - urls = status_info['urls'] - good_urls = [] - bad_urls = [] - for url in urls: - # split them into good and bad lists based on delay - if not url.delay or url.delay > bad_timedelta: - bad_urls.append(url) - else: - good_urls.append(url) - - context = status_info.copy() - context.update({ - 'good_urls': good_urls, - 'bad_urls': bad_urls, - 'error_logs': get_mirror_errors(), - }) - return direct_to_template(request, 'mirrors/status.html', context) - -# vim: set ts=4 sw=4 et: diff --git a/mirrors/views/__init__.py b/mirrors/views/__init__.py new file mode 100644 index 00000000..01e8519d --- /dev/null +++ b/mirrors/views/__init__.py @@ -0,0 +1,149 @@ +from datetime import timedelta +from itertools import groupby +from operator import attrgetter, itemgetter + +from django.db import connection +from django.http import Http404 +from django.shortcuts import get_object_or_404, render +from django.utils.timezone import now +from django.views.decorators.http import condition +from django_countries.fields import Country + +from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) +from ..utils import get_mirror_statuses, get_mirror_errors + + +def mirrors(request): + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror_id', 'protocol__protocol').distinct() + countries = MirrorUrl.objects.values_list( + 'mirror_id', 'country').order_by( + 'mirror_id', 'country').distinct() + + if not request.user.is_authenticated(): + mirror_list = mirror_list.filter(public=True, active=True) + protos = protos.filter( + mirror__public=True, mirror__active=True, active=True) + countries = countries.filter( + mirror__public=True, mirror__active=True, active=True) + + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} + countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))} + + for mirror in mirror_list: + item_protos = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in item_protos] + mirror.country = None + item_countries = countries.get(mirror.id, []) + if len(item_countries) == 1: + mirror.country = Country(item_countries[0][1]) + + return render(request, 'mirrors/mirrors.html', + {'mirror_list': mirror_list}) + + +def mirror_details(request, name): + mirror = get_object_or_404(Mirror, name=name) + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active): + raise Http404 + error_cutoff = timedelta(days=7) + + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = mirror.urls.select_related('protocol') + if not authorized: + all_urls = all_urls.filter(active=True) + all_urls = set(all_urls) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) + + error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, + show_all=True) + + context = { + 'mirror': mirror, + 'urls': all_urls, + 'cutoff': error_cutoff, + 'error_logs': error_logs, + } + return render(request, 'mirrors/mirror_details.html', context) + + +def url_details(request, name, url_id): + url = get_object_or_404(MirrorUrl.objects.select_related(), + id=url_id, mirror__name=name) + mirror = url.mirror + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active or not url.active): + raise Http404 + error_cutoff = timedelta(days=7) + cutoff_time = now() - error_cutoff + logs = MirrorLog.objects.select_related('location').filter( + url=url, check_time__gte=cutoff_time).order_by('-check_time') + + context = { + 'url': url, + 'logs': logs, + } + return render(request, 'mirrors/url_details.html', context) + + +def status_last_modified(request, *args, **kwargs): + cursor = connection.cursor() + cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog") + return cursor.fetchone()[0] + + +@condition(last_modified_func=status_last_modified) +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + bad_timedelta = timedelta(days=3) + status_info = get_mirror_statuses() + + urls = status_info['urls'] + good_urls = [] + bad_urls = [] + for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue + # split them into good and bad lists based on delay + if url.completion_pct is None: + # skip URLs that have never been checked + continue + elif not url.delay or url.delay > bad_timedelta: + bad_urls.append(url) + else: + good_urls.append(url) + + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url'].mirror.tier == tier] + + context = status_info.copy() + context.update({ + 'good_urls': sorted(good_urls, key=attrgetter('score')), + 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), + 'error_logs': error_logs, + 'tier': tier, + }) + return render(request, 'mirrors/status.html', context) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/views/api.py b/mirrors/views/api.py new file mode 100644 index 00000000..b72585e6 --- /dev/null +++ b/mirrors/views/api.py @@ -0,0 +1,108 @@ +from datetime import timedelta +import json + +from django.core.serializers.json import DjangoJSONEncoder +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404 +from django.utils.timezone import now + +from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) +from ..utils import get_mirror_statuses, DEFAULT_CUTOFF + + +class MirrorStatusJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl + serialization. The base class takes care of datetime.datetime types.''' + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') + + def default(self, obj): + if isinstance(obj, timedelta): + # always returned as integer seconds + return obj.days * 24 * 3600 + obj.seconds + if isinstance(obj, MirrorUrl): + data = {attr: getattr(obj, attr) for attr in self.url_attributes} + country = obj.country + data['country'] = unicode(country.name) + data['country_code'] = country.code + return data + if isinstance(obj, MirrorProtocol): + return unicode(obj) + return super(MirrorStatusJSONEncoder, self).default(obj) + + +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - DEFAULT_CUTOFF + data['logs'] = list(obj.logs.filter( + check_time__gte=cutoff).order_by('check_time')) + return data + if isinstance(obj, MirrorLog): + data = {attr: getattr(obj, attr) for attr in self.log_attributes} + data['error'] = obj.error or None + return data + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if isinstance(obj, CheckLocation): + return { + 'id': obj.pk, + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + status_info = get_mirror_statuses() + data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def mirror_details_json(request, name): + authorized = request.user.is_authenticated() + mirror = get_object_or_404(Mirror, name=name) + if not authorized and (not mirror.public or not mirror.active): + raise Http404 + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) + data = status_info.copy() + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = list(CheckLocation.objects.all().order_by('pk')) + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py new file mode 100644 index 00000000..3c68d036 --- /dev/null +++ b/mirrors/views/mirrorlist.py @@ -0,0 +1,129 @@ +from operator import attrgetter, itemgetter + +from django import forms +from django.db.models import Q +from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple +from django.shortcuts import get_object_or_404, redirect, render +from django.views.decorators.csrf import csrf_exempt +from django_countries import countries + +from ..models import MirrorUrl, MirrorProtocol +from ..utils import get_mirror_statuses + + +class MirrorlistForm(forms.Form): + country = forms.MultipleChoiceField(required=False, + widget=SelectMultiple(attrs={'size': '12'})) + protocol = forms.MultipleChoiceField(required=False, + widget=CheckboxSelectMultiple) + ip_version = forms.MultipleChoiceField(required=False, + label="IP version", choices=(('4','IPv4'), ('6','IPv6')), + widget=CheckboxSelectMultiple) + use_mirror_status = forms.BooleanField(required=False) + + def __init__(self, *args, **kwargs): + super(MirrorlistForm, self).__init__(*args, **kwargs) + fields = self.fields + fields['country'].choices = [('all','All')] + self.get_countries() + fields['country'].initial = ['all'] + protos = [(p.protocol, p.protocol) for p in + MirrorProtocol.objects.filter(is_download=True)] + initial = MirrorProtocol.objects.filter(is_download=True, default=True) + fields['protocol'].choices = protos + fields['protocol'].initial = [p.protocol for p in initial] + fields['ip_version'].initial = ['4'] + + def get_countries(self): + country_codes = set() + country_codes.update(MirrorUrl.objects.filter(active=True, + mirror__active=True).exclude(country='').values_list( + 'country', flat=True).order_by().distinct()) + code_list = [(code, countries.name(code)) for code in country_codes] + return sorted(code_list, key=itemgetter(1)) + + def as_div(self): + "Returns this form rendered as HTML <divs>s." + return self._html_output( + normal_row = u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>', + error_row = u'%s', + row_ender = '</div>', + help_text_html = u' <span class="helptext">%s</span>', + errors_on_separate_row = True) + + +@csrf_exempt +def generate_mirrorlist(request): + if request.method == 'POST' or len(request.GET) > 0: + form = MirrorlistForm(data=request.REQUEST) + if form.is_valid(): + countries = form.cleaned_data['country'] + protocols = form.cleaned_data['protocol'] + use_status = form.cleaned_data['use_mirror_status'] + ipv4 = '4' in form.cleaned_data['ip_version'] + ipv6 = '6' in form.cleaned_data['ip_version'] + return find_mirrors(request, countries, protocols, + use_status, ipv4, ipv6) + else: + form = MirrorlistForm() + + return render(request, 'mirrors/mirrorlist_generate.html', + {'mirrorlist_form': form}) + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = {u.id: u.score for u in status_info['urls']} + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + +def find_mirrors(request, countries=None, protocols=None, use_status=False, + ipv4_supported=True, ipv6_supported=True): + if not protocols: + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) + qset = MirrorUrl.objects.select_related().filter( + protocol__in=protocols, active=True, + mirror__public=True, mirror__active=True) + if countries and 'all' not in countries: + qset = qset.filter(country__in=countries) + + ip_version = Q() + if ipv4_supported: + ip_version |= Q(has_ipv4=True) + if ipv6_supported: + ip_version |= Q(has_ipv6=True) + qset = qset.filter(ip_version) + + if not use_status: + sort_key = attrgetter('country.name', 'mirror.name', 'url') + urls = sorted(qset, key=sort_key) + template = 'mirrors/mirrorlist.txt' + else: + urls = status_filter(qset) + template = 'mirrors/mirrorlist_status.txt' + + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') + + +def find_mirrors_simple(request, protocol): + if protocol == 'smart': + return redirect('mirrorlist_simple', 'http', permanent=True) + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) + +# vim: set ts=4 sw=4 et: |