diff options
Diffstat (limited to 'mirrors')
21 files changed, 1197 insertions, 173 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py index b7b9894c..d6ea3950 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -4,7 +4,9 @@ from urlparse import urlparse, urlunsplit from django import forms from django.contrib import admin -from .models import Mirror, MirrorProtocol, MirrorUrl, MirrorRsync +from .models import (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, + CheckLocation) + class MirrorUrlForm(forms.ModelForm): class Meta: @@ -26,12 +28,14 @@ class MirrorUrlForm(forms.ModelForm): url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', '')) return url + class MirrorUrlInlineAdmin(admin.TabularInline): model = MirrorUrl form = MirrorUrlForm readonly_fields = ('protocol', 'has_ipv4', 'has_ipv6') extra = 3 + # ripped off from django.forms.fields, adding netmask ability IPV4NM_RE = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$') @@ -43,16 +47,19 @@ class IPAddressNetmaskField(forms.fields.RegexField): def __init__(self, *args, **kwargs): super(IPAddressNetmaskField, self).__init__(IPV4NM_RE, *args, **kwargs) + class MirrorRsyncForm(forms.ModelForm): class Meta: model = MirrorRsync ip = IPAddressNetmaskField(label='IP') + class MirrorRsyncInlineAdmin(admin.TabularInline): model = MirrorRsync form = MirrorRsyncForm extra = 2 + class MirrorAdminForm(forms.ModelForm): class Meta: model = Mirror @@ -60,22 +67,31 @@ class MirrorAdminForm(forms.ModelForm): queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), required=False) + class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm - list_display = ('name', 'tier', 'country', 'active', 'public', - 'isos', 'admin_email') - list_filter = ('tier', 'active', 'public', 'country') - search_fields = ('name',) + list_display = ('name', 'tier', 'active', 'public', + 'isos', 'admin_email', 'alternate_email') + list_filter = ('tier', 'active', 'public') + search_fields = ('name', 'admin_email', 'alternate_email') inlines = [ MirrorUrlInlineAdmin, MirrorRsyncInlineAdmin, ] + class MirrorProtocolAdmin(admin.ModelAdmin): list_display = ('protocol', 'is_download', 'default') list_filter = ('is_download', 'default') + +class CheckLocationAdmin(admin.ModelAdmin): + list_display = ('hostname', 'source_ip', 'country', 'created') + search_fields = ('hostname', 'source_ip') + + admin.site.register(Mirror, MirrorAdmin) admin.site.register(MirrorProtocol, MirrorProtocolAdmin) +admin.site.register(CheckLocation, CheckLocationAdmin) # vim: set ts=4 sw=4 et: diff --git a/mirrors/fixtures/mirrorprotocols.json b/mirrors/fixtures/mirrorprotocols.json index 72ed1a7f..8822ef8e 100644 --- a/mirrors/fixtures/mirrorprotocols.json +++ b/mirrors/fixtures/mirrorprotocols.json @@ -7,7 +7,7 @@ "default": true, "protocol": "http" } - }, + }, { "pk": 2, "model": "mirrors.mirrorprotocol", @@ -16,7 +16,7 @@ "default": false, "protocol": "ftp" } - }, + }, { "pk": 3, "model": "mirrors.mirrorprotocol", @@ -25,5 +25,14 @@ "default": false, "protocol": "rsync" } + }, + { + "pk": 5, + "model": "mirrors.mirrorprotocol", + "fields": { + "is_download": true, + "default": false, + "protocol": "https" + } } ] diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7ffb7773..d6de8f22 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -9,24 +9,30 @@ we encounter errors, record those as well. Usage: ./manage.py mirrorcheck """ -from django.core.management.base import NoArgsCommand -from django.db import transaction - from collections import deque from datetime import datetime +from httplib import HTTPException import logging +import os +from optparse import make_option +from pytz import utc import re import socket +import subprocess import sys import time +import tempfile from threading import Thread import types -from pytz import utc from Queue import Queue, Empty import urllib2 -from main.utils import utc_now -from mirrors.models import MirrorUrl, MirrorLog +from django.core.management.base import NoArgsCommand +from django.db import transaction +from django.utils.timezone import now + +from mirrors.models import MirrorUrl, MirrorLog, CheckLocation + logging.basicConfig( level=logging.WARNING, @@ -35,7 +41,14 @@ logging.basicConfig( stream=sys.stderr) logger = logging.getLogger() + class Command(NoArgsCommand): + option_list = NoArgsCommand.option_list + ( + make_option('-t', '--timeout', dest='timeout', type='float', default=10.0, + help='Timeout value for connecting to URL'), + make_option('-l', '--location', dest='location', type='int', + help='ID of CheckLocation object to use for this run'), + ) help = "Runs a check on all known mirror URLs to determine their up-to-date status." def handle_noargs(self, **options): @@ -47,36 +60,73 @@ class Command(NoArgsCommand): elif v == 2: logger.level = logging.DEBUG - return check_current_mirrors() + timeout = options.get('timeout') + + urls = MirrorUrl.objects.select_related('protocol').filter( + mirror__active=True, mirror__public=True) + + location = options.get('location', None) + if location: + location = CheckLocation.objects.get(id=location) + family = location.family + monkeypatch_getaddrinfo(family) + if family == socket.AF_INET6: + urls = urls.filter(has_ipv6=True) + elif family == socket.AF_INET: + urls = urls.filter(has_ipv4=True) + + pool = MirrorCheckPool(urls, location, timeout) + pool.run() + return 0 + + +def monkeypatch_getaddrinfo(force_family=socket.AF_INET): + '''Force the Python socket module to connect over the designated family; + e.g. socket.AF_INET or socket.AF_INET6.''' + orig = socket.getaddrinfo + + def wrapper(host, port, family=0, socktype=0, proto=0, flags=0): + return orig(host, port, force_family, socktype, proto, flags) + + socket.getaddrinfo = wrapper + + +def parse_lastsync(log, data): + '''lastsync file should be an epoch value created by us.''' + try: + parsed_time = datetime.utcfromtimestamp(int(data)) + log.last_sync = parsed_time.replace(tzinfo=utc) + except ValueError: + # it is bad news to try logging the lastsync value; + # sometimes we get a crazy-encoded web page. + # if we couldn't parse a time, this is a failure. + log.last_sync = None + log.error = "Could not parse time from lastsync" + log.is_success = False + + +def check_mirror_url(mirror_url, location, timeout): + if location: + if location.family == socket.AF_INET6: + ipopt = '--ipv6' + elif location.family == socket.AF_INET: + ipopt = '--ipv4' -def check_mirror_url(mirror_url): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) + headers = {'User-Agent': 'archweb/1.0'} + req = urllib2.Request(url, None, headers) try: start = time.time() - result = urllib2.urlopen(url, timeout=10) + result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() end = time.time() - # lastsync should be an epoch value created by us - parsed_time = None - try: - parsed_time = datetime.utcfromtimestamp(int(data)) - parsed_time = parsed_time.replace(tzinfo=utc) - except ValueError: - # it is bad news to try logging the lastsync value; - # sometimes we get a crazy-encoded web page. - pass - - log.last_sync = parsed_time - # if we couldn't parse a time, this is a failure - if parsed_time is None: - log.error = "Could not parse time from lastsync" - log.is_success = False + parse_lastsync(log, data) log.duration = end - start logger.debug("success: %s, %.2f", url, log.duration) - except urllib2.HTTPError, e: + except urllib2.HTTPError as e: if e.code == 404: # we have a duration, just not a success end = time.time() @@ -84,7 +134,7 @@ def check_mirror_url(mirror_url): log.is_success = False log.error = str(e) logger.debug("failed: %s, %s", url, log.error) - except urllib2.URLError, e: + except urllib2.URLError as e: log.is_success = False log.error = e.reason if isinstance(e.reason, types.StringTypes) and \ @@ -97,35 +147,102 @@ def check_mirror_url(mirror_url): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except socket.timeout, e: + except HTTPException as e: + # e.g., BadStatusLine + log.is_success = False + log.error = "Exception in processing HTTP request." + logger.debug("failed: %s, %s", url, log.error) + except socket.timeout as e: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) + except socket.error as e: + log.is_success = False + log.error = str(e) + logger.debug("failed: %s, %s", url, log.error) return log -def mirror_url_worker(work, output): + +def check_rsync_url(mirror_url, location, timeout): + url = mirror_url.url + 'lastsync' + logger.info("checking URL %s", url) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) + + tempdir = tempfile.mkdtemp() + ipopt = '' + if location: + if location.family == socket.AF_INET6: + ipopt = '--ipv6' + elif location.family == socket.AF_INET: + ipopt = '--ipv4' + lastsync_path = os.path.join(tempdir, 'lastsync') + rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout, + "--timeout=%d" % timeout] + if ipopt: + rsync_cmd.append(ipopt) + rsync_cmd.append(url) + rsync_cmd.append(lastsync_path) + try: + with open(os.devnull, 'w') as devnull: + logger.debug("rsync cmd: %s", ' '.join(rsync_cmd)) + proc = subprocess.Popen(rsync_cmd, stdout=devnull, + stderr=subprocess.PIPE) + start = time.time() + _, errdata = proc.communicate() + end = time.time() + log.duration = end - start + if proc.returncode != 0: + logger.debug("error: %s, %s", url, errdata) + log.is_success = False + log.error = errdata.strip() + # look at rsync error code- if we had a command error or timed out, + # don't record a duration as it is misleading + if proc.returncode in (1, 30, 35): + log.duration = None + else: + logger.debug("success: %s, %.2f", url, log.duration) + with open(lastsync_path, 'r') as lastsync: + parse_lastsync(log, lastsync.read()) + finally: + if os.path.exists(lastsync_path): + os.unlink(lastsync_path) + os.rmdir(tempdir) + + return log + + +def mirror_url_worker(work, output, location, timeout): while True: try: - item = work.get(block=False) + url = work.get(block=False) try: - log = check_mirror_url(item) - output.append(log) + if url.protocol.protocol == 'rsync': + log = check_rsync_url(url, location, timeout) + elif (url.protocol.protocol == 'ftp' and location and + location.family == socket.AF_INET6): + # IPv6 + FTP don't work; skip checking completely + log = None + else: + log = check_mirror_url(url, location, timeout) + if log: + output.append(log) finally: work.task_done() except Empty: return 0 + class MirrorCheckPool(object): - def __init__(self, work, num_threads=10): + def __init__(self, urls, location, timeout=10, num_threads=10): self.tasks = Queue() self.logs = deque() - for i in list(work): - self.tasks.put(i) + for url in list(urls): + self.tasks.put(url) self.threads = [] for i in range(num_threads): thread = Thread(target=mirror_url_worker, - args=(self.tasks, self.logs)) + args=(self.tasks, self.logs, location, timeout)) thread.daemon = True self.threads.append(thread) @@ -136,21 +253,8 @@ class MirrorCheckPool(object): thread.start() logger.debug("joining on all threads") self.tasks.join() - logger.debug("processing log entries") + logger.debug("processing %d log entries", len(self.logs)) MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") -def check_current_mirrors(): - urls = MirrorUrl.objects.filter( - protocol__is_download=True, - mirror__active=True, mirror__public=True) - - pool = MirrorCheckPool(urls) - pool.run() - return 0 - -# For lack of a better place to put it, here is a query to get latest check -# result joined with mirror details: -# SELECT mu.*, m.*, ml.* FROM mirrors_mirrorurl mu JOIN mirrors_mirror m ON mu.mirror_id = m.id JOIN mirrors_mirrorlog ml ON mu.id = ml.url_id LEFT JOIN mirrors_mirrorlog ml2 ON ml.url_id = ml2.url_id AND ml.id < ml2.id WHERE ml2.id IS NULL AND m.active = 1 AND m.public = 1; - # vim: set ts=4 sw=4 et: diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 4e812f2d..a6c2523e 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -41,13 +41,19 @@ def resolve_mirrors(): logger.debug("requesting list of mirror URLs") for mirrorurl in MirrorUrl.objects.filter(mirror__active=True): try: + # save old values, we can skip no-op updates this way + oldvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) logger.debug("resolving %3i (%s)", mirrorurl.id, mirrorurl.hostname) families = mirrorurl.address_families() mirrorurl.has_ipv4 = socket.AF_INET in families mirrorurl.has_ipv6 = socket.AF_INET6 in families logger.debug("%s: v4: %s v6: %s", mirrorurl.hostname, mirrorurl.has_ipv4, mirrorurl.has_ipv6) - mirrorurl.save(force_update=True) + # now check new values, only update if new != old + newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) + if newvals != oldvals: + logger.debug("values changed for %s", mirrorurl) + mirrorurl.save(update_fields=('has_ipv4', 'has_ipv6')) except socket.error, e: logger.warn("error resolving %s: %s", mirrorurl.hostname, e) diff --git a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py index 5e44d211..0506e2cd 100644 --- a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py +++ b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py @@ -7,7 +7,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) + db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorprotocol', 'is_download') diff --git a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py index a5e34589..5a40207d 100644 --- a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py +++ b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py @@ -7,8 +7,8 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) - db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) + db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorurl', 'has_ipv4') diff --git a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py index d30c78c7..66e60090 100644 --- a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py +++ b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py @@ -6,7 +6,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) + db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorprotocol', 'default') diff --git a/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py new file mode 100644 index 00000000..60c4ec26 --- /dev/null +++ b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.TextField')(default='')) + + def backwards(self, orm): + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.CharField')(max_length=255)) + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py new file mode 100644 index 00000000..a08699e8 --- /dev/null +++ b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + def forwards(self, orm): + db.add_column('mirrors_mirror', 'alternate_email', + self.gf('django.db.models.fields.EmailField')(default='', max_length=255, blank=True), + keep_default=False) + + def backwards(self, orm): + db.delete_column('mirrors_mirror', 'alternate_email') + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0019_move_country_data_to_url.py b/mirrors/migrations/0019_move_country_data_to_url.py new file mode 100644 index 00000000..81b7bb3e --- /dev/null +++ b/mirrors/migrations/0019_move_country_data_to_url.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models + +class Migration(DataMigration): + + def forwards(self, orm): + for url in orm.MirrorUrl.objects.select_related('mirror').all(): + # set the country field on the URL if we have one, + # and it isn't already set to anything. + if url.country or not url.mirror.country: + continue + orm.MirrorUrl.objects.filter(pk=url.pk).update( + country=url.mirror.country) + + def backwards(self, orm): + pass + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] + symmetrical = True diff --git a/mirrors/migrations/0020_auto__del_field_mirror_country.py b/mirrors/migrations/0020_auto__del_field_mirror_country.py new file mode 100644 index 00000000..c2220a50 --- /dev/null +++ b/mirrors/migrations/0020_auto__del_field_mirror_country.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.delete_column('mirrors_mirror', 'country') + + + def backwards(self, orm): + db.add_column('mirrors_mirror', 'country', + self.gf('django_countries.fields.CountryField')(blank=True, default='', max_length=2, db_index=True), + keep_default=False) + + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py new file mode 100644 index 00000000..bbf14bb0 --- /dev/null +++ b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44)) + + def backwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=24)) + + models = { + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0022_auto__add_checklocation.py b/mirrors/migrations/0022_auto__add_checklocation.py new file mode 100644 index 00000000..896b2dab --- /dev/null +++ b/mirrors/migrations/0022_auto__add_checklocation.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.create_table(u'mirrors_checklocation', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('hostname', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('source_ip', self.gf('django.db.models.fields.GenericIPAddressField')(unique=True, max_length=39)), + ('country', self.gf('django_countries.fields.CountryField')(max_length=2)), + ('created', self.gf('django.db.models.fields.DateTimeField')()), + )) + db.send_create_signal(u'mirrors', ['CheckLocation']) + + + def backwards(self, orm): + db.delete_table(u'mirrors_checklocation') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py new file mode 100644 index 00000000..1a1f48a0 --- /dev/null +++ b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models +from pytz import utc + + +class Migration(SchemaMigration): + + def forwards(self, orm): + default = datetime.datetime(2000, 1, 1, 0, 0).replace(tzinfo=utc) + db.add_column(u'mirrors_mirrorurl', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirrorrsync', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirrorprotocol', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirror', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorurl', 'created') + db.delete_column(u'mirrors_mirrorrsync', 'created') + db.delete_column(u'mirrors_mirrorprotocol', 'created') + db.delete_column(u'mirrors_mirror', 'created') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py new file mode 100644 index 00000000..acf8df17 --- /dev/null +++ b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.add_column(u'mirrors_mirrorlog', 'location', + self.gf('django.db.models.fields.related.ForeignKey')(related_name='logs', null=True, to=orm['mirrors.CheckLocation']), + keep_default=False) + + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorlog', 'location_id') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 19437610..e41f6b22 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,55 +1,55 @@ import socket from urlparse import urlparse -from django.db import models from django.core.exceptions import ValidationError +from django.db import models +from django.db.models.signals import pre_save from django_countries import CountryField - -TIER_CHOICES = ( - (0, 'Tier 0'), - (1, 'Tier 1'), - (2, 'Tier 2'), - (-1, 'Untiered'), -) +from main.utils import set_created_field class Mirror(models.Model): + TIER_CHOICES = ( + (0, 'Tier 0'), + (1, 'Tier 1'), + (2, 'Tier 2'), + (-1, 'Untiered'), + ) + name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) - country = CountryField(blank=True, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) + alternate_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) active = models.BooleanField(default=True) isos = models.BooleanField("ISOs", default=True) rsync_user = models.CharField(max_length=50, blank=True, default='') rsync_password = models.CharField(max_length=50, blank=True, default='') notes = models.TextField(blank=True) + created = models.DateTimeField(editable=False) class Meta: - ordering = ('country', 'name') + ordering = ('name',) def __unicode__(self): return self.name - def supported_protocols(self): - protocols = MirrorProtocol.objects.filter( - urls__mirror=self).order_by('protocol').distinct() - return sorted(protocols) - def downstream(self): return Mirror.objects.filter(upstream=self).order_by('name') def get_absolute_url(self): return '/mirrors/%s/' % self.name + class MirrorProtocol(models.Model): protocol = models.CharField(max_length=10, unique=True) is_download = models.BooleanField(default=True, help_text="Is protocol useful for end-users, e.g. FTP/HTTP") default = models.BooleanField(default=True, help_text="Included by default when building mirror list?") + created = models.DateTimeField(editable=False) def __unicode__(self): return self.protocol @@ -57,6 +57,7 @@ class MirrorProtocol(models.Model): class Meta: ordering = ('protocol',) + class MirrorUrl(models.Model): url = models.CharField("URL", max_length=255, unique=True) protocol = models.ForeignKey(MirrorProtocol, related_name="urls", @@ -67,6 +68,7 @@ class MirrorUrl(models.Model): editable=False) has_ipv6 = models.BooleanField("IPv6 capable", default=False, editable=False) + created = models.DateTimeField(editable=False) def address_families(self): hostname = urlparse(self.url).hostname @@ -78,10 +80,6 @@ class MirrorUrl(models.Model): def hostname(self): return urlparse(self.url).hostname - @property - def real_country(self): - return self.country or self.mirror.country - def clean(self): try: # Auto-map the protocol field by looking at the URL @@ -104,28 +102,60 @@ class MirrorUrl(models.Model): class Meta: verbose_name = 'mirror URL' + class MirrorRsync(models.Model): - ip = models.CharField("IP", max_length=24) + # max length is 40 chars for full-form IPv6 addr + subnet + ip = models.CharField("IP", max_length=44) mirror = models.ForeignKey(Mirror, related_name="rsync_ips") + created = models.DateTimeField(editable=False) def __unicode__(self): - return "%s" % (self.ip) + return self.ip class Meta: verbose_name = 'mirror rsync IP' + +class CheckLocation(models.Model): + hostname = models.CharField(max_length=255) + source_ip = models.GenericIPAddressField(verbose_name='source IP', + unpack_ipv4=True, unique=True) + country = CountryField() + created = models.DateTimeField(editable=False) + + class Meta: + ordering = ('hostname', 'source_ip') + + def __unicode__(self): + return self.hostname + + @property + def family(self): + info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0, + socket.AI_NUMERICHOST) + families = [x[0] for x in info] + return families[0] + + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") + location = models.ForeignKey(CheckLocation, related_name="logs", null=True) check_time = models.DateTimeField(db_index=True) last_sync = models.DateTimeField(null=True) duration = models.FloatField(null=True) is_success = models.BooleanField(default=True) - error = models.CharField(max_length=255, blank=True, default='') + error = models.TextField(blank=True, default='') def __unicode__(self): return "Check of %s at %s" % (self.url.url, self.check_time) class Meta: verbose_name = 'mirror check log' + get_latest_by = 'check_time' + + +for model in (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, CheckLocation): + pre_save.connect(set_created_field, sender=model, + dispatch_uid="mirrors.models") # vim: set ts=4 sw=4 et: diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js new file mode 100644 index 00000000..8ec85c40 --- /dev/null +++ b/mirrors/static/mirror_status.js @@ -0,0 +1,141 @@ +function mirror_status(chart_id, data_url) { + var jq_div = jQuery(chart_id); + + var draw_graph = function(data) { + var margin = {top: 20, right: 20, bottom: 30, left: 40}, + width = jq_div.width() - margin.left - margin.right, + height = jq_div.height() - margin.top - margin.bottom; + + var color = d3.scale.category10(), + x = d3.time.scale.utc().range([0, width]), + y = d3.scale.linear().range([height, 0]), + x_axis = d3.svg.axis().scale(x).orient("bottom"), + y_axis = d3.svg.axis().scale(y).orient("left"); + + /* remove any existing graph first if we are redrawing after resize */ + d3.select(chart_id).select("svg").remove(); + var svg = d3.select(chart_id).append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + x.domain([ + d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.check_time; }); }), + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) + ]).nice(d3.time.hour); + y.domain([ + 0, + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) + ]).nice(); + + /* build the axis lines... */ + svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(x_axis) + .append("text") + .attr("class", "label") + .attr("x", width) + .attr("y", -6) + .style("text-anchor", "end") + .text("Check Time (UTC)"); + + svg.append("g") + .attr("class", "y axis") + .call(y_axis) + .append("text") + .attr("class", "label") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text("Duration (seconds)"); + + var line = d3.svg.line() + .interpolate("basis") + .x(function(d) { return x(d.check_time); }) + .y(function(d) { return y(d.duration); }); + + /* ...then the points and lines between them. */ + var urls = svg.selectAll(".url") + .data(data) + .enter() + .append("g") + .attr("class", "url"); + + urls.append("path") + .attr("class", "url-line") + .attr("d", function(d) { return line(d.logs); }) + .style("stroke", function(d) { return color(d.url); }); + + urls.selectAll("circle") + .data(function(u) { + return jQuery.map(u.logs, function(l, i) { + return {url: u.url, check_time: l.check_time, duration: l.duration}; + }); + }) + .enter() + .append("circle") + .attr("class", "url-dot") + .attr("r", 3.5) + .attr("cx", function(d) { return x(d.check_time); }) + .attr("cy", function(d) { return y(d.duration); }) + .style("fill", function(d) { return color(d.url); }) + .append("title") + .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); }); + + /* add a legend for good measure */ + var legend = svg.selectAll(".legend") + .data(color.domain()) + .enter().append("g") + .attr("class", "legend") + .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); + + legend.append("rect") + .attr("x", width - 18) + .attr("width", 18) + .attr("height", 18) + .style("fill", color); + + legend.append("text") + .attr("x", width - 24) + .attr("y", 9) + .attr("dy", ".35em") + .style("text-anchor", "end") + .text(function(d) { return d; }); + }; + + /* invoke the data-fetch + first draw */ + var cached_data = null; + d3.json(data_url, function(json) { + cached_data = jQuery.map(json.urls, function(url, i) { + return { + url: url.url, + logs: jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } + return { + duration: log.duration, + check_time: new Date(log.check_time) + }; + }) + }; + }); + draw_graph(cached_data); + }); + + /* then hook up a resize handler to redraw if necessary */ + var resize_timeout = null; + var real_resize = function() { + resize_timeout = null; + draw_graph(cached_data); + }; + jQuery(window).resize(function() { + if (resize_timeout) { + clearTimeout(resize_timeout); + } + resize_timeout = setTimeout(real_resize, 200); + }); +} diff --git a/mirrors/urls.py b/mirrors/urls.py index f002e9d6..4e929410 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -4,7 +4,10 @@ urlpatterns = patterns('mirrors.views', (r'^$', 'mirrors', {}, 'mirror-list'), (r'^status/$', 'status', {}, 'mirror-status'), (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), + (r'^status/tier/(?P<tier>\d+)/$', 'status', {}, 'mirror-status-tier'), + (r'^status/tier/(?P<tier>\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'), + (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'), ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index e0f44c78..1444eca9 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -3,10 +3,7 @@ from django.conf.urls import patterns urlpatterns = patterns('mirrors.views', (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), (r'^all/$', 'find_mirrors', {'countries': ['all']}), - (r'^all/ftp/$', 'find_mirrors', - {'countries': ['all'], 'protocols': ['ftp']}), - (r'^all/http/$', 'find_mirrors', - {'countries': ['all'], 'protocols': ['http']}), + (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple') ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/utils.py b/mirrors/utils.py index 32fa3587..3ab176b3 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,14 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.timezone import now from django_countries.fields import Country -from main.utils import cache_function, utc_now +from main.utils import cache_function, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl -default_cutoff = timedelta(hours=24) +DEFAULT_CUTOFF = timedelta(hours=24) def annotate_url(url, delays): '''Given a MirrorURL object, add a few more attributes to it regarding @@ -30,36 +31,57 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff - protocols = list(MirrorProtocol.objects.filter(is_download=True)) - # I swear, this actually has decent performance... - urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): + cutoff_time = now() - cutoff + + valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, - protocol__in=protocols, - logs__check_time__gte=cutoff_time).annotate( + logs__check_time__gte=cutoff_time).distinct() + + if mirror_ids: + valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) + + url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( + id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration'), - duration_stddev=StdDev('logs__duration')) + duration_avg=Avg('logs__duration')) + + vendor = database_vendor(MirrorUrl) + if vendor != 'sqlite': + url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) + + urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + id__in=valid_urls).order_by('mirror__id', 'url') # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the # results here. - times = MirrorLog.objects.filter(is_success=True, last_sync__isnull=False, + times = MirrorLog.objects.values_list( + 'url_id', 'check_time', 'last_sync').filter( + is_success=True, last_sync__isnull=False, check_time__gte=cutoff_time) + if mirror_ids: + times = times.filter(url__mirror_id__in=mirror_ids) delays = {} - for log in times: - delay = log.check_time - log.last_sync - delays.setdefault(log.url_id, []).append(delay) + for url_id, check_time, last_sync in times: + delay = check_time - last_sync + delays.setdefault(url_id, []).append(delay) if urls: + url_data = dict((item['id'], item) for item in url_data) + for url in urls: + for k, v in url_data.get(url.id, {}).items(): + if k not in ('id', 'mirror_id'): + setattr(url, k, v) last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) - check_info = MirrorLog.objects.filter( - check_time__gte=cutoff_time).aggregate( + check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) + if mirror_ids: + check_info = check_info.filter(url__mirror_id__in=mirror_ids) + check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: check_frequency = (check_info['mx'] - check_info['mn']) \ @@ -72,6 +94,9 @@ def get_mirror_statuses(cutoff=default_cutoff): check_frequency = None for url in urls: + # fake the standard deviation for local testing setups + if vendor == 'sqlite': + setattr(url, 'duration_stddev', 0.0) annotate_url(url, delays) return { @@ -84,28 +109,31 @@ def get_mirror_statuses(cutoff=default_cutoff): @cache_function(117) -def get_mirror_errors(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): + cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'error').annotate( + 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') + + if mirror_ids: + urls = urls.filter(mirror_id__in=mirror_ids) + errors = list(errors) for err in errors: - ctry_code = err['url__country'] or err['url__mirror__country'] - err['country'] = Country(ctry_code) + err['country'] = Country(err['url__country']) return errors @cache_function(295) -def get_mirror_url_for_download(cutoff=default_cutoff): +def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): '''Find a good mirror URL to use for package downloads. If we have mirror status data available, it is used to determine a good choice by looking at the last batch of status rows.''' - cutoff_time = utc_now() - cutoff + cutoff_time = now() - cutoff status_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) @@ -123,7 +151,7 @@ def get_mirror_url_for_download(cutoff=default_cutoff): mirror_urls = MirrorUrl.objects.filter( mirror__public=True, mirror__active=True, protocol__default=True) # look first for a country-agnostic URL, then fall back to any HTTP URL - filtered_urls = mirror_urls.filter(mirror__country='')[:1] + filtered_urls = mirror_urls.filter(country='')[:1] if not filtered_urls: filtered_urls = mirror_urls[:1] if not filtered_urls: diff --git a/mirrors/views.py b/mirrors/views.py index c52656f7..56397633 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,4 +1,6 @@ from datetime import timedelta +from itertools import groupby +import json from operator import attrgetter, itemgetter from django import forms @@ -6,14 +8,13 @@ from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, render +from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt -from django.views.generic.simple import direct_to_template -from django.utils import simplejson from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol -from .utils import get_mirror_statuses, get_mirror_errors +from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog +from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF COUNTRY_LOOKUP = dict(COUNTRIES) @@ -41,9 +42,6 @@ class MirrorlistForm(forms.Form): def get_countries(self): country_codes = set() - country_codes.update(Mirror.objects.filter(active=True).exclude( - country='').values_list( - 'country', flat=True).order_by().distinct()) country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) @@ -75,22 +73,50 @@ def generate_mirrorlist(request): else: form = MirrorlistForm() - return direct_to_template(request, 'mirrors/mirrorlist_generate.html', + return render(request, 'mirrors/mirrorlist_generate.html', {'mirrorlist_form': form}) +def default_protocol_filter(original_urls): + key_func = attrgetter('country') + sorted_urls = sorted(original_urls, key=key_func) + urls = [] + for _, group in groupby(sorted_urls, key=key_func): + cntry_urls = list(group) + if any(url.protocol.default for url in cntry_urls): + cntry_urls = [url for url in cntry_urls if url.protocol.default] + urls.extend(cntry_urls) + return urls + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = {u.id: u.score for u in status_info['urls']} + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True): + ipv4_supported=True, ipv6_supported=True, smart_protocol=False): if not protocols: - protocols = MirrorProtocol.objects.filter( - is_download=True).values_list('protocol', flat=True) + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) qset = MirrorUrl.objects.select_related().filter( - protocol__protocol__in=protocols, - mirror__public=True, mirror__active=True, - ) + protocol__in=protocols, + mirror__public=True, mirror__active=True) if countries and 'all' not in countries: - qset = qset.filter(Q(country__in=countries) | - Q(mirror__country__in=countries)) + qset = qset.filter(country__in=countries) ip_version = Q() if ipv4_supported: @@ -99,36 +125,47 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, ip_version |= Q(has_ipv6=True) qset = qset.filter(ip_version) + if smart_protocol: + urls = default_protocol_filter(qset) + else: + urls = qset + if not use_status: - urls = qset.order_by('mirror__name', 'url') - urls = sorted(urls, key=attrgetter('real_country')) + sort_key = attrgetter('country.name', 'mirror.name', 'url') + urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: - status_info = get_mirror_statuses() - scores = dict([(u.id, u.score) for u in status_info['urls']]) - urls = [] - for u in qset: - u.score = scores.get(u.id, None) - # also include mirrors that don't have an up to date score - # (as opposed to those that have been set with no score) - if (u.id not in scores) or \ - (u.score and u.score < 100.0): - urls.append(u) - # if a url doesn't have a score, treat it as the highest possible - urls = sorted(urls, key=lambda x: x.score or 100.0) + urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' - return direct_to_template(request, template, { - 'mirror_urls': urls, - }, - mimetype='text/plain') + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') + + +def find_mirrors_simple(request, protocol): + if protocol == 'smart': + # generate a 'smart' mirrorlist, one that only includes FTP mirrors if + # no HTTP mirror is available in that country. + return find_mirrors(request, smart_protocol=True) + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror__id', 'protocol__protocol').distinct() if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) - return direct_to_template(request, 'mirrors/mirrors.html', + protos = protos.filter(mirror__public=True, mirror__active=True) + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} + for mirror in mirror_list: + items = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in items] + return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) @@ -138,20 +175,38 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses() - checked_urls = [url for url in status_info['urls'] \ - if url.mirror_id == mirror.id] - all_urls = mirror.urls.select_related('protocol') - # get each item from checked_urls and supplement with anything in all_urls - # if it wasn't there - all_urls = set(checked_urls).union(all_urls) - all_urls = sorted(all_urls, key=attrgetter('url')) - - return direct_to_template(request, 'mirrors/mirror_details.html', + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = set(mirror.urls.select_related('protocol')) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) + + return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) -def status(request): +def mirror_details_json(request, name): + mirror = get_object_or_404(Mirror, name=name) + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + data = status_info.copy() + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() @@ -159,26 +214,35 @@ def status(request): good_urls = [] bad_urls = [] for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue # split them into good and bad lists based on delay if not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url__mirror__tier'] == tier] + context = status_info.copy() context.update({ 'good_urls': sorted(good_urls, key=attrgetter('score')), 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': get_mirror_errors(), + 'error_logs': error_logs, + 'tier': tier, }) - return direct_to_template(request, 'mirrors/status.html', context) + return render(request, 'mirrors/status.html', context) class MirrorStatusJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl serialization. The base class takes care of datetime.datetime types.''' - url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct', - 'delay', 'duration_avg', 'duration_stddev', 'score'] + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') def default(self, obj): if isinstance(obj, timedelta): @@ -188,10 +252,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): # mainly for queryset serialization return list(obj) if isinstance(obj, MirrorUrl): - data = dict((attr, getattr(obj, attr)) - for attr in self.url_attributes) - # get any override on the country attribute first - country = obj.real_country + data = {attr: getattr(obj, attr) for attr in self.url_attributes} + country = obj.country data['country'] = unicode(country.name) data['country_code'] = country.code return data @@ -200,13 +262,34 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): return super(MirrorStatusJSONEncoder, self).default(obj) -def status_json(request): +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - DEFAULT_CUTOFF + data['logs'] = obj.logs.filter( + check_time__gte=cutoff).order_by('check_time') + return data + if isinstance(obj, MirrorLog): + return {attr: getattr(obj, attr) for attr in self.log_attributes} + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 status_info = get_mirror_statuses() data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] data['version'] = 3 - to_json = simplejson.dumps(data, ensure_ascii=False, - cls=MirrorStatusJSONEncoder) - response = HttpResponse(to_json, mimetype='application/json') + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') return response # vim: set ts=4 sw=4 et: |