diff options
| author | Luke Shumaker <LukeShu@sbcglobal.net> | 2013-04-22 00:36:57 -0400 |
|---|---|---|
| committer | Luke Shumaker <LukeShu@sbcglobal.net> | 2013-04-22 00:36:57 -0400 |
| commit | df7a6aa620af6a165bdacd755757f8cb1179331c (patch) | |
| tree | 384b4c62d1f50d8effb733d81d2a810666807624 /mirrors | |
| parent | 94f972bb892dbf9a86f089f1872ae6d849c0cd0e (diff) | |
| parent | a22557811a24b68ef85d4271787c48d8d1e4fc99 (diff) | |
Merge branch 'archweb-generic2'
Conflicts:
README.BRANDING
local_settings.py.example
packages/templatetags/package_extras.py
public/views.py
releng/views.py
settings.py
sitestatic/archnavbar/archnavbar.css
sitestatic/silhouette.png
templates/base.html
templates/packages/differences.html
templates/packages/opensearch.xml
templates/packages/search.html
templates/public/donate.html
templates/public/download.html
templates/public/feeds.html
templates/public/index.html
urls.py
Diffstat (limited to 'mirrors')
23 files changed, 1475 insertions, 217 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py index b7b9894c..9c88207d 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -1,10 +1,11 @@ -import re from urlparse import urlparse, urlunsplit from django import forms from django.contrib import admin -from .models import Mirror, MirrorProtocol, MirrorUrl, MirrorRsync +from .models import (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, + CheckLocation) + class MirrorUrlForm(forms.ModelForm): class Meta: @@ -26,33 +27,25 @@ class MirrorUrlForm(forms.ModelForm): url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', '')) return url + class MirrorUrlInlineAdmin(admin.TabularInline): model = MirrorUrl form = MirrorUrlForm readonly_fields = ('protocol', 'has_ipv4', 'has_ipv6') extra = 3 -# ripped off from django.forms.fields, adding netmask ability -IPV4NM_RE = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$') - -class IPAddressNetmaskField(forms.fields.RegexField): - default_error_messages = { - 'invalid': u'Enter a valid IPv4 address, possibly including netmask.', - } - - def __init__(self, *args, **kwargs): - super(IPAddressNetmaskField, self).__init__(IPV4NM_RE, *args, **kwargs) class MirrorRsyncForm(forms.ModelForm): class Meta: model = MirrorRsync - ip = IPAddressNetmaskField(label='IP') + class MirrorRsyncInlineAdmin(admin.TabularInline): model = MirrorRsync form = MirrorRsyncForm extra = 2 + class MirrorAdminForm(forms.ModelForm): class Meta: model = Mirror @@ -60,22 +53,31 @@ class MirrorAdminForm(forms.ModelForm): queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), required=False) + class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm - list_display = ('name', 'tier', 'country', 'active', 'public', - 'isos', 'admin_email') - list_filter = ('tier', 'active', 'public', 'country') - search_fields = ('name',) + list_display = ('name', 'tier', 'active', 'public', + 'isos', 'admin_email', 'alternate_email') + list_filter = ('tier', 'active', 'public') + search_fields = ('name', 'admin_email', 'alternate_email') inlines = [ MirrorUrlInlineAdmin, MirrorRsyncInlineAdmin, ] + class MirrorProtocolAdmin(admin.ModelAdmin): list_display = ('protocol', 'is_download', 'default') list_filter = ('is_download', 'default') + +class CheckLocationAdmin(admin.ModelAdmin): + list_display = ('hostname', 'source_ip', 'country', 'created') + search_fields = ('hostname', 'source_ip') + + admin.site.register(Mirror, MirrorAdmin) admin.site.register(MirrorProtocol, MirrorProtocolAdmin) +admin.site.register(CheckLocation, CheckLocationAdmin) # vim: set ts=4 sw=4 et: diff --git a/mirrors/fields.py b/mirrors/fields.py new file mode 100644 index 00000000..206c9d7d --- /dev/null +++ b/mirrors/fields.py @@ -0,0 +1,49 @@ +from IPy import IP + +from django import forms +from django.core import validators +from django.core.exceptions import ValidationError +from django.db import models +from south.modelsinspector import add_introspection_rules + + +class IPNetworkFormField(forms.Field): + def to_python(self, value): + if value in validators.EMPTY_VALUES: + return None + try: + value = IP(value) + except ValueError as e: + raise ValidationError(str(e)) + return value + + +class IPNetworkField(models.Field): + __metaclass__ = models.SubfieldBase + description = "IPv4 or IPv6 address or subnet" + + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 44 + super(IPNetworkField, self).__init__(*args, **kwargs) + + def get_internal_type(self): + return "IPAddressField" + + def to_python(self, value): + if not value: + return None + return IP(value) + + def get_prep_value(self, value): + value = self.to_python(value) + if not value: + return None + return str(value) + + def formfield(self, **kwargs): + defaults = {'form_class': IPNetworkFormField} + defaults.update(kwargs) + return super(IPNetworkField, self).formfield(**defaults) + + +add_introspection_rules([], ["^mirrors\.fields\.IPNetworkField"]) diff --git a/mirrors/fixtures/mirrorprotocols.json b/mirrors/fixtures/mirrorprotocols.json index 72ed1a7f..8822ef8e 100644 --- a/mirrors/fixtures/mirrorprotocols.json +++ b/mirrors/fixtures/mirrorprotocols.json @@ -7,7 +7,7 @@ "default": true, "protocol": "http" } - }, + }, { "pk": 2, "model": "mirrors.mirrorprotocol", @@ -16,7 +16,7 @@ "default": false, "protocol": "ftp" } - }, + }, { "pk": 3, "model": "mirrors.mirrorprotocol", @@ -25,5 +25,14 @@ "default": false, "protocol": "rsync" } + }, + { + "pk": 5, + "model": "mirrors.mirrorprotocol", + "fields": { + "is_download": true, + "default": false, + "protocol": "https" + } } ] diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7ffb7773..e7dd7b49 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -9,24 +9,30 @@ we encounter errors, record those as well. Usage: ./manage.py mirrorcheck """ -from django.core.management.base import NoArgsCommand -from django.db import transaction - from collections import deque from datetime import datetime +from httplib import HTTPException import logging +import os +from optparse import make_option +from pytz import utc import re import socket +import subprocess import sys import time +import tempfile from threading import Thread import types -from pytz import utc from Queue import Queue, Empty import urllib2 -from main.utils import utc_now -from mirrors.models import MirrorUrl, MirrorLog +from django.core.management.base import NoArgsCommand +from django.db import transaction +from django.utils.timezone import now + +from mirrors.models import MirrorUrl, MirrorLog, CheckLocation + logging.basicConfig( level=logging.WARNING, @@ -35,7 +41,14 @@ logging.basicConfig( stream=sys.stderr) logger = logging.getLogger() + class Command(NoArgsCommand): + option_list = NoArgsCommand.option_list + ( + make_option('-t', '--timeout', dest='timeout', type='float', default=10.0, + help='Timeout value for connecting to URL'), + make_option('-l', '--location', dest='location', type='int', + help='ID of CheckLocation object to use for this run'), + ) help = "Runs a check on all known mirror URLs to determine their up-to-date status." def handle_noargs(self, **options): @@ -47,36 +60,67 @@ class Command(NoArgsCommand): elif v == 2: logger.level = logging.DEBUG - return check_current_mirrors() + timeout = options.get('timeout') + + urls = MirrorUrl.objects.select_related('protocol').filter( + mirror__active=True, mirror__public=True) + + location = options.get('location', None) + if location: + location = CheckLocation.objects.get(id=location) + family = location.family + monkeypatch_getaddrinfo(family) + if family == socket.AF_INET6: + urls = urls.filter(has_ipv6=True) + elif family == socket.AF_INET: + urls = urls.filter(has_ipv4=True) + + pool = MirrorCheckPool(urls, location, timeout) + pool.run() + return 0 + + +def monkeypatch_getaddrinfo(force_family=socket.AF_INET): + '''Force the Python socket module to connect over the designated family; + e.g. socket.AF_INET or socket.AF_INET6.''' + orig = socket.getaddrinfo + + def wrapper(host, port, family=0, socktype=0, proto=0, flags=0): + return orig(host, port, force_family, socktype, proto, flags) + + socket.getaddrinfo = wrapper + + +def parse_lastsync(log, data): + '''lastsync file should be an epoch value created by us.''' + try: + parsed_time = datetime.utcfromtimestamp(int(data)) + log.last_sync = parsed_time.replace(tzinfo=utc) + except ValueError: + # it is bad news to try logging the lastsync value; + # sometimes we get a crazy-encoded web page. + # if we couldn't parse a time, this is a failure. + log.last_sync = None + log.error = "Could not parse time from lastsync" + log.is_success = False + -def check_mirror_url(mirror_url): +def check_mirror_url(mirror_url, location, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) + headers = {'User-Agent': 'archweb/1.0'} + req = urllib2.Request(url, None, headers) + start = time.time() try: - start = time.time() - result = urllib2.urlopen(url, timeout=10) + result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() end = time.time() - # lastsync should be an epoch value created by us - parsed_time = None - try: - parsed_time = datetime.utcfromtimestamp(int(data)) - parsed_time = parsed_time.replace(tzinfo=utc) - except ValueError: - # it is bad news to try logging the lastsync value; - # sometimes we get a crazy-encoded web page. - pass - - log.last_sync = parsed_time - # if we couldn't parse a time, this is a failure - if parsed_time is None: - log.error = "Could not parse time from lastsync" - log.is_success = False + parse_lastsync(log, data) log.duration = end - start logger.debug("success: %s, %.2f", url, log.duration) - except urllib2.HTTPError, e: + except urllib2.HTTPError as e: if e.code == 404: # we have a duration, just not a success end = time.time() @@ -84,7 +128,7 @@ def check_mirror_url(mirror_url): log.is_success = False log.error = str(e) logger.debug("failed: %s, %s", url, log.error) - except urllib2.URLError, e: + except urllib2.URLError as e: log.is_success = False log.error = e.reason if isinstance(e.reason, types.StringTypes) and \ @@ -97,35 +141,102 @@ def check_mirror_url(mirror_url): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except socket.timeout, e: + except HTTPException: + # e.g., BadStatusLine + log.is_success = False + log.error = "Exception in processing HTTP request." + logger.debug("failed: %s, %s", url, log.error) + except socket.timeout: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) + except socket.error as e: + log.is_success = False + log.error = str(e) + logger.debug("failed: %s, %s", url, log.error) + + return log + + +def check_rsync_url(mirror_url, location, timeout): + url = mirror_url.url + 'lastsync' + logger.info("checking URL %s", url) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) + + tempdir = tempfile.mkdtemp() + ipopt = '' + if location: + if location.family == socket.AF_INET6: + ipopt = '--ipv6' + elif location.family == socket.AF_INET: + ipopt = '--ipv4' + lastsync_path = os.path.join(tempdir, 'lastsync') + rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout, + "--timeout=%d" % timeout] + if ipopt: + rsync_cmd.append(ipopt) + rsync_cmd.append(url) + rsync_cmd.append(lastsync_path) + try: + with open(os.devnull, 'w') as devnull: + logger.debug("rsync cmd: %s", ' '.join(rsync_cmd)) + proc = subprocess.Popen(rsync_cmd, stdout=devnull, + stderr=subprocess.PIPE) + start = time.time() + _, errdata = proc.communicate() + end = time.time() + log.duration = end - start + if proc.returncode != 0: + logger.debug("error: %s, %s", url, errdata) + log.is_success = False + log.error = errdata.strip() + # look at rsync error code- if we had a command error or timed out, + # don't record a duration as it is misleading + if proc.returncode in (1, 30, 35): + log.duration = None + else: + logger.debug("success: %s, %.2f", url, log.duration) + with open(lastsync_path, 'r') as lastsync: + parse_lastsync(log, lastsync.read()) + finally: + if os.path.exists(lastsync_path): + os.unlink(lastsync_path) + os.rmdir(tempdir) return log -def mirror_url_worker(work, output): + +def mirror_url_worker(work, output, location, timeout): while True: try: - item = work.get(block=False) + url = work.get(block=False) try: - log = check_mirror_url(item) - output.append(log) + if url.protocol.protocol == 'rsync': + log = check_rsync_url(url, location, timeout) + elif (url.protocol.protocol == 'ftp' and location and + location.family == socket.AF_INET6): + # IPv6 + FTP don't work; skip checking completely + log = None + else: + log = check_mirror_url(url, location, timeout) + if log: + output.append(log) finally: work.task_done() except Empty: return 0 + class MirrorCheckPool(object): - def __init__(self, work, num_threads=10): + def __init__(self, urls, location, timeout=10, num_threads=10): self.tasks = Queue() self.logs = deque() - for i in list(work): - self.tasks.put(i) + for url in list(urls): + self.tasks.put(url) self.threads = [] for i in range(num_threads): thread = Thread(target=mirror_url_worker, - args=(self.tasks, self.logs)) + args=(self.tasks, self.logs, location, timeout)) thread.daemon = True self.threads.append(thread) @@ -136,21 +247,8 @@ class MirrorCheckPool(object): thread.start() logger.debug("joining on all threads") self.tasks.join() - logger.debug("processing log entries") + logger.debug("processing %d log entries", len(self.logs)) MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") -def check_current_mirrors(): - urls = MirrorUrl.objects.filter( - protocol__is_download=True, - mirror__active=True, mirror__public=True) - - pool = MirrorCheckPool(urls) - pool.run() - return 0 - -# For lack of a better place to put it, here is a query to get latest check -# result joined with mirror details: -# SELECT mu.*, m.*, ml.* FROM mirrors_mirrorurl mu JOIN mirrors_mirror m ON mu.mirror_id = m.id JOIN mirrors_mirrorlog ml ON mu.id = ml.url_id LEFT JOIN mirrors_mirrorlog ml2 ON ml.url_id = ml2.url_id AND ml.id < ml2.id WHERE ml2.id IS NULL AND m.active = 1 AND m.public = 1; - # vim: set ts=4 sw=4 et: diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 4e812f2d..a6c2523e 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -41,13 +41,19 @@ def resolve_mirrors(): logger.debug("requesting list of mirror URLs") for mirrorurl in MirrorUrl.objects.filter(mirror__active=True): try: + # save old values, we can skip no-op updates this way + oldvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) logger.debug("resolving %3i (%s)", mirrorurl.id, mirrorurl.hostname) families = mirrorurl.address_families() mirrorurl.has_ipv4 = socket.AF_INET in families mirrorurl.has_ipv6 = socket.AF_INET6 in families logger.debug("%s: v4: %s v6: %s", mirrorurl.hostname, mirrorurl.has_ipv4, mirrorurl.has_ipv6) - mirrorurl.save(force_update=True) + # now check new values, only update if new != old + newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) + if newvals != oldvals: + logger.debug("values changed for %s", mirrorurl) + mirrorurl.save(update_fields=('has_ipv4', 'has_ipv6')) except socket.error, e: logger.warn("error resolving %s: %s", mirrorurl.hostname, e) diff --git a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py index 5e44d211..0506e2cd 100644 --- a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py +++ b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py @@ -7,7 +7,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) + db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorprotocol', 'is_download') diff --git a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py index a5e34589..5a40207d 100644 --- a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py +++ b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py @@ -7,8 +7,8 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) - db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) + db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorurl', 'has_ipv4') diff --git a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py index d30c78c7..66e60090 100644 --- a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py +++ b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py @@ -6,7 +6,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) + db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorprotocol', 'default') diff --git a/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py new file mode 100644 index 00000000..60c4ec26 --- /dev/null +++ b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.TextField')(default='')) + + def backwards(self, orm): + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.CharField')(max_length=255)) + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py new file mode 100644 index 00000000..a08699e8 --- /dev/null +++ b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + def forwards(self, orm): + db.add_column('mirrors_mirror', 'alternate_email', + self.gf('django.db.models.fields.EmailField')(default='', max_length=255, blank=True), + keep_default=False) + + def backwards(self, orm): + db.delete_column('mirrors_mirror', 'alternate_email') + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0019_move_country_data_to_url.py b/mirrors/migrations/0019_move_country_data_to_url.py new file mode 100644 index 00000000..81b7bb3e --- /dev/null +++ b/mirrors/migrations/0019_move_country_data_to_url.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models + +class Migration(DataMigration): + + def forwards(self, orm): + for url in orm.MirrorUrl.objects.select_related('mirror').all(): + # set the country field on the URL if we have one, + # and it isn't already set to anything. + if url.country or not url.mirror.country: + continue + orm.MirrorUrl.objects.filter(pk=url.pk).update( + country=url.mirror.country) + + def backwards(self, orm): + pass + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] + symmetrical = True diff --git a/mirrors/migrations/0020_auto__del_field_mirror_country.py b/mirrors/migrations/0020_auto__del_field_mirror_country.py new file mode 100644 index 00000000..c2220a50 --- /dev/null +++ b/mirrors/migrations/0020_auto__del_field_mirror_country.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.delete_column('mirrors_mirror', 'country') + + + def backwards(self, orm): + db.add_column('mirrors_mirror', 'country', + self.gf('django_countries.fields.CountryField')(blank=True, default='', max_length=2, db_index=True), + keep_default=False) + + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py new file mode 100644 index 00000000..bbf14bb0 --- /dev/null +++ b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44)) + + def backwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=24)) + + models = { + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0022_auto__add_checklocation.py b/mirrors/migrations/0022_auto__add_checklocation.py new file mode 100644 index 00000000..896b2dab --- /dev/null +++ b/mirrors/migrations/0022_auto__add_checklocation.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.create_table(u'mirrors_checklocation', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('hostname', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('source_ip', self.gf('django.db.models.fields.GenericIPAddressField')(unique=True, max_length=39)), + ('country', self.gf('django_countries.fields.CountryField')(max_length=2)), + ('created', self.gf('django.db.models.fields.DateTimeField')()), + )) + db.send_create_signal(u'mirrors', ['CheckLocation']) + + + def backwards(self, orm): + db.delete_table(u'mirrors_checklocation') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py new file mode 100644 index 00000000..1a1f48a0 --- /dev/null +++ b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models +from pytz import utc + + +class Migration(SchemaMigration): + + def forwards(self, orm): + default = datetime.datetime(2000, 1, 1, 0, 0).replace(tzinfo=utc) + db.add_column(u'mirrors_mirrorurl', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirrorrsync', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirrorprotocol', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirror', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorurl', 'created') + db.delete_column(u'mirrors_mirrorrsync', 'created') + db.delete_column(u'mirrors_mirrorprotocol', 'created') + db.delete_column(u'mirrors_mirror', 'created') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py new file mode 100644 index 00000000..acf8df17 --- /dev/null +++ b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.add_column(u'mirrors_mirrorlog', 'location', + self.gf('django.db.models.fields.related.ForeignKey')(related_name='logs', null=True, to=orm['mirrors.CheckLocation']), + keep_default=False) + + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorlog', 'location_id') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py new file mode 100644 index 00000000..b359b637 --- /dev/null +++ b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + if db.backend_name == 'postgres': + # For PostgreSQL, because it uses the 'inet' type and not a varchar + # column, we need to add an explict 'USING' cast to the SQL + # statement. We then execute the alter_column as well to ensure any + # of the other side-effects happen. + db.execute('ALTER TABLE "mirrors_mirrorrsync" ALTER COLUMN "ip" TYPE inet USING "ip"::inet') + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('mirrors.fields.IPNetworkField')(max_length=44)) + + def backwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44)) + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 19437610..d8ac7952 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,55 +1,56 @@ import socket from urlparse import urlparse -from django.db import models from django.core.exceptions import ValidationError +from django.db import models +from django.db.models.signals import pre_save from django_countries import CountryField - -TIER_CHOICES = ( - (0, 'Tier 0'), - (1, 'Tier 1'), - (2, 'Tier 2'), - (-1, 'Untiered'), -) +from .fields import IPNetworkField +from main.utils import set_created_field class Mirror(models.Model): + TIER_CHOICES = ( + (0, 'Tier 0'), + (1, 'Tier 1'), + (2, 'Tier 2'), + (-1, 'Untiered'), + ) + name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) - country = CountryField(blank=True, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) + alternate_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) active = models.BooleanField(default=True) isos = models.BooleanField("ISOs", default=True) rsync_user = models.CharField(max_length=50, blank=True, default='') rsync_password = models.CharField(max_length=50, blank=True, default='') notes = models.TextField(blank=True) + created = models.DateTimeField(editable=False) class Meta: - ordering = ('country', 'name') + ordering = ('name',) def __unicode__(self): return self.name - def supported_protocols(self): - protocols = MirrorProtocol.objects.filter( - urls__mirror=self).order_by('protocol').distinct() - return sorted(protocols) - def downstream(self): return Mirror.objects.filter(upstream=self).order_by('name') def get_absolute_url(self): return '/mirrors/%s/' % self.name + class MirrorProtocol(models.Model): protocol = models.CharField(max_length=10, unique=True) is_download = models.BooleanField(default=True, help_text="Is protocol useful for end-users, e.g. FTP/HTTP") default = models.BooleanField(default=True, help_text="Included by default when building mirror list?") + created = models.DateTimeField(editable=False) def __unicode__(self): return self.protocol @@ -57,6 +58,7 @@ class MirrorProtocol(models.Model): class Meta: ordering = ('protocol',) + class MirrorUrl(models.Model): url = models.CharField("URL", max_length=255, unique=True) protocol = models.ForeignKey(MirrorProtocol, related_name="urls", @@ -67,6 +69,7 @@ class MirrorUrl(models.Model): editable=False) has_ipv6 = models.BooleanField("IPv6 capable", default=False, editable=False) + created = models.DateTimeField(editable=False) def address_families(self): hostname = urlparse(self.url).hostname @@ -78,10 +81,6 @@ class MirrorUrl(models.Model): def hostname(self): return urlparse(self.url).hostname - @property - def real_country(self): - return self.country or self.mirror.country - def clean(self): try: # Auto-map the protocol field by looking at the URL @@ -93,7 +92,7 @@ class MirrorUrl(models.Model): families = self.address_families() self.has_ipv4 = socket.AF_INET in families self.has_ipv6 = socket.AF_INET6 in families - except socket.error as e: + except socket.error: # We don't fail in this case; we'll just set both to False self.has_ipv4 = False self.has_ipv6 = False @@ -104,28 +103,69 @@ class MirrorUrl(models.Model): class Meta: verbose_name = 'mirror URL' + class MirrorRsync(models.Model): - ip = models.CharField("IP", max_length=24) + # max length is 40 chars for full-form IPv6 addr + subnet + ip = IPNetworkField("IP") mirror = models.ForeignKey(Mirror, related_name="rsync_ips") + created = models.DateTimeField(editable=False) def __unicode__(self): - return "%s" % (self.ip) + return self.ip class Meta: verbose_name = 'mirror rsync IP' + +class CheckLocation(models.Model): + hostname = models.CharField(max_length=255) + source_ip = models.GenericIPAddressField(verbose_name='source IP', + unpack_ipv4=True, unique=True) + country = CountryField() + created = models.DateTimeField(editable=False) + + class Meta: + ordering = ('hostname', 'source_ip') + + def __unicode__(self): + return self.hostname + + @property + def family(self): + info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0, + socket.AI_NUMERICHOST) + families = [x[0] for x in info] + return families[0] + + @property + def ip_version(self): + '''Returns integer '4' or '6'.''' + if self.family == socket.AF_INET6: + return 6 + if self.family == socket.AF_INET: + return 4 + return None + + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") + location = models.ForeignKey(CheckLocation, related_name="logs", null=True) check_time = models.DateTimeField(db_index=True) last_sync = models.DateTimeField(null=True) duration = models.FloatField(null=True) is_success = models.BooleanField(default=True) - error = models.CharField(max_length=255, blank=True, default='') + error = models.TextField(blank=True, default='') def __unicode__(self): return "Check of %s at %s" % (self.url.url, self.check_time) class Meta: verbose_name = 'mirror check log' + get_latest_by = 'check_time' + + +for model in (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, CheckLocation): + pre_save.connect(set_created_field, sender=model, + dispatch_uid="mirrors.models") # vim: set ts=4 sw=4 et: diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js new file mode 100644 index 00000000..241f5c61 --- /dev/null +++ b/mirrors/static/mirror_status.js @@ -0,0 +1,173 @@ +function draw_graphs(location_url, log_url, container_id) { + jQuery.when(jQuery.getJSON(location_url), jQuery.getJSON(log_url)) + .then(function(loc_data, log_data) { + /* use the same color selection for a given URL in every graph */ + var color = d3.scale.category10(); + jQuery.each(loc_data[0].locations, function(i, val) { + mirror_status(container_id, val, log_data[0], color); + }); + }); +} + +function mirror_status(container_id, check_loc, log_data, color) { + + var draw_graph = function(chart_id, data) { + var jq_div = jQuery(chart_id); + var margin = {top: 20, right: 20, bottom: 30, left: 40}, + width = jq_div.width() - margin.left - margin.right, + height = jq_div.height() - margin.top - margin.bottom; + + var x = d3.time.scale.utc().range([0, width]), + y = d3.scale.linear().range([height, 0]), + x_axis = d3.svg.axis().scale(x).orient("bottom"), + y_axis = d3.svg.axis().scale(y).orient("left"); + + /* remove any existing graph first if we are redrawing after resize */ + d3.select(chart_id).select("svg").remove(); + var svg = d3.select(chart_id).append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + x.domain([ + d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.check_time; }); }), + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) + ]).nice(d3.time.hour); + y.domain([ + 0, + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) + ]).nice(); + + /* build the axis lines... */ + svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(x_axis) + .append("text") + .attr("class", "label") + .attr("x", width) + .attr("y", -6) + .style("text-anchor", "end") + .text("Check Time (UTC)"); + + svg.append("g") + .attr("class", "y axis") + .call(y_axis) + .append("text") + .attr("class", "label") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text("Duration (seconds)"); + + var line = d3.svg.line() + .interpolate("basis") + .x(function(d) { return x(d.check_time); }) + .y(function(d) { return y(d.duration); }); + + /* ...then the points and lines between them. */ + var urls = svg.selectAll(".url") + .data(data) + .enter() + .append("g") + .attr("class", "url"); + + urls.append("path") + .attr("class", "url-line") + .attr("d", function(d) { return line(d.logs); }) + .style("stroke", function(d) { return color(d.url); }); + + urls.selectAll("circle") + .data(function(u) { + return jQuery.map(u.logs, function(l, i) { + return {url: u.url, check_time: l.check_time, duration: l.duration}; + }); + }) + .enter() + .append("circle") + .attr("class", "url-dot") + .attr("r", 3.5) + .attr("cx", function(d) { return x(d.check_time); }) + .attr("cy", function(d) { return y(d.duration); }) + .style("fill", function(d) { return color(d.url); }) + .append("title") + .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); }); + + /* add a legend for good measure */ + var active = jQuery.map(data, function(item, i) { return item.url; }); + var legend = svg.selectAll(".legend") + .data(active) + .enter().append("g") + .attr("class", "legend") + .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); + + legend.append("rect") + .attr("x", width - 18) + .attr("width", 18) + .attr("height", 18) + .style("fill", color); + + legend.append("text") + .attr("x", width - 24) + .attr("y", 9) + .attr("dy", ".35em") + .style("text-anchor", "end") + .text(function(d) { return d; }); + }; + + var filter_data = function(json, location_id) { + return jQuery.map(json.urls, function(url, i) { + var logs = jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } + /* screen by location ID if we were given one */ + if (location_id && log.location_id !== location_id) { + return null; + } + return { + duration: log.duration, + check_time: new Date(log.check_time) + }; + }); + /* don't return URLs without any log info */ + if (logs.length === 0) { + return null; + } + return { + url: url.url, + logs: logs + }; + }); + }; + + var cached_data = filter_data(log_data, check_loc.id); + /* we had a check location with no log data handed to us, skip graphing */ + if (cached_data.length === 0) { + return; + } + + /* create the containers, defer the actual graph drawing */ + var chart_id = 'status-chart-' + check_loc.id; + jQuery(container_id).append('<h3><span class="fam-flag fam-flag-' + check_loc.country_code.toLowerCase() + '" title="' + check_loc.country + '"></span> ' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '</h3>'); + jQuery(container_id).append('<div id="' + chart_id + '" class="visualize-mirror visualize-chart"></div>'); + jQuery(container_id).append('<br/>'); + setTimeout(function() { + draw_graph('#' + chart_id, cached_data); + }, 0); + + /* then hook up a resize handler to redraw if necessary */ + var resize_timeout = null; + var real_resize = function() { + resize_timeout = null; + draw_graph('#' + chart_id, cached_data); + }; + jQuery(window).resize(function() { + if (resize_timeout) { + clearTimeout(resize_timeout); + } + resize_timeout = setTimeout(real_resize, 200); + }); +} diff --git a/mirrors/urls.py b/mirrors/urls.py index f002e9d6..7cf76aa1 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -4,7 +4,11 @@ urlpatterns = patterns('mirrors.views', (r'^$', 'mirrors', {}, 'mirror-list'), (r'^status/$', 'status', {}, 'mirror-status'), (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), + (r'^status/tier/(?P<tier>\d+)/$', 'status', {}, 'mirror-status-tier'), + (r'^status/tier/(?P<tier>\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), + (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'), + (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'), ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index e0f44c78..1444eca9 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -3,10 +3,7 @@ from django.conf.urls import patterns urlpatterns = patterns('mirrors.views', (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), (r'^all/$', 'find_mirrors', {'countries': ['all']}), - (r'^all/ftp/$', 'find_mirrors', - {'countries': ['all'], 'protocols': ['ftp']}), - (r'^all/http/$', 'find_mirrors', - {'countries': ['all'], 'protocols': ['http']}), + (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple') ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/utils.py b/mirrors/utils.py index 32fa3587..ba45da5f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,21 +1,107 @@ from datetime import timedelta -from django.db.models import Avg, Count, Max, Min, StdDev +from django.db import connection +from django.db.models import Count, Max, Min +from django.utils.dateparse import parse_datetime +from django.utils.timezone import now from django_countries.fields import Country -from main.utils import cache_function, utc_now -from .models import MirrorLog, MirrorProtocol, MirrorUrl +from main.utils import cache_function, database_vendor +from .models import MirrorLog, MirrorUrl -default_cutoff = timedelta(hours=24) +DEFAULT_CUTOFF = timedelta(hours=24) -def annotate_url(url, delays): + +def dictfetchall(cursor): + "Returns all rows from a cursor as a dict." + desc = cursor.description + return [ + dict(zip([col[0] for col in desc], row)) + for row in cursor.fetchall() + ] + + +def status_data(cutoff_time, mirror_id=None): + if mirror_id is not None: + params = [cutoff_time, mirror_id] + mirror_where = 'AND u.mirror_id = %s' + else: + params = [cutoff_time] + mirror_where = '' + + vendor = database_vendor(MirrorUrl) + if vendor == 'sqlite': + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + 0.0 AS duration_stddev, + AVG(STRFTIME('%%s', check_time) - STRFTIME('%%s', last_sync)) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id +""" + else: + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + STDDEV(l.duration) AS duration_stddev, + AVG(check_time - last_sync) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id +""" + + cursor = connection.cursor() + cursor.execute(sql, params) + url_data = dictfetchall(cursor) + + # sqlite loves to return less than ideal types + if vendor == 'sqlite': + for item in url_data: + if item['delay'] is not None: + item['delay'] = timedelta(seconds=item['delay']) + if item['last_sync'] is not None: + item['last_sync'] = parse_datetime(item['last_sync']) + item['last_check'] = parse_datetime(item['last_check']) + + return {item['url_id']: item for item in url_data} + + +def annotate_url(url, url_data): '''Given a MirrorURL object, add a few more attributes to it regarding status, including completion_pct, delay, and score.''' - url.completion_pct = float(url.success_count) / url.check_count - if url.id in delays: - url_delays = delays[url.id] - url.delay = sum(url_delays, timedelta()) / len(url_delays) + known_attrs = ( + ('success_count', 0), + ('check_count', 0), + ('completion_pct', None), + ('last_check', None), + ('last_sync', None), + ('delay', None), + ('score', None), + ) + for k, v in known_attrs: + setattr(url, k, v) + for k, v in url_data.items(): + if k not in ('url_id', 'mirror_id'): + setattr(url, k, v) + + if url.check_count > 0: + url.completion_pct = float(url.success_count) / url.check_count + + if url.delay is not None: hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 if url.completion_pct > 0: @@ -23,43 +109,33 @@ def annotate_url(url, delays): else: # arbitrary small value divisor = 0.005 - url.score = (hours + url.duration_avg + url.duration_stddev) / divisor - else: - url.delay = None - url.score = None + stddev = url.duration_stddev or 0.0 + url.score = (hours + url.duration_avg + stddev) / divisor -@cache_function(123) -def get_mirror_statuses(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff - protocols = list(MirrorProtocol.objects.filter(is_download=True)) - # I swear, this actually has decent performance... - urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): + cutoff_time = now() - cutoff + + valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, - protocol__in=protocols, - logs__check_time__gte=cutoff_time).annotate( - check_count=Count('logs'), - success_count=Count('logs__duration'), - last_sync=Max('logs__last_sync'), - last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration'), - duration_stddev=StdDev('logs__duration')) - - # The Django ORM makes it really hard to get actual average delay in the - # above query, so run a seperate query for it and we will process the - # results here. - times = MirrorLog.objects.filter(is_success=True, last_sync__isnull=False, - check_time__gte=cutoff_time) - delays = {} - for log in times: - delay = log.check_time - log.last_sync - delays.setdefault(log.url_id, []).append(delay) + logs__check_time__gte=cutoff_time).distinct() + + if mirror_id: + valid_urls = valid_urls.filter(mirror_id=mirror_id) + + url_data = status_data(cutoff_time, mirror_id) + urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + id__in=valid_urls).order_by('mirror__id', 'url') if urls: - last_check = max([u.last_check for u in urls]) + for url in urls: + annotate_url(url, url_data.get(url.id, {})) + last_check = max([u.last_check for u in urls if u.last_check]) num_checks = max([u.check_count for u in urls]) - check_info = MirrorLog.objects.filter( - check_time__gte=cutoff_time).aggregate( + check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) + if mirror_id: + check_info = check_info.filter(url__mirror_id=mirror_id) + check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: check_frequency = (check_info['mx'] - check_info['mn']) \ @@ -71,9 +147,6 @@ def get_mirror_statuses(cutoff=default_cutoff): num_checks = 0 check_frequency = None - for url in urls: - annotate_url(url, delays) - return { 'cutoff': cutoff, 'last_check': last_check, @@ -83,29 +156,31 @@ def get_mirror_statuses(cutoff=default_cutoff): } -@cache_function(117) -def get_mirror_errors(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): + cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'error').annotate( + 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') + + if mirror_id: + errors = errors.filter(url__mirror_id=mirror_id) + errors = list(errors) for err in errors: - ctry_code = err['url__country'] or err['url__mirror__country'] - err['country'] = Country(ctry_code) + err['country'] = Country(err['url__country']) return errors @cache_function(295) -def get_mirror_url_for_download(cutoff=default_cutoff): +def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): '''Find a good mirror URL to use for package downloads. If we have mirror status data available, it is used to determine a good choice by looking at the last batch of status rows.''' - cutoff_time = utc_now() - cutoff + cutoff_time = now() - cutoff status_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) @@ -123,7 +198,7 @@ def get_mirror_url_for_download(cutoff=default_cutoff): mirror_urls = MirrorUrl.objects.filter( mirror__public=True, mirror__active=True, protocol__default=True) # look first for a country-agnostic URL, then fall back to any HTTP URL - filtered_urls = mirror_urls.filter(mirror__country='')[:1] + filtered_urls = mirror_urls.filter(country='')[:1] if not filtered_urls: filtered_urls = mirror_urls[:1] if not filtered_urls: diff --git a/mirrors/views.py b/mirrors/views.py index c52656f7..73d40297 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,4 +1,6 @@ from datetime import timedelta +from itertools import groupby +import json from operator import attrgetter, itemgetter from django import forms @@ -6,16 +8,14 @@ from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, render +from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt -from django.views.generic.simple import direct_to_template -from django.utils import simplejson from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol -from .utils import get_mirror_statuses, get_mirror_errors - -COUNTRY_LOOKUP = dict(COUNTRIES) +from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) +from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF class MirrorlistForm(forms.Form): @@ -27,6 +27,8 @@ class MirrorlistForm(forms.Form): widget=CheckboxSelectMultiple) use_mirror_status = forms.BooleanField(required=False) + countries = dict(COUNTRIES) + def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) fields = self.fields @@ -41,13 +43,10 @@ class MirrorlistForm(forms.Form): def get_countries(self): country_codes = set() - country_codes.update(Mirror.objects.filter(active=True).exclude( - country='').values_list( - 'country', flat=True).order_by().distinct()) country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) - countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes] + countries = [(code, self.countries[code]) for code in country_codes] return sorted(countries, key=itemgetter(1)) def as_div(self): @@ -75,22 +74,50 @@ def generate_mirrorlist(request): else: form = MirrorlistForm() - return direct_to_template(request, 'mirrors/mirrorlist_generate.html', + return render(request, 'mirrors/mirrorlist_generate.html', {'mirrorlist_form': form}) +def default_protocol_filter(original_urls): + key_func = attrgetter('country') + sorted_urls = sorted(original_urls, key=key_func) + urls = [] + for _, group in groupby(sorted_urls, key=key_func): + cntry_urls = list(group) + if any(url.protocol.default for url in cntry_urls): + cntry_urls = [url for url in cntry_urls if url.protocol.default] + urls.extend(cntry_urls) + return urls + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = {u.id: u.score for u in status_info['urls']} + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True): + ipv4_supported=True, ipv6_supported=True, smart_protocol=False): if not protocols: - protocols = MirrorProtocol.objects.filter( - is_download=True).values_list('protocol', flat=True) + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) qset = MirrorUrl.objects.select_related().filter( - protocol__protocol__in=protocols, - mirror__public=True, mirror__active=True, - ) + protocol__in=protocols, + mirror__public=True, mirror__active=True) if countries and 'all' not in countries: - qset = qset.filter(Q(country__in=countries) | - Q(mirror__country__in=countries)) + qset = qset.filter(country__in=countries) ip_version = Q() if ipv4_supported: @@ -99,36 +126,47 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, ip_version |= Q(has_ipv6=True) qset = qset.filter(ip_version) + if smart_protocol: + urls = default_protocol_filter(qset) + else: + urls = qset + if not use_status: - urls = qset.order_by('mirror__name', 'url') - urls = sorted(urls, key=attrgetter('real_country')) + sort_key = attrgetter('country.name', 'mirror.name', 'url') + urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: - status_info = get_mirror_statuses() - scores = dict([(u.id, u.score) for u in status_info['urls']]) - urls = [] - for u in qset: - u.score = scores.get(u.id, None) - # also include mirrors that don't have an up to date score - # (as opposed to those that have been set with no score) - if (u.id not in scores) or \ - (u.score and u.score < 100.0): - urls.append(u) - # if a url doesn't have a score, treat it as the highest possible - urls = sorted(urls, key=lambda x: x.score or 100.0) + urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' - return direct_to_template(request, template, { - 'mirror_urls': urls, - }, - mimetype='text/plain') + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') + + +def find_mirrors_simple(request, protocol): + if protocol == 'smart': + # generate a 'smart' mirrorlist, one that only includes FTP mirrors if + # no HTTP mirror is available in that country. + return find_mirrors(request, smart_protocol=True) + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror__id', 'protocol__protocol').distinct() if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) - return direct_to_template(request, 'mirrors/mirrors.html', + protos = protos.filter(mirror__public=True, mirror__active=True) + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} + for mirror in mirror_list: + items = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in items] + return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) @@ -138,20 +176,38 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses() - checked_urls = [url for url in status_info['urls'] \ - if url.mirror_id == mirror.id] - all_urls = mirror.urls.select_related('protocol') - # get each item from checked_urls and supplement with anything in all_urls - # if it wasn't there - all_urls = set(checked_urls).union(all_urls) - all_urls = sorted(all_urls, key=attrgetter('url')) - - return direct_to_template(request, 'mirrors/mirror_details.html', + status_info = get_mirror_statuses(mirror_id=mirror.id) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = set(mirror.urls.select_related('protocol')) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) + + return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) -def status(request): +def mirror_details_json(request, name): + mirror = get_object_or_404(Mirror, name=name) + status_info = get_mirror_statuses(mirror_id=mirror.id) + data = status_info.copy() + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() @@ -159,26 +215,35 @@ def status(request): good_urls = [] bad_urls = [] for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue # split them into good and bad lists based on delay if not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url__mirror__tier'] == tier] + context = status_info.copy() context.update({ 'good_urls': sorted(good_urls, key=attrgetter('score')), 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': get_mirror_errors(), + 'error_logs': error_logs, + 'tier': tier, }) - return direct_to_template(request, 'mirrors/status.html', context) + return render(request, 'mirrors/status.html', context) class MirrorStatusJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl serialization. The base class takes care of datetime.datetime types.''' - url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct', - 'delay', 'duration_avg', 'duration_stddev', 'score'] + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') def default(self, obj): if isinstance(obj, timedelta): @@ -188,10 +253,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): # mainly for queryset serialization return list(obj) if isinstance(obj, MirrorUrl): - data = dict((attr, getattr(obj, attr)) - for attr in self.url_attributes) - # get any override on the country attribute first - country = obj.real_country + data = {attr: getattr(obj, attr) for attr in self.url_attributes} + country = obj.country data['country'] = unicode(country.name) data['country_code'] = country.code return data @@ -200,13 +263,63 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): return super(MirrorStatusJSONEncoder, self).default(obj) -def status_json(request): +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - DEFAULT_CUTOFF + data['logs'] = obj.logs.filter( + check_time__gte=cutoff).order_by('check_time') + return data + if isinstance(obj, MirrorLog): + return {attr: getattr(obj, attr) for attr in self.log_attributes} + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 status_info = get_mirror_statuses() data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] data['version'] = 3 - to_json = simplejson.dumps(data, ensure_ascii=False, - cls=MirrorStatusJSONEncoder) - response = HttpResponse(to_json, mimetype='application/json') + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, CheckLocation): + return { + 'id': obj.pk, + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = CheckLocation.objects.all().order_by('pk') + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') return response # vim: set ts=4 sw=4 et: |