Merge branch 'master' of git://projects.archlinux.org/archweb

Conflicts:
	devel/views.py
	feeds.py
	templates/devel/index.html
	templates/packages/flag.html
	templates/public/index.html
	todolists/views.py
	urls.py

8 files changed, 200 insertions, 64 deletions

diff --git a/mirrors/admin.py b/mirrors/admin.py
index b9c2876a..b7b478de 100644
--- a/mirrors/admin.py
+++ b/mirrors/admin.py
@@ -60,7 +60,7 @@ class MirrorAdminForm(forms.ModelForm):
 class MirrorAdmin(admin.ModelAdmin):
     form = MirrorAdminForm
     list_display = ('name', 'tier', 'country', 'active', 'public', 'isos', 'admin_email', 'supported_protocols')
-    list_filter = ('tier', 'country', 'active', 'public')
+    list_filter = ('tier', 'active', 'public', 'country') 
     search_fields = ('name',)
     inlines = [
             MirrorUrlInlineAdmin,
diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py
index 51be71ea..ea43d558 100644
--- a/mirrors/management/commands/mirrorcheck.py
+++ b/mirrors/management/commands/mirrorcheck.py
@@ -13,7 +13,7 @@ from django.core.management.base import NoArgsCommand
 from django.db import transaction
 
 from collections import deque
-from datetime import datetime, timedelta
+from datetime import datetime
 import logging
 import re
 import socket
@@ -52,22 +52,6 @@ class Command(NoArgsCommand):
 
         return check_current_mirrors()
 
-def parse_rfc3339_datetime(time_string):
-    # '2010-09-02 11:05:06+02:00'
-    m = re.match('^(\d{4})-(\d{2})-(\d{2}) '
-            '(\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time_string)
-    if m:
-        vals = m.groups()
-        parsed = datetime(int(vals[0]), int(vals[1]), int(vals[2]),
-                int(vals[3]), int(vals[4]), int(vals[5]))
-        # now account for time zone offset
-        sign = vals[6]
-        offset = timedelta(hours=int(sign + vals[7]),
-                minutes=int(sign + vals[8]))
-        # subtract the offset, e.g. '-04:00' should be moved up 4 hours
-        return parsed - offset
-    return None
-
 def check_mirror_url(mirror_url):
     url = mirror_url.url + 'lastsync'
     logger.info("checking URL %s", url)
@@ -78,18 +62,14 @@ def check_mirror_url(mirror_url):
         data = result.read()
         result.close()
         end = time.time()
-        # lastsync should be an epoch value, but some mirrors
-        # are creating their own in RFC-3339 format:
-        #     '2010-09-02 11:05:06+02:00'
+        # lastsync should be an epoch value created by us
         parsed_time = None
         try:
             parsed_time = datetime.utcfromtimestamp(int(data))
         except ValueError:
             # it is bad news to try logging the lastsync value;
             # sometimes we get a crazy-encoded web page.
-            logger.info("attempting to parse generated lastsync file"
-                    " from mirror %s", url)
-            parsed_time = parse_rfc3339_datetime(data)
+            pass
 
         log.last_sync = parsed_time
         # if we couldn't parse a time, this is a failure
@@ -154,8 +134,8 @@ class MirrorCheckPool(object):
     @transaction.commit_on_success
     def run(self):
         logger.debug("starting threads")
-        for t in self.threads:
-            t.start()
+        for thread in self.threads:
+            thread.start()
         logger.debug("joining on all threads")
         self.tasks.join()
         logger.debug("processing log entries")
diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py
index 8a628bd4..4e812f2d 100644
--- a/mirrors/management/commands/mirrorresolv.py
+++ b/mirrors/management/commands/mirrorresolv.py
@@ -49,6 +49,6 @@ def resolve_mirrors():
                     mirrorurl.has_ipv4, mirrorurl.has_ipv6)
             mirrorurl.save(force_update=True)
         except socket.error, e:
-            logger.warn("error resolving %s: %s", hostname, e)
+            logger.warn("error resolving %s: %s", mirrorurl.hostname, e)
 
 # vim: set ts=4 sw=4 et:
diff --git a/mirrors/migrations/0007_unique_names_urls.py b/mirrors/migrations/0007_unique_names_urls.py
new file mode 100644
index 00000000..49c0fbb7
--- /dev/null
+++ b/mirrors/migrations/0007_unique_names_urls.py
@@ -0,0 +1,66 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        db.create_unique('mirrors_mirror', ['name'])
+        db.create_unique('mirrors_mirrorurl', ['url'])
+
+    def backwards(self, orm):
+        db.delete_unique('mirrors_mirrorurl', ['url'])
+        db.delete_unique('mirrors_mirror', ['name'])
+
+    models = {
+        'mirrors.mirror': {
+            'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+            'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+            'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+            'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True'})
+        },
+        'mirrors.mirrorlog': {
+            'Meta': {'object_name': 'MirrorLog'},
+            'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+            'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"})
+        },
+        'mirrors.mirrorprotocol': {
+            'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+        },
+        'mirrors.mirrorrsync': {
+            'Meta': {'object_name': 'MirrorRsync'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}),
+            'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"})
+        },
+        'mirrors.mirrorurl': {
+            'Meta': {'object_name': 'MirrorUrl'},
+            'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}),
+            'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.MirrorProtocol']"}),
+            'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['mirrors']
diff --git a/mirrors/migrations/0008_auto__add_field_mirrorurl_country.py b/mirrors/migrations/0008_auto__add_field_mirrorurl_country.py
new file mode 100644
index 00000000..660ac080
--- /dev/null
+++ b/mirrors/migrations/0008_auto__add_field_mirrorurl_country.py
@@ -0,0 +1,67 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'MirrorUrl.country'
+        db.add_column('mirrors_mirrorurl', 'country', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True), keep_default=False)
+
+    def backwards(self, orm):
+        # Deleting field 'MirrorUrl.country'
+        db.delete_column('mirrors_mirrorurl', 'country')
+
+    models = {
+        'mirrors.mirror': {
+            'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+            'country': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+            'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+            'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+            'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True'})
+        },
+        'mirrors.mirrorlog': {
+            'Meta': {'object_name': 'MirrorLog'},
+            'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+            'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"})
+        },
+        'mirrors.mirrorprotocol': {
+            'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+        },
+        'mirrors.mirrorrsync': {
+            'Meta': {'object_name': 'MirrorRsync'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}),
+            'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"})
+        },
+        'mirrors.mirrorurl': {
+            'Meta': {'object_name': 'MirrorUrl'},
+            'country': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}),
+            'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.MirrorProtocol']"}),
+            'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['mirrors']
diff --git a/mirrors/models.py b/mirrors/models.py
index 401821a8..bcde210c 100644
--- a/mirrors/models.py
+++ b/mirrors/models.py
@@ -12,9 +12,9 @@ TIER_CHOICES = (
 )
 
 class Mirror(models.Model):
-    name = models.CharField(max_length=255)
+    name = models.CharField(max_length=255, unique=True)
     tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES)
-    upstream = models.ForeignKey('self', null=True)
+    upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
     country = models.CharField(max_length=255, db_index=True)
     admin_email = models.EmailField(max_length=255, blank=True)
     public = models.BooleanField(default=True)
@@ -54,10 +54,12 @@ class MirrorProtocol(models.Model):
         ordering = ('protocol',)
 
 class MirrorUrl(models.Model):
-    url = models.CharField(max_length=255)
+    url = models.CharField(max_length=255, unique=True)
     protocol = models.ForeignKey(MirrorProtocol, related_name="urls",
-            editable=False)
+            editable=False, on_delete=models.PROTECT)
     mirror = models.ForeignKey(Mirror, related_name="urls")
+    country = models.CharField(max_length=255, blank=True, null=True,
+            db_index=True)
     has_ipv4 = models.BooleanField("IPv4 capable", default=True,
             editable=False)
     has_ipv6 = models.BooleanField("IPv6 capable", default=False,
@@ -73,6 +75,10 @@ class MirrorUrl(models.Model):
     def hostname(self):
         return urlparse(self.url).hostname
 
+    @property
+    def real_country(self):
+        return self.country or self.mirror.country
+
     def clean(self):
         try:
             # Auto-map the protocol field by looking at the URL
diff --git a/mirrors/utils.py b/mirrors/utils.py
index 124b66e6..686ec581 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -7,6 +7,25 @@ import datetime
 
 default_cutoff = datetime.timedelta(hours=24)
 
+def annotate_url(url, delays):
+    '''Given a MirrorURL object, add a few more attributes to it regarding
+    status, including completion_pct, delay, and score.'''
+    url.completion_pct = float(url.success_count) / url.check_count
+    if url.id in delays:
+        url_delays = delays[url.id]
+        url.delay = sum(url_delays, datetime.timedelta()) / len(url_delays)
+        hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0
+
+        if url.completion_pct > 0:
+            divisor = url.completion_pct
+        else:
+            # arbitrary small value
+            divisor = 0.005
+        url.score = (hours + url.duration_avg + url.duration_stddev) / divisor
+    else:
+        url.delay = None
+        url.score = None
+
 @cache_function(300)
 def get_mirror_statuses(cutoff=default_cutoff):
     cutoff_time = datetime.datetime.utcnow() - cutoff
@@ -31,8 +50,8 @@ def get_mirror_statuses(cutoff=default_cutoff):
             check_time__gte=cutoff_time)
     delays = {}
     for log in times:
-        d = log.check_time - log.last_sync
-        delays.setdefault(log.url_id, []).append(d)
+        delay = log.check_time - log.last_sync
+        delays.setdefault(log.url_id, []).append(delay)
 
     if urls:
         last_check = max([u.last_check for u in urls])
@@ -44,29 +63,14 @@ def get_mirror_statuses(cutoff=default_cutoff):
             check_frequency = (check_info['mx'] - check_info['mn']) \
                     / (num_checks - 1)
         else:
-            check_frequency = None;
+            check_frequency = None
     else:
         last_check = None
         num_checks = 0
         check_frequency = None
 
     for url in urls:
-        url.completion_pct = float(url.success_count) / url.check_count
-        if url.id in delays:
-            url_delays = delays[url.id]
-            d = sum(url_delays, datetime.timedelta()) / len(url_delays)
-            url.delay = d
-            hours = d.days * 24.0 + d.seconds / 3600.0
-
-            if url.completion_pct > 0:
-                divisor = url.completion_pct
-            else:
-                # arbitrary small value
-                divisor = 0.005
-            url.score = (hours + url.duration_avg + url.duration_stddev) / divisor
-        else:
-            url.delay = None
-            url.score = None
+        annotate_url(url, delays)
 
     return {
         'cutoff': cutoff,
@@ -82,10 +86,13 @@ def get_mirror_errors(cutoff=default_cutoff):
     errors = MirrorLog.objects.filter(
             is_success=False, check_time__gte=cutoff_time,
             url__mirror__active=True, url__mirror__public=True).values(
-            'url__url', 'url__protocol__protocol', 'url__mirror__country',
-            'error').annotate(
+            'url__url', 'url__country', 'url__protocol__protocol',
+            'url__mirror__country', 'error').annotate(
             error_count=Count('error'), last_occurred=Max('check_time')
             ).order_by('-last_occurred', '-error_count')
-    return list(errors)
+    errors = list(errors)
+    for err in errors:
+        err['country'] = err['url__country'] or err['url__mirror__country']
+    return errors
 
 # vim: set ts=4 sw=4 et:
diff --git a/mirrors/views.py b/mirrors/views.py
index a2b94de8..f03a2e8a 100644
--- a/mirrors/views.py
+++ b/mirrors/views.py
@@ -1,6 +1,5 @@
 from django import forms
 from django.core.serializers.json import DjangoJSONEncoder
-from django.db.models import Avg, Count, Max, Min, StdDev
 from django.db.models import Q
 from django.http import Http404, HttpResponse
 from django.shortcuts import get_object_or_404
@@ -23,10 +22,10 @@ class MirrorlistForm(forms.Form):
 
     def __init__(self, *args, **kwargs):
         super(MirrorlistForm, self).__init__(*args, **kwargs)
-        mirrors = Mirror.objects.filter(active=True).values_list(
+        countries = Mirror.objects.filter(active=True).values_list(
                 'country', flat=True).distinct().order_by('country')
         self.fields['country'].choices = [('all','All')] + make_choice(
-                mirrors)
+                countries)
         self.fields['country'].initial = ['all']
         protos = make_choice(
                 MirrorProtocol.objects.filter(is_download=True))
@@ -61,7 +60,8 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
             mirror__public=True, mirror__active=True, mirror__isos=True
     )
     if countries and 'all' not in countries:
-        qset = qset.filter(mirror__country__in=countries)
+        qset = qset.filter(Q(country__in=countries) |
+                Q(mirror__country__in=countries))
 
     ip_version = Q()
     if ipv4_supported:
@@ -71,7 +71,8 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
     qset = qset.filter(ip_version)
 
     if not use_status:
-        urls = qset.order_by('mirror__country', 'mirror__name', 'url')
+        urls = qset.order_by('mirror__name', 'url')
+        urls = sorted(urls, key=lambda x: x.real_country)
         template = 'mirrors/mirrorlist.txt'
     else:
         status_info = get_mirror_statuses()
@@ -94,20 +95,29 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
             mimetype='text/plain')
 
 def mirrors(request):
-    mirrors = Mirror.objects.select_related().order_by('tier', 'country')
+    mirror_list = Mirror.objects.select_related().order_by('tier', 'country')
     if not request.user.is_authenticated():
-        mirrors = mirrors.filter(public=True, active=True)
+        mirror_list = mirror_list.filter(public=True, active=True)
     return direct_to_template(request, 'mirrors/mirrors.html',
-            {'mirror_list': mirrors})
+            {'mirror_list': mirror_list})
 
 def mirror_details(request, name):
     mirror = get_object_or_404(Mirror, name=name)
     if not request.user.is_authenticated() and \
             (not mirror.public or not mirror.active):
-        # TODO: maybe this should be 403? but that would leak existence
         raise Http404
+
+    status_info = get_mirror_statuses()
+    checked_urls = [url for url in status_info['urls'] \
+            if url.mirror_id == mirror.id]
+    all_urls = mirror.urls.select_related('protocol')
+    # get each item from checked_urls and supplement with anything in all_urls
+    # if it wasn't there
+    all_urls = set(checked_urls).union(all_urls)
+    all_urls = sorted(all_urls, key=lambda x: x.url)
+
     return direct_to_template(request, 'mirrors/mirror_details.html',
-            {'mirror': mirror})
+            {'mirror': mirror, 'urls': all_urls})
 
 def status(request):
     bad_timedelta = datetime.timedelta(days=3)
@@ -149,7 +159,7 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder):
             for attr in self.url_attributes:
                 data[attr] = getattr(obj, attr)
             # separate because it isn't on the URL directly
-            data['country'] = obj.mirror.country
+            data['country'] = obj.real_country
             return data
         if isinstance(obj, MirrorProtocol):
             return unicode(obj)