Merge branch 'archweb-generic' into master-nomake

Conflicts:
	README.md

30 files changed, 394 insertions, 112 deletions

diff --git a/README.BRANDING b/README.BRANDING
index a3855f77..e75eaeb9 100644
--- a/README.BRANDING
+++ b/README.BRANDING
@@ -9,6 +9,7 @@ Files used to configure branding/url stuff
  * `settings.py`
  * `templates/templatetags/package_extras.py`
  * `main/templatetags/wiki.py`
+ * `main/templatetags/bugs.py`
 
 Files with minor Arch stuff that's just easier to patch
 -------------------------------------------------------
diff --git a/TODO b/TODO
deleted file mode 100644
index 608d8470..00000000
--- a/TODO
+++ /dev/null
@@ -1,4 +0,0 @@
-TODO:
-    - refactor stats by templates in dashboard, maybe a templatetag
-
-
diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 8b591aeb..1945469f 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -82,8 +82,7 @@ class RepoPackage(object):
             'md5sum', 'sha256sum', 'url', 'packager' )
     number = ( 'csize', 'isize' )
     collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends',
-            'conflicts', 'provides', 'replaces', 'groups', 'license',
-            'files' )
+            'conflicts', 'provides', 'replaces', 'groups', 'license')
 
     def __init__(self, repo):
         self.repo = repo
@@ -98,7 +97,6 @@ class RepoPackage(object):
             setattr(self, k, ())
         self.builddate = None
         self.files = None
-        self.has_files = False
 
     def populate(self, values):
         for k, v in values.iteritems():
@@ -122,14 +120,22 @@ class RepoPackage(object):
                     logger.warning(
                             'Package %s had unparsable build date %s',
                             self.name, v[0])
-            elif k == 'files':
-                self.files = tuple(v)
-                self.has_files = True
             else:
                 # anything left in collections
                 setattr(self, k, tuple(v))
 
     @property
+    def files_list(self):
+        data_file = io.TextIOWrapper(io.BytesIO(self.files), encoding='UTF-8')
+        try:
+            info = parse_info(data_file)
+        except UnicodeDecodeError:
+            logger.warn("Could not correctly decode files list for %s",
+                    self.name)
+            return None
+        return info['files']
+
+    @property
     def full_version(self):
         '''Very similar to the main.models.Package method.'''
         if self.epoch > 0:
@@ -265,6 +271,24 @@ def delete_pkg_files(dbpkg):
     cursor.execute('DELETE FROM package_files WHERE pkg_id = %s', [dbpkg.id])
 
 
+def batched_bulk_create(model, all_objects):
+    cutoff = 10000
+    length = len(all_objects)
+    if length < cutoff:
+        return model.objects.bulk_create(all_objects)
+
+    def chunks():
+        offset = 0
+        while offset < length:
+            yield all_objects[offset:offset + cutoff]
+            offset += cutoff
+
+    for items in chunks():
+        ret = model.objects.bulk_create(items)
+
+    return ret
+
+
 def populate_files(dbpkg, repopkg, force=False):
     if not force:
         if not pkg_same_version(repopkg, dbpkg):
@@ -278,15 +302,18 @@ def populate_files(dbpkg, repopkg, force=False):
             return
 
     # only delete files if we are reading a DB that contains them
-    if repopkg.has_files:
+    if repopkg.files:
+        files = repopkg.files_list
+        # we had files data, but it couldn't be parsed, so skip
+        if not files:
+            return
         delete_pkg_files(dbpkg)
         logger.info("adding %d files for package %s",
-                len(repopkg.files), dbpkg.pkgname)
+                len(files), dbpkg.pkgname)
         pkg_files = []
         # sort in normal alpha-order that pacman uses, rather than makepkg's
         # default breadth-first, directory-first ordering
-        files = sorted(repopkg.files)
-        for f in files:
+        for f in sorted(files):
             if '/' in f:
                 dirname, filename = f.rsplit('/', 1)
                 dirname += '/'
@@ -299,7 +326,7 @@ def populate_files(dbpkg, repopkg, force=False):
                     directory=dirname,
                     filename=filename)
             pkg_files.append(pkgfile)
-        PackageFile.objects.bulk_create(pkg_files)
+        batched_bulk_create(PackageFile, pkg_files)
         dbpkg.files_last_update = now()
         dbpkg.save()
 
@@ -494,24 +521,27 @@ def parse_repo(repopath):
 
     repodb = tarfile.open(repopath, "r")
     logger.debug("Starting package parsing")
-    dbfiles = ('desc', 'depends', 'files')
     newpkg = lambda: RepoPackage(reponame)
     pkgs = defaultdict(newpkg)
     for tarinfo in repodb.getmembers():
         if tarinfo.isreg():
             pkgid, fname = os.path.split(tarinfo.name)
-            if fname not in dbfiles:
-                continue
-            data_file = repodb.extractfile(tarinfo)
-            data_file = io.TextIOWrapper(io.BytesIO(data_file.read()),
-                    encoding='UTF-8')
-            try:
-                pkgs[pkgid].populate(parse_info(data_file))
-            except UnicodeDecodeError:
-                logger.warn("Could not correctly decode %s, skipping file",
-                        tarinfo.name)
-            data_file.close()
-            del data_file
+            if fname == 'files':
+                # don't parse yet for speed and memory consumption reasons
+                files_data = repodb.extractfile(tarinfo)
+                pkgs[pkgid].files = files_data.read()
+                del files_data
+            elif fname in ('desc', 'depends'):
+                data_file = repodb.extractfile(tarinfo)
+                data_file = io.TextIOWrapper(io.BytesIO(data_file.read()),
+                        encoding='UTF-8')
+                try:
+                    pkgs[pkgid].populate(parse_info(data_file))
+                except UnicodeDecodeError:
+                    logger.warn("Could not correctly decode %s, skipping file",
+                            tarinfo.name)
+                data_file.close()
+                del data_file
 
             logger.debug("Done parsing file %s/%s", pkgid, fname)
 
diff --git a/main/admin.py b/main/admin.py
index 6aff12e5..ec2b5bc8 100644
--- a/main/admin.py
+++ b/main/admin.py
@@ -1,23 +1,27 @@
 from django.contrib import admin
 from main.models import Arch, Donor, Package, Repo
 
+
 class DonorAdmin(admin.ModelAdmin):
     list_display = ('name', 'visible', 'created')
     list_filter = ('visible', 'created')
     search_fields = ('name',)
     exclude = ('created',)
 
+
 class ArchAdmin(admin.ModelAdmin):
     list_display = ('name', 'agnostic', 'required_signoffs')
     list_filter = ('agnostic',)
     search_fields = ('name',)
 
+
 class RepoAdmin(admin.ModelAdmin):
     list_display = ('name', 'testing', 'staging', 'bugs_project',
             'bugs_category', 'svn_root')
     list_filter = ('testing', 'staging')
     search_fields = ('name',)
 
+
 class PackageAdmin(admin.ModelAdmin):
     list_display = ('pkgname', 'full_version', 'repo', 'arch', 'packager',
             'last_update', 'build_date')
diff --git a/main/models.py b/main/models.py
index 8d9d4c89..bf7a9409 100644
--- a/main/models.py
+++ b/main/models.py
@@ -97,7 +97,7 @@ class Package(models.Model):
     pkgrel = models.CharField(max_length=255)
     epoch = models.PositiveIntegerField(default=0)
     pkgdesc = models.TextField('description', null=True)
-    url = models.CharField(max_length=255, null=True)
+    url = models.CharField('URL', max_length=255, null=True)
     filename = models.CharField(max_length=255)
     compressed_size = PositiveBigIntegerField()
     installed_size = PositiveBigIntegerField()
diff --git a/main/templatetags/bugs.py b/main/templatetags/bugs.py
new file mode 100644
index 00000000..d5764435
--- /dev/null
+++ b/main/templatetags/bugs.py
@@ -0,0 +1,6 @@
+from django import template
+register = template.Library()
+
+@register.simple_tag
+def bug_link(bugid):
+    return "<a href=\"https://bugs.archlinux.org/task/"+bugid+"\">FS#"+bugid+"</a>"
diff --git a/main/templatetags/pgp.py b/main/templatetags/pgp.py
index e93e5bca..cc080439 100644
--- a/main/templatetags/pgp.py
+++ b/main/templatetags/pgp.py
@@ -3,6 +3,7 @@ from django.conf import settings
 from django.utils.html import conditional_escape
 from django.utils.safestring import mark_safe
 
+from ..utils import cache_function
 from devel.models import DeveloperKey
 
 register = template.Library()
@@ -42,15 +43,22 @@ def pgp_key_link(key_id, link_text=None):
     values = (url, format_key(key_id), link_text)
     return '<a href="%s" title="PGP key search for %s">%s</a>' % values
 
-@register.simple_tag
-def user_pgp_key_link(key_id):
-    normalized = key_id[-16:]
+
+@cache_function(1800)
+def name_for_key(normalized):
     try:
         matching_key = DeveloperKey.objects.select_related(
                 'owner').get(key=normalized, owner_id__isnull=False)
+        return matching_key.owner.get_full_name()
     except DeveloperKey.DoesNotExist:
-        return pgp_key_link(key_id)
-    return pgp_key_link(key_id, matching_key.owner.get_full_name())
+        return None
+
+
+@register.simple_tag
+def user_pgp_key_link(key_id):
+    normalized = key_id[-16:]
+    name = name_for_key(normalized)
+    return pgp_key_link(key_id, name)
 
 
 @register.filter(needs_autoescape=True)
diff --git a/mirrors/admin.py b/mirrors/admin.py
index e35d9ce7..17365486 100644
--- a/mirrors/admin.py
+++ b/mirrors/admin.py
@@ -54,7 +54,7 @@ class MirrorAdminForm(forms.ModelForm):
         model = Mirror
         fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email',
                 'public', 'active', 'isos', 'rsync_user', 'rsync_password',
-                'notes')
+                'bug', 'notes')
 
     upstream = forms.ModelChoiceField(
             queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1),
@@ -67,6 +67,7 @@ class MirrorAdmin(admin.ModelAdmin):
             'isos', 'admin_email', 'alternate_email')
     list_filter = ('tier', 'active', 'public')
     search_fields = ('name', 'admin_email', 'alternate_email')
+    readonly_fields = ('created',)
     inlines = [
             MirrorUrlInlineAdmin,
             MirrorRsyncInlineAdmin,
diff --git a/mirrors/migrations/0027_auto__add_field_mirror_bug.py b/mirrors/migrations/0027_auto__add_field_mirror_bug.py
new file mode 100644
index 00000000..f7304ba8
--- /dev/null
+++ b/mirrors/migrations/0027_auto__add_field_mirror_bug.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+    def forwards(self, orm):
+        db.add_column(u'mirrors_mirror', 'bug',
+                      self.gf('django.db.models.fields.PositiveIntegerField')(null=True),
+                      keep_default=False)
+        # UPDATE mirrors_mirror m
+        # SET bug = (
+        #   SELECT extracted::int FROM (
+        #    SELECT id, substring(notes from 'FS#([\d]+)') AS extracted FROM mirrors_mirror
+        #   ) a
+        #   WHERE extracted IS NOT NULL AND a.id = m.id
+        # )
+        # WHERE notes LIKE '%FS#%';
+
+    def backwards(self, orm):
+        db.delete_column(u'mirrors_mirror', 'bug')
+
+
+    models = {
+        u'mirrors.checklocation': {
+            'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'},
+            'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {}),
+            'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'})
+        },
+        u'mirrors.mirror': {
+            'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+            'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+            'bug': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+            'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+            'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+            'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+        },
+        u'mirrors.mirrorlog': {
+            'Meta': {'object_name': 'MirrorLog'},
+            'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+            'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}),
+            'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"})
+        },
+        u'mirrors.mirrorprotocol': {
+            'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+            'created': ('django.db.models.fields.DateTimeField', [], {}),
+            'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+        },
+        u'mirrors.mirrorrsync': {
+            'Meta': {'ordering': "('ip',)", 'object_name': 'MirrorRsync'},
+            'created': ('django.db.models.fields.DateTimeField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}),
+            'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"})
+        },
+        u'mirrors.mirrorurl': {
+            'Meta': {'object_name': 'MirrorUrl'},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {}),
+            'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}),
+            'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}),
+            'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['mirrors']
diff --git a/mirrors/models.py b/mirrors/models.py
index da3d8c0d..57664562 100644
--- a/mirrors/models.py
+++ b/mirrors/models.py
@@ -1,3 +1,4 @@
+from datetime import timedelta
 import socket
 from urlparse import urlparse
 
@@ -28,6 +29,7 @@ class Mirror(models.Model):
     isos = models.BooleanField("ISOs", default=True)
     rsync_user = models.CharField(max_length=50, blank=True, default='')
     rsync_password = models.CharField(max_length=50, blank=True, default='')
+    bug = models.PositiveIntegerField("Flyspray bug", null=True, blank=True)
     notes = models.TextField(blank=True)
     created = models.DateTimeField(editable=False)
 
@@ -158,6 +160,14 @@ class MirrorLog(models.Model):
     is_success = models.BooleanField(default=True)
     error = models.TextField(blank=True, default='')
 
+    def delay(self):
+        if self.last_sync is None:
+            return None
+        # sanity check, this shouldn't happen
+        if self.check_time < self.last_sync:
+            return timedelta()
+        return self.check_time - self.last_sync
+
     def __unicode__(self):
         return "Check of %s at %s" % (self.url.url, self.check_time)
 
diff --git a/mirrors/templatetags/mirror_status.py b/mirrors/templatetags/mirror_status.py
index 9a363fbe..b3810d9a 100644
--- a/mirrors/templatetags/mirror_status.py
+++ b/mirrors/templatetags/mirror_status.py
@@ -1,6 +1,5 @@
 from datetime import timedelta
 from django import template
-from django.template.defaultfilters import floatformat
 
 register = template.Library()
 
@@ -27,10 +26,16 @@ def hours(value):
     return '%d hours' % hrs
 
 @register.filter
-def percentage(value, arg=-1):
+def floatvalue(value, arg=2):
+    if value is None:
+        return u''
+    return '%.*f' % (arg, value)
+
+@register.filter
+def percentage(value, arg=1):
     if not value and type(value) != float:
         return u''
     new_val = value * 100.0
-    return floatformat(new_val, arg) + '%'
+    return '%.*f%%' % (arg, new_val)
 
 # vim: set ts=4 sw=4 et:
diff --git a/mirrors/urls.py b/mirrors/urls.py
index 7cf76aa1..b1054380 100644
--- a/mirrors/urls.py
+++ b/mirrors/urls.py
@@ -9,6 +9,7 @@ urlpatterns = patterns('mirrors.views',
     (r'^locations/json/$',      'locations_json',  {}, 'mirror-locations-json'),
     (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'),
     (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'),
+    (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', 'url_details'),
 )
 
 # vim: set ts=4 sw=4 et:
diff --git a/mirrors/utils.py b/mirrors/utils.py
index 633731a7..0dd26ae0 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -22,7 +22,8 @@ def dictfetchall(cursor):
     ]
 
 @cache_function(178)
-def status_data(cutoff_time, mirror_id=None):
+def status_data(cutoff=DEFAULT_CUTOFF, mirror_id=None):
+    cutoff_time = now() - cutoff
     if mirror_id is not None:
         params = [cutoff_time, mirror_id]
         mirror_where = 'AND u.mirror_id = %s'
@@ -113,20 +114,19 @@ def annotate_url(url, url_data):
         url.score = (hours + url.duration_avg + stddev) / divisor
 
 
-def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None):
+def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False):
     cutoff_time = now() - cutoff
 
-    # TODO: this prevents grabbing data points from any mirror that was active,
-    # receiving checks, and then marked private. we can probably be smarter and
-    # filter the data later?
-    valid_urls = MirrorUrl.objects.filter(active=True,
-            mirror__active=True, mirror__public=True,
+    valid_urls = MirrorUrl.objects.filter(
             logs__check_time__gte=cutoff_time).distinct()
 
     if mirror_id:
         valid_urls = valid_urls.filter(mirror_id=mirror_id)
+    if not show_all:
+        valid_urls = valid_urls.filter(active=True, mirror__active=True,
+                mirror__public=True)
 
-    url_data = status_data(cutoff_time, mirror_id)
+    url_data = status_data(cutoff, mirror_id)
     urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter(
             id__in=valid_urls).order_by('mirror__id', 'url')
 
@@ -159,11 +159,11 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None):
     }
 
 
-def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None):
+def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False):
     cutoff_time = now() - cutoff
     errors = MirrorLog.objects.filter(
-            is_success=False, check_time__gte=cutoff_time, url__active=True,
-            url__mirror__active=True, url__mirror__public=True).values(
+            is_success=False, check_time__gte=cutoff_time,
+            url__mirror__public=True).values(
             'url__url', 'url__country', 'url__protocol__protocol',
             'url__mirror__tier', 'error').annotate(
             error_count=Count('error'), last_occurred=Max('check_time')
@@ -171,6 +171,9 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None):
 
     if mirror_id:
         errors = errors.filter(url__mirror_id=mirror_id)
+    if not show_all:
+        errors = errors.filter(url__active=True, url__mirror__active=True,
+                url__mirror__public=True)
 
     errors = list(errors)
     for err in errors:
diff --git a/mirrors/views.py b/mirrors/views.py
index ec056696..34336165 100644
--- a/mirrors/views.py
+++ b/mirrors/views.py
@@ -153,15 +153,20 @@ def mirrors(request):
 
 def mirror_details(request, name):
     mirror = get_object_or_404(Mirror, name=name)
-    if not request.user.is_authenticated() and \
+    authorized = request.user.is_authenticated()
+    if not authorized and \
             (not mirror.public or not mirror.active):
         raise Http404
     error_cutoff = timedelta(days=7)
 
-    status_info = get_mirror_statuses(mirror_id=mirror.id)
+    status_info = get_mirror_statuses(mirror_id=mirror.id,
+            show_all=authorized)
     checked_urls = {url for url in status_info['urls'] \
             if url.mirror_id == mirror.id}
-    all_urls = set(mirror.urls.filter(active=True).select_related('protocol'))
+    all_urls = mirror.urls.select_related('protocol')
+    if not authorized:
+        all_urls = all_urls.filter(active=True)
+    all_urls = set(all_urls)
     # Add dummy data for URLs that we haven't checked recently
     other_urls = all_urls.difference(checked_urls)
     for url in other_urls:
@@ -170,7 +175,8 @@ def mirror_details(request, name):
             setattr(url, attr, None)
     all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url'))
 
-    error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff)
+    error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff,
+            show_all=True)
 
     context = {
         'mirror': mirror,
@@ -180,9 +186,12 @@ def mirror_details(request, name):
     }
     return render(request, 'mirrors/mirror_details.html', context)
 
+
 def mirror_details_json(request, name):
+    authorized = request.user.is_authenticated()
     mirror = get_object_or_404(Mirror, name=name)
-    status_info = get_mirror_statuses(mirror_id=mirror.id)
+    status_info = get_mirror_statuses(mirror_id=mirror.id,
+            show_all=authorized)
     data = status_info.copy()
     data['version'] = 3
     to_json = json.dumps(data, ensure_ascii=False,
@@ -191,6 +200,26 @@ def mirror_details_json(request, name):
     return response
 
 
+def url_details(request, name, url_id):
+    url = get_object_or_404(MirrorUrl.objects.select_related(),
+            id=url_id, mirror__name=name)
+    mirror = url.mirror
+    authorized = request.user.is_authenticated()
+    if not authorized and \
+            (not mirror.public or not mirror.active or not url.active):
+        raise Http404
+    error_cutoff = timedelta(days=7)
+    cutoff_time = now() - error_cutoff
+    logs = MirrorLog.objects.select_related('location').filter(
+            url=url, check_time__gte=cutoff_time).order_by('-check_time')
+
+    context = {
+        'url': url,
+        'logs': logs,
+    }
+    return render(request, 'mirrors/url_details.html', context)
+
+
 def status(request, tier=None):
     if tier is not None:
         tier = int(tier)
diff --git a/packages/admin.py b/packages/admin.py
index 4680c755..5df0043a 100644
--- a/packages/admin.py
+++ b/packages/admin.py
@@ -15,10 +15,9 @@ class PackageRelationAdmin(admin.ModelAdmin):
 class FlagRequestAdmin(admin.ModelAdmin):
     list_display = ('pkgbase', 'full_version', 'repo', 'created', 'who',
             'is_spam', 'is_legitimate', 'message')
-    list_filter = ('is_spam', 'is_legitimate', 'repo')
+    list_filter = ('is_spam', 'is_legitimate', 'repo', 'created')
     search_fields = ('pkgbase', 'user_email', 'message')
     ordering = ('-created',)
-    date_hierarchy = 'created'
 
     def get_queryset(self, request):
         qs = super(FlagRequestAdmin, self).queryset(request)
@@ -28,19 +27,17 @@ class FlagRequestAdmin(admin.ModelAdmin):
 class SignoffAdmin(admin.ModelAdmin):
     list_display = ('pkgbase', 'full_version', 'arch', 'repo',
             'user', 'created', 'revoked')
-    list_filter = ('arch', 'repo', 'user')
+    list_filter = ('arch', 'repo', 'user', 'created')
     search_fields = ('pkgbase', 'user__username')
     ordering = ('-created',)
-    date_hierarchy = 'created'
 
 
 class SignoffSpecificationAdmin(admin.ModelAdmin):
     list_display = ('pkgbase', 'full_version', 'arch', 'repo',
             'user', 'created', 'comments')
-    list_filter = ('arch', 'repo', 'user')
+    list_filter = ('arch', 'repo', 'user', 'created')
     search_fields = ('pkgbase', 'user__username')
     ordering = ('-created',)
-    date_hierarchy = 'created'
 
     def get_queryset(self, request):
         qs = super(SignoffSpecificationAdmin, self).queryset(request)
@@ -50,10 +47,9 @@ class SignoffSpecificationAdmin(admin.ModelAdmin):
 class UpdateAdmin(admin.ModelAdmin):
     list_display = ('pkgname', 'repo', 'arch', 'action_flag',
             'old_version', 'new_version', 'created')
-    list_filter = ('action_flag', 'repo', 'arch')
+    list_filter = ('action_flag', 'repo', 'arch', 'created')
     search_fields = ('pkgname',)
     ordering = ('-created',)
-    date_hierarchy = 'created'
     raw_id_fields = ('package',)
 
 
diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py
index 20c127c1..70456985 100644
--- a/packages/templatetags/package_extras.py
+++ b/packages/templatetags/package_extras.py
@@ -67,13 +67,16 @@ def do_buildsortqs(parser, token):
 
 
 @register.simple_tag
-def pkg_details_link(pkg, link_title=None):
+def pkg_details_link(pkg, link_title=None, honor_flagged=False):
     if not pkg:
         return link_title or ''
     if link_title is None:
         link_title = pkg.pkgname
+    link_content = link_title
+    if honor_flagged and pkg.flag_date:
+        link_content = '<span class="flagged">%s</span>' % link_title
     link = '<a href="%s" title="View package details for %s">%s</a>'
-    return link % (pkg.get_absolute_url(), pkg.pkgname, link_title)
+    return link % (pkg.get_absolute_url(), pkg.pkgname, link_content)
 
 
 @register.simple_tag
diff --git a/public/utils.py b/public/utils.py
index fcfd0f77..11091883 100644
--- a/public/utils.py
+++ b/public/utils.py
@@ -2,7 +2,7 @@ from collections import defaultdict
 from operator import attrgetter
 
 from main.models import Arch, Repo, Package
-from main.utils import cache_function, groupby_preserve_order, PackageStandin
+from main.utils import groupby_preserve_order, PackageStandin
 
 class RecentUpdate(object):
     def __init__(self, packages):
@@ -58,7 +58,6 @@ class RecentUpdate(object):
         return "RecentUpdate '%s %s' <%d packages>" % (
                 self.pkgbase, self.version, len(self.packages))
 
-@cache_function(62)
 def get_recent_updates(number=15, testing=True, staging=False):
     repos = Repo.objects.all()
     if not testing:
diff --git a/public/views.py b/public/views.py
index 92065e3a..f04f357b 100644
--- a/public/views.py
+++ b/public/views.py
@@ -20,12 +20,14 @@ from .utils import get_recent_updates
 @cache_control(max_age=300)
 def index(request):
     if request.user.is_authenticated():
-        pkgs = get_recent_updates(testing=True, staging=True)
+        def updates():
+            return get_recent_updates(testing=True, staging=True)
     else:
-        pkgs = get_recent_updates()
+        def updates():
+            return get_recent_updates()
     context = {
         'news_updates': News.objects.order_by('-postdate', '-id')[:15],
-        'pkg_updates': pkgs,
+        'pkg_updates': updates,
     }
     return render(request, 'public/index.html', context)
 
diff --git a/requirements.txt b/requirements.txt
index 08d89107..de8a04e9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,10 @@
 -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin
-Django==1.6
+Django==1.6.1
 IPy==0.81
 Markdown==2.3.1
-South==0.8.2
+South==0.8.4
 bencode==1.0
 django-countries==1.5
-jsmin==2.0.6
+jsmin==2.0.8
 pgpdump==1.4
 pytz>=2013.8
diff --git a/requirements_prod.txt b/requirements_prod.txt
index 565c3c6c..e609c5d6 100644
--- a/requirements_prod.txt
+++ b/requirements_prod.txt
@@ -1,11 +1,11 @@
 -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin
-Django==1.6
+Django==1.6.1
 IPy==0.81
 Markdown==2.3.1
-South==0.8.2
+South==0.8.4
 bencode==1.0
 django-countries==1.5
-jsmin==2.0.6
+jsmin==2.0.8
 pgpdump==1.4
 psycopg2==2.5.1
 pyinotify==0.9.4
diff --git a/templates/admin/index.html b/templates/admin/index.html
index 203206d5..fddd55e5 100644
--- a/templates/admin/index.html
+++ b/templates/admin/index.html
@@ -31,11 +31,15 @@
 
 {% if app_list %}
     {% for app in app_list %}
-        <div class="module">
-        <table summary="{% blocktrans with name=app.name %}Models available in the {{ name }} application.{% endblocktrans %}">
-        <caption><a href="{{ app.app_url }}" class="section">{% blocktrans with name=app.name %}{{ name }}{% endblocktrans %}</a></caption>
+        <div class="app-{{ app.app_label }} module">
+        <table>
+        <caption>
+            <a href="{{ app.app_url }}" class="section" title="{% blocktrans with name=app.name %}Models in the {{ name }} application{% endblocktrans %}">
+                {% blocktrans with name=app.name %}{{ name }}{% endblocktrans %}
+            </a>
+        </caption>
         {% for model in app.models %}
-            <tr>
+            <tr class="model-{{ model.object_name|lower }}">
             {% if model.admin_url %}
                 <th scope="row"><a href="{{ model.admin_url }}">{{ model.name }}</a></th>
             {% else %}
diff --git a/templates/devel/admin_log.html b/templates/devel/admin_log.html
index 1629c104..05130491 100644
--- a/templates/devel/admin_log.html
+++ b/templates/devel/admin_log.html
@@ -48,7 +48,7 @@
                 {% if entry.is_deletion %}
                 {{ entry.object_repr }}
                 {% else %}
-                <a href="/admin/{{ entry.get_admin_url }}">{{ entry.object_repr }}</a>
+                <a href="{{ entry.get_admin_url }}">{{ entry.object_repr }}</a>
                 {% endif %}
             </td>
             <td>{{ entry.change_message }}</td>
diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html
index 5138f3b9..b75c5316 100644
--- a/templates/mirrors/mirror_details.html
+++ b/templates/mirrors/mirror_details.html
@@ -42,6 +42,10 @@
             <td>{{ mirror.active|yesno|capfirst }}</td>
         </tr>
         <tr>
+            <th>Created:</th>
+            <td>{{ mirror.created }}</td>
+        </tr>
+        <tr>
             <th>Rsync IPs:</th>
             <td class="wrap">{{mirror.rsync_ips.all|join:', '}}</td>
         </tr>
@@ -54,8 +58,12 @@
             <td>{% if mirror.alternate_email %}<a href="mailto:{{ mirror.alternate_email }}">{{ mirror.alternate_email }}</a>{% else %}None{% endif %}</td>
         </tr>
         <tr>
+            <th>Flyspray Issue:</th>
+            <td>{% if mirror.bug %}{% bug_link mirror.bug %}{% endif %}</td>
+        </tr>
+        <tr>
             <th>Notes:</th>
-            <td>{{ mirror.notes|linebreaks }}</td>
+            <td class="wrap">{{ mirror.notes|linebreaks }}</td>
         </tr>
         <tr>
             <th>Upstream:</th>
@@ -67,14 +75,13 @@
         <tr>
             <th>Downstream:</th>
             {% with mirror.downstream as ds_mirrors %}
-            <td>{% if ds_mirrors %}
+            <td class="wrap">{% if ds_mirrors %}
                 {% for ds in ds_mirrors %}
                 <a href="{{ ds.get_absolute_url }}"
-                    title="Mirror details for {{ ds.name }}">{{ ds.name }}</a>
-                {% if not ds.active %}<span class="testing-dep">(inactive)</span>{% endif %}
-                {% if not ds.public %}<span class="testing-dep">(private)</span>{% endif %}
-                <br/>
-                {% endfor %}
+                    title="Mirror details for {{ ds.name }}">{{ ds.name }}</a>{% comment %}
+                {% endcomment %}{% if not ds.active %} <span class="testing-dep">(inactive)</span>{% endif %}{% comment %}
+                {% endcomment %}{% if not ds.public %} <span class="testing-dep">(private)</span>{% endif %}{% comment %}
+                {% endcomment %}{% if not forloop.last %}, {% endif %}{% endfor %}
                 {% else %}None{% endif %}</td>
             {% endwith %}
         </tr>
@@ -96,7 +103,8 @@
                 <th>μ Delay (hh:mm)</th>
                 <th>μ Duration (secs)</th>
                 <th>σ Duration (secs)</th>
-                <th>Mirror Score</th>
+                <th>Score</th>
+                <th>Details</th>
             </tr>
         </thead>
         <tbody>
@@ -110,9 +118,10 @@
                 <td>{{ m_url.last_sync|date:'Y-m-d H:i'|default:'unknown' }}</td>
                 <td>{{ m_url.completion_pct|percentage:1 }}</td>
                 <td>{{ m_url.delay|duration|default:'unknown' }}</td>
-                <td>{{ m_url.duration_avg|floatformat:2 }}</td>
-                <td>{{ m_url.duration_stddev|floatformat:2 }}</td>
-                <td>{{ m_url.score|floatformat:1|default:'∞' }}</td>
+                <td>{{ m_url.duration_avg|floatvalue:2 }}</td>
+                <td>{{ m_url.duration_stddev|floatvalue:2 }}</td>
+                <td>{{ m_url.score|floatvalue:1|default:'∞' }}</td>
+                <td><a href="{{ m_url.id }}/">Details</a></td>
             </tr>
             {% endfor %}
         </tbody>
diff --git a/templates/mirrors/mirrorlist_status.txt b/templates/mirrors/mirrorlist_status.txt
index cdbc7adb..746aae76 100644
--- a/templates/mirrors/mirrorlist_status.txt
+++ b/templates/mirrors/mirrorlist_status.txt
@@ -1,4 +1,4 @@
-{% comment %}
+{% load mirror_status %}{% comment %}
 Yes, ugly templates are ugly, but in order to keep line breaks where we want
 them, sacrifices have to be made. If editing this template, it is easiest to
 forget about where line breaks are happening until you are done getting the
@@ -9,6 +9,6 @@ content right, and then go back later to fix it all up.
 ## Generated on {% now "Y-m-d" %}
 ##
 {% for mirror_url in mirror_urls %}
-## Score: {{ mirror_url.score|floatformat:1|default:'unknown' }}, {{ mirror_url.country.name|default:'Worldwide' }}
+## Score: {{ mirror_url.score|floatvalue:1|default:'unknown' }}, {{ mirror_url.country.name|default:'Worldwide' }}
 #Server = {{ mirror_url.url}}$repo/os/$arch{% endfor %}
 {% endautoescape %}
diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html
index 5a275e33..4d75cd8e 100644
--- a/templates/mirrors/status.html
+++ b/templates/mirrors/status.html
@@ -59,7 +59,7 @@
 
     <a name="outofsync" id="outofsync"></a>
     <h3>Out of Sync Mirrors</h3>
-    {% with urls=bad_urls  table_id='outofsync_mirrors' %}
+    {% with urls=bad_urls table_id='outofsync_mirrors' %}
     {% include "mirrors/status_table.html" %}
     {% endwith %}
 
diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html
index 6fc07a31..00b9c1df 100644
--- a/templates/mirrors/status_table.html
+++ b/templates/mirrors/status_table.html
@@ -20,9 +20,9 @@
             <td class="country">{% country_flag m_url.country %}{{ m_url.country.name }}</td>
             <td>{{ m_url.completion_pct|percentage:1 }}</td>
             <td>{{ m_url.delay|duration|default:'unknown' }}</td>
-            <td>{{ m_url.duration_avg|floatformat:2 }}</td>
-            <td>{{ m_url.duration_stddev|floatformat:2 }}</td>
-            <td>{{ m_url.score|floatformat:1|default:'∞' }}</td>
+            <td>{{ m_url.duration_avg|floatvalue:2 }}</td>
+            <td>{{ m_url.duration_stddev|floatvalue:2 }}</td>
+            <td>{{ m_url.score|floatvalue:1|default:'∞' }}</td>
         </tr>{% endfor %}
     </tbody>
 </table>
diff --git a/templates/mirrors/url_details.html b/templates/mirrors/url_details.html
new file mode 100644
index 00000000..0b9d2916
--- /dev/null
+++ b/templates/mirrors/url_details.html
@@ -0,0 +1,89 @@
+{% extends "base.html" %}
+{% load cycle from future %}
+{% load static from staticfiles %}
+{% load mirror_status %}
+{% load flags %}
+
+{% block title %}Arch Linux - {{ url.url }} - URL Details{% endblock %}
+
+{% block head %}<link rel="stylesheet" type="text/css" href="{% static "flags/fam.css" %}" media="screen, projection" />{% endblock %}
+
+{% block content %}
+<div class="box">
+    <h2>URL Details: {{ url.url }}</h2>
+
+    <table class="compact">
+        <tr>
+            <th>URL:</th>
+            <td>{{ url.url }}</td>
+        </tr>
+        <tr>
+            <th>Protocol:</th>
+            <td>{{ url.protocol }}</td>
+        </tr>
+        <tr>
+            <th>Country:</th>
+            <td class="country">{% country_flag url.country %}{{ url.country.name }}</td>
+        </tr>
+        <tr>
+            <th>IPv4:</th>
+            <td>{{ url.has_ipv4|yesno|capfirst }}</td>
+        </tr>
+        <tr>
+            <th>IPv6:</th>
+            <td>{{ url.has_ipv6|yesno|capfirst }}</td>
+        </tr>
+        {% if user.is_authenticated %}
+        <tr>
+            <th>Active:</th>
+            <td>{{ url.active|yesno|capfirst }}</td>
+        </tr>
+        <tr>
+            <th>Created:</th>
+            <td>{{ url.created }}</td>
+        </tr>
+        {% endif %}
+    </table>
+
+    <h3>Check Logs</h3>
+
+    <table id="check_logs" class="results">
+        <thead>
+            <tr>
+                <th>Check Time</th>
+                <th>Check Location</th>
+                <th>Check IP</th>
+                <th>Last Sync</th>
+                <th>Delay (hh:mm)</th>
+                <th>Duration (secs)</th>
+                <th>Success?</th>
+                <th>Error Message</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for log in logs %}<tr class="{% cycle 'odd' 'even' %}">
+                <td>{{ log.check_time|date:'Y-m-d H:i' }}</td>
+                <td class="country">{% country_flag log.location.country %}{{ log.location.country.name }}</td>
+                <td>{{ log.location.source_ip }}</td>
+                <td>{{ log.last_sync|date:'Y-m-d H:i' }}</td>
+                <td>{{ log.delay|duration }}</td>
+                <td>{{ log.duration|floatvalue }}</td>
+                <td>{{ log.is_success|yesno|capfirst }}</td>
+                <td class="wrap">{{ log.error|linebreaksbr }}</td>
+            </tr>{% endfor %}
+        </tbody>
+    </table>
+</div>
+{% endblock %}
+
+{% block script_block %}
+{% load cdn %}{% jquery %}{% jquery_tablesorter %}
+<script type="text/javascript" src="{% static "archweb.js" %}"></script>
+<script type="text/javascript">
+$(document).ready(function() {
+    $("#check_logs:has(tbody tr)").tablesorter(
+        {widgets: ['zebra'], sortList: [[0,1]],
+        headers: { 5: { sorter: 'mostlydigit' } } });
+});
+</script>
+{% endblock %}
diff --git a/templates/packages/differences.html b/templates/packages/differences.html
index 0ff5e2ee..6220392c 100644
--- a/templates/packages/differences.html
+++ b/templates/packages/differences.html
@@ -1,6 +1,7 @@
 {% extends "base.html" %}
 {% load cycle from future %}
 {% load static from staticfiles %}
+{% load package_extras %}
 
 {% block title %}{{ BRANDING_DISTRONAME }} - Package Differences Reports{% endblock %}
 {% block navbarclass %}anb-packages{% endblock %}
@@ -74,14 +75,10 @@
                 <td>{{ diff.pkgname }}</td>
                 <td>{{ diff.repo.name }}</td>
                 {% if diff.pkg_a %}
-                <td><a href="{{ diff.pkg_a.get_absolute_url }}"
-                        title="View package details for {{ diff.pkg_a.pkgname }}">
-                        <span{% if diff.pkg_a.flag_date %} class="flagged"{% endif %}>{{ diff.pkg_a.full_version }}</span></a></td>
+                <td>{% pkg_details_link diff.pkg_a diff.pkg_a.full_version True %}</td>
                 {% else %}<td>-</td>{% endif %}
                 {% if diff.pkg_b %}
-                <td><a href="{{ diff.pkg_b.get_absolute_url }}"
-                        title="View package details for {{ diff.pkg_b.pkgname }}">
-                        <span{% if diff.pkg_b.flag_date %} class="flagged"{% endif %}>{{ diff.pkg_b.full_version }}</span></a></td>
+                <td>{% pkg_details_link diff.pkg_b diff.pkg_b.full_version True %}</td>
                 {% else %}<td>-</td>{% endif %}
             </tr>
             {% endfor %}
@@ -107,12 +104,10 @@
         <tbody>
             {% for pkg1, pkg2 in multilib_differences %}
             <tr class="{% cycle 'odd' 'even' %}">
-                <td><a href="{{ pkg1.get_absolute_url }}"
-                        title="View package details for {{ pkg1.pkgname }}">{{ pkg1.pkgname }}</a></td>
+                <td>{% pkg_details_link pkg1 %}</td>
                 <td><span{% if pkg1.flag_date %} class="flagged"{% endif %}>{{ pkg1.full_version }}</span></td>
                 <td><span{% if pkg2.flag_date %} class="flagged"{% endif %}>{{ pkg2.full_version }}</span></td>
-                <td><a href="{{ pkg2.get_absolute_url }}"
-                        title="View package details for {{ pkg2.pkgname }}">{{ pkg2.pkgname }}</a></td>
+                <td>{% pkg_details_link pkg2 %}</td>
                 <td>{{ pkg2.repo }}</td>
                 <td>{{ pkg1.last_update|date }}</td>
                 <td>{{ pkg2.last_update|date }}</td>
diff --git a/templates/public/index.html b/templates/public/index.html
index 3a5d98d6..f577dd9f 100644
--- a/templates/public/index.html
+++ b/templates/public/index.html
@@ -108,7 +108,7 @@
 </div>
 {% endcache %}
 
-{% cache 59 main-page-right secure %}
+{% cache 115 main-page-right secure %}
 <div id="nav-sidebar" class="widget">
     <h4>Documentation</h4>
 
diff --git a/templates/public/keys.html b/templates/public/keys.html
index ab89423e..0818719c 100644
--- a/templates/public/keys.html
+++ b/templates/public/keys.html
@@ -86,16 +86,16 @@
             </tr>
         </thead>
         <tbody>
-            {% for user in active_users %}
+            {% for user in active_users %}{% with user_key=user.userprofile.pgp_key %}
             <tr>
                 <th>{{ user.get_full_name }}</th>
-                <td>{% pgp_key_link user.userprofile.pgp_key %}</td>
+                <td>{% pgp_key_link user_key %}</td>
                 {% spaceless %}{% for key in keys %}
-                {% signature_exists signatures key.pgp_key user.userprofile.pgp_key as signed %}
+                {% signature_exists signatures key.pgp_key user_key as signed %}
                 <td class="signed-{{ signed|yesno }}">{{ signed|yesno|capfirst }}</td>
                 {% endfor %}{% endspaceless %}
             </tr>
-            {% endfor %}
+            {% endwith %}{% endfor %}
         </tbody>
     </table>
 </div>