Merge tag 'release_2015-04-13' into archweb-generic

Fix up 'None' display in places

8 files changed, 410 insertions, 389 deletions

diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py
index 8c17c78f..1f16a375 100644
--- a/mirrors/management/commands/mirrorcheck.py
+++ b/mirrors/management/commands/mirrorcheck.py
@@ -238,7 +238,7 @@ class MirrorCheckPool(object):
         for url in list(urls):
             self.tasks.put(url)
         self.threads = []
-        for i in range(num_threads):
+        for _ in range(num_threads):
             thread = Thread(target=mirror_url_worker,
                     args=(self.tasks, self.logs, location, timeout))
             thread.daemon = True
diff --git a/mirrors/urls.py b/mirrors/urls.py
index b1054380..fc510fbb 100644
--- a/mirrors/urls.py
+++ b/mirrors/urls.py
@@ -1,15 +1,18 @@
 from django.conf.urls import patterns
 
-urlpatterns = patterns('mirrors.views',
-    (r'^$',                     'mirrors', {}, 'mirror-list'),
-    (r'^status/$',              'status',  {}, 'mirror-status'),
-    (r'^status/json/$',         'status_json',  {}, 'mirror-status-json'),
-    (r'^status/tier/(?P<tier>\d+)/$', 'status', {}, 'mirror-status-tier'),
-    (r'^status/tier/(?P<tier>\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'),
-    (r'^locations/json/$',      'locations_json',  {}, 'mirror-locations-json'),
-    (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'),
-    (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'),
-    (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', 'url_details'),
+from .views import mirrors, status, mirror_details, url_details
+from .views.api import status_json, mirror_details_json, locations_json
+
+urlpatterns = patterns('',
+    (r'^$', mirrors, {}, 'mirror-list'),
+    (r'^status/$', status,  {}, 'mirror-status'),
+    (r'^status/json/$', status_json,  {}, 'mirror-status-json'),
+    (r'^status/tier/(?P<tier>\d+)/$', status, {}, 'mirror-status-tier'),
+    (r'^status/tier/(?P<tier>\d+)/json/$', status_json, {}, 'mirror-status-tier-json'),
+    (r'^locations/json/$', locations_json,  {}, 'mirror-locations-json'),
+    (r'^(?P<name>[\.\-\w]+)/$', mirror_details),
+    (r'^(?P<name>[\.\-\w]+)/json/$', mirror_details_json),
+    (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', url_details),
 )
 
 # vim: set ts=4 sw=4 et:
diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py
index bba54ec9..a64656a9 100644
--- a/mirrors/urls_mirrorlist.py
+++ b/mirrors/urls_mirrorlist.py
@@ -1,7 +1,7 @@
 from django.conf.urls import patterns
 
 
-urlpatterns = patterns('mirrors.views',
+urlpatterns = patterns('mirrors.views.mirrorlist',
     (r'^$',         'generate_mirrorlist', {}, 'mirrorlist'),
     (r'^all/$',     'find_mirrors', {'countries': ['all']}),
     (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple',
diff --git a/mirrors/utils.py b/mirrors/utils.py
index 930adb8d..7c2f5d17 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -4,7 +4,6 @@ from django.db import connection
 from django.db.models import Count, Max, Min
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import now
-from django_countries.fields import Country
 
 from main.utils import cache_function, database_vendor
 from .models import MirrorLog, MirrorUrl
@@ -159,9 +158,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False):
     cutoff_time = now() - cutoff
     errors = MirrorLog.objects.filter(
             is_success=False, check_time__gte=cutoff_time,
-            url__mirror__public=True).values(
-            'url__url', 'url__country', 'url__protocol__protocol',
-            'url__mirror__tier', 'error').annotate(
+            url__mirror__public=True).values('url__id', 'error').annotate(
             error_count=Count('error'), last_occurred=Max('check_time')
             ).order_by('-last_occurred', '-error_count')
 
@@ -172,8 +169,11 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False):
                 url__mirror__public=True)
 
     errors = list(errors)
+    to_fetch = [err['url__id'] for err in errors]
+    urls = MirrorUrl.objects.select_related(
+            'mirror', 'protocol').in_bulk(to_fetch)
     for err in errors:
-        err['country'] = Country(err['url__country'], flag_url='')
+        err['url'] = urls[err['url__id']]
     return errors
 
 
@@ -183,12 +183,12 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF):
     status data available, it is used to determine a good choice by looking at
     the last batch of status rows.'''
     cutoff_time = now() - cutoff
-    status_data = MirrorLog.objects.filter(
+    log_data = MirrorLog.objects.filter(
             check_time__gte=cutoff_time).aggregate(
             Max('check_time'), Max('last_sync'))
-    if status_data['check_time__max'] is not None:
-        min_check_time = status_data['check_time__max'] - timedelta(minutes=5)
-        min_sync_time = status_data['last_sync__max'] - timedelta(minutes=20)
+    if log_data['check_time__max'] is not None:
+        min_check_time = log_data['check_time__max'] - timedelta(minutes=5)
+        min_sync_time = log_data['last_sync__max'] - timedelta(minutes=20)
         best_logs = MirrorLog.objects.select_related('url').filter(
                 is_success=True,
                 check_time__gte=min_check_time, last_sync__gte=min_sync_time,
diff --git a/mirrors/views.py b/mirrors/views.py
deleted file mode 100644
index 65fa0123..00000000
--- a/mirrors/views.py
+++ /dev/null
@@ -1,368 +0,0 @@
-from datetime import timedelta
-from itertools import groupby
-import json
-from operator import attrgetter, itemgetter
-
-from django import forms
-from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple
-from django.core.serializers.json import DjangoJSONEncoder
-from django.db import connection
-from django.db.models import Q
-from django.http import Http404, HttpResponse
-from django.shortcuts import get_object_or_404, redirect, render
-from django.utils.timezone import now
-from django.views.decorators.cache import cache_page
-from django.views.decorators.csrf import csrf_exempt
-from django.views.decorators.http import condition
-from django_countries import countries
-from django_countries.fields import Country
-
-from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog,
-        CheckLocation)
-from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF
-
-
-class MirrorlistForm(forms.Form):
-    country = forms.MultipleChoiceField(required=False,
-            widget=SelectMultiple(attrs={'size': '12'}))
-    protocol = forms.MultipleChoiceField(required=False,
-            widget=CheckboxSelectMultiple)
-    ip_version = forms.MultipleChoiceField(required=False,
-            label="IP version", choices=(('4','IPv4'), ('6','IPv6')),
-            widget=CheckboxSelectMultiple)
-    use_mirror_status = forms.BooleanField(required=False)
-
-    def __init__(self, *args, **kwargs):
-        super(MirrorlistForm, self).__init__(*args, **kwargs)
-        fields = self.fields
-        fields['country'].choices = [('all','All')] + self.get_countries()
-        fields['country'].initial = ['all']
-        protos = [(p.protocol, p.protocol) for p in
-                MirrorProtocol.objects.filter(is_download=True)]
-        initial = MirrorProtocol.objects.filter(is_download=True, default=True)
-        fields['protocol'].choices = protos
-        fields['protocol'].initial = [p.protocol for p in initial]
-        fields['ip_version'].initial = ['4']
-
-    def get_countries(self):
-        country_codes = set()
-        country_codes.update(MirrorUrl.objects.filter(active=True,
-                mirror__active=True).exclude(country='').values_list(
-                'country', flat=True).order_by().distinct())
-        code_list = [(code, countries.name(code)) for code in country_codes]
-        return sorted(code_list, key=itemgetter(1))
-
-    def as_div(self):
-        "Returns this form rendered as HTML <divs>s."
-        return self._html_output(
-            normal_row = u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
-            error_row = u'%s',
-            row_ender = '</div>',
-            help_text_html = u' <span class="helptext">%s</span>',
-            errors_on_separate_row = True)
-
-
-@csrf_exempt
-def generate_mirrorlist(request):
-    if request.method == 'POST' or len(request.GET) > 0:
-        form = MirrorlistForm(data=request.REQUEST)
-        if form.is_valid():
-            countries = form.cleaned_data['country']
-            protocols = form.cleaned_data['protocol']
-            use_status = form.cleaned_data['use_mirror_status']
-            ipv4 = '4' in form.cleaned_data['ip_version']
-            ipv6 = '6' in form.cleaned_data['ip_version']
-            return find_mirrors(request, countries, protocols,
-                    use_status, ipv4, ipv6)
-    else:
-        form = MirrorlistForm()
-
-    return render(request, 'mirrors/mirrorlist_generate.html',
-            {'mirrorlist_form': form})
-
-
-def status_filter(original_urls):
-    status_info = get_mirror_statuses()
-    scores = {u.id: u.score for u in status_info['urls']}
-    urls = []
-    for u in original_urls:
-        u.score = scores.get(u.id, None)
-        # also include mirrors that don't have an up to date score
-        # (as opposed to those that have been set with no score)
-        if (u.id not in scores) or (u.score and u.score < 100.0):
-            urls.append(u)
-    # if a url doesn't have a score, treat it as the highest possible
-    return sorted(urls, key=lambda x: x.score or 100.0)
-
-
-def find_mirrors(request, countries=None, protocols=None, use_status=False,
-        ipv4_supported=True, ipv6_supported=True):
-    if not protocols:
-        protocols = MirrorProtocol.objects.filter(is_download=True)
-    elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol:
-        # we already have a queryset, no need to query again
-        pass
-    else:
-        protocols = MirrorProtocol.objects.filter(protocol__in=protocols)
-    qset = MirrorUrl.objects.select_related().filter(
-            protocol__in=protocols, active=True,
-            mirror__public=True, mirror__active=True)
-    if countries and 'all' not in countries:
-        qset = qset.filter(country__in=countries)
-
-    ip_version = Q()
-    if ipv4_supported:
-        ip_version |= Q(has_ipv4=True)
-    if ipv6_supported:
-        ip_version |= Q(has_ipv6=True)
-    qset = qset.filter(ip_version)
-
-    if not use_status:
-        sort_key = attrgetter('country.name', 'mirror.name', 'url')
-        urls = sorted(qset, key=sort_key)
-        template = 'mirrors/mirrorlist.txt'
-    else:
-        urls = status_filter(qset)
-        template = 'mirrors/mirrorlist_status.txt'
-
-    context = {
-        'mirror_urls': urls,
-    }
-    return render(request, template, context, content_type='text/plain')
-
-
-def find_mirrors_simple(request, protocol):
-    if protocol == 'smart':
-        return redirect('mirrorlist_simple', 'http', permanent=True)
-    proto = get_object_or_404(MirrorProtocol, protocol=protocol)
-    return find_mirrors(request, protocols=[proto])
-
-
-def mirrors(request):
-    mirror_list = Mirror.objects.select_related().order_by('tier', 'name')
-    protos = MirrorUrl.objects.values_list(
-            'mirror_id', 'protocol__protocol').order_by(
-            'mirror_id', 'protocol__protocol').distinct()
-    countries = MirrorUrl.objects.values_list(
-            'mirror_id', 'country').order_by(
-            'mirror_id', 'country').distinct()
-
-    if not request.user.is_authenticated():
-        mirror_list = mirror_list.filter(public=True, active=True)
-        protos = protos.filter(
-                mirror__public=True, mirror__active=True, active=True)
-        countries = countries.filter(
-                mirror__public=True, mirror__active=True, active=True)
-
-    protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))}
-    countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))}
-
-    for mirror in mirror_list:
-        item_protos = protos.get(mirror.id, [])
-        mirror.protocols = [item[1] for item in item_protos]
-        mirror.country = None
-        item_countries = countries.get(mirror.id, [])
-        if len(item_countries) == 1:
-            mirror.country = Country(item_countries[0][1])
-
-    return render(request, 'mirrors/mirrors.html',
-            {'mirror_list': mirror_list})
-
-
-def mirror_details(request, name):
-    mirror = get_object_or_404(Mirror, name=name)
-    authorized = request.user.is_authenticated()
-    if not authorized and \
-            (not mirror.public or not mirror.active):
-        raise Http404
-    error_cutoff = timedelta(days=7)
-
-    status_info = get_mirror_statuses(mirror_id=mirror.id,
-            show_all=authorized)
-    checked_urls = {url for url in status_info['urls'] \
-            if url.mirror_id == mirror.id}
-    all_urls = mirror.urls.select_related('protocol')
-    if not authorized:
-        all_urls = all_urls.filter(active=True)
-    all_urls = set(all_urls)
-    # Add dummy data for URLs that we haven't checked recently
-    other_urls = all_urls.difference(checked_urls)
-    for url in other_urls:
-        for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg',
-                'duration_stddev', 'score'):
-            setattr(url, attr, None)
-    all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url'))
-
-    error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff,
-            show_all=True)
-
-    context = {
-        'mirror': mirror,
-        'urls': all_urls,
-        'cutoff': error_cutoff,
-        'error_logs': error_logs,
-    }
-    return render(request, 'mirrors/mirror_details.html', context)
-
-
-def mirror_details_json(request, name):
-    authorized = request.user.is_authenticated()
-    mirror = get_object_or_404(Mirror, name=name)
-    if not authorized and (not mirror.public or not mirror.active):
-        raise Http404
-    status_info = get_mirror_statuses(mirror_id=mirror.id,
-            show_all=authorized)
-    data = status_info.copy()
-    data['version'] = 3
-    to_json = json.dumps(data, ensure_ascii=False,
-            cls=ExtendedMirrorStatusJSONEncoder)
-    response = HttpResponse(to_json, content_type='application/json')
-    return response
-
-
-def url_details(request, name, url_id):
-    url = get_object_or_404(MirrorUrl.objects.select_related(),
-            id=url_id, mirror__name=name)
-    mirror = url.mirror
-    authorized = request.user.is_authenticated()
-    if not authorized and \
-            (not mirror.public or not mirror.active or not url.active):
-        raise Http404
-    error_cutoff = timedelta(days=7)
-    cutoff_time = now() - error_cutoff
-    logs = MirrorLog.objects.select_related('location').filter(
-            url=url, check_time__gte=cutoff_time).order_by('-check_time')
-
-    context = {
-        'url': url,
-        'logs': logs,
-    }
-    return render(request, 'mirrors/url_details.html', context)
-
-
-def status_last_modified(request, *args, **kwargs):
-    cursor = connection.cursor()
-    cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog")
-    return cursor.fetchone()[0]
-
-
-@condition(last_modified_func=status_last_modified)
-def status(request, tier=None):
-    if tier is not None:
-        tier = int(tier)
-        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
-            raise Http404
-    bad_timedelta = timedelta(days=3)
-    status_info = get_mirror_statuses()
-
-    urls = status_info['urls']
-    good_urls = []
-    bad_urls = []
-    for url in urls:
-        # screen by tier if we were asked to
-        if tier is not None and url.mirror.tier != tier:
-            continue
-        # split them into good and bad lists based on delay
-        if url.completion_pct is None:
-            # skip URLs that have never been checked
-            continue
-        elif not url.delay or url.delay > bad_timedelta:
-            bad_urls.append(url)
-        else:
-            good_urls.append(url)
-
-    error_logs = get_mirror_errors()
-    if tier is not None:
-        error_logs = [log for log in error_logs
-                if log['url__mirror__tier'] == tier]
-
-    context = status_info.copy()
-    context.update({
-        'good_urls': sorted(good_urls, key=attrgetter('score')),
-        'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max),
-        'error_logs': error_logs,
-        'tier': tier,
-    })
-    return render(request, 'mirrors/status.html', context)
-
-
-class MirrorStatusJSONEncoder(DjangoJSONEncoder):
-    '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl
-    serialization. The base class takes care of datetime.datetime types.'''
-    url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct',
-            'delay', 'duration_avg', 'duration_stddev', 'score')
-
-    def default(self, obj):
-        if isinstance(obj, timedelta):
-            # always returned as integer seconds
-            return obj.days * 24 * 3600 + obj.seconds
-        if isinstance(obj, MirrorUrl):
-            data = {attr: getattr(obj, attr) for attr in self.url_attributes}
-            country = obj.country
-            data['country'] = unicode(country.name)
-            data['country_code'] = country.code
-            return data
-        if isinstance(obj, MirrorProtocol):
-            return unicode(obj)
-        return super(MirrorStatusJSONEncoder, self).default(obj)
-
-
-class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder):
-    '''Adds URL check history information.'''
-    log_attributes = ('check_time', 'last_sync', 'duration', 'is_success',
-            'location_id')
-
-    def default(self, obj):
-        if isinstance(obj, MirrorUrl):
-            data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
-            cutoff = now() - DEFAULT_CUTOFF
-            data['logs'] = list(obj.logs.filter(
-                    check_time__gte=cutoff).order_by('check_time'))
-            return data
-        if isinstance(obj, MirrorLog):
-            return {attr: getattr(obj, attr) for attr in self.log_attributes}
-        return super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
-
-
-@cache_page(67)
-@condition(last_modified_func=status_last_modified)
-def status_json(request, tier=None):
-    if tier is not None:
-        tier = int(tier)
-        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
-            raise Http404
-    status_info = get_mirror_statuses()
-    data = status_info.copy()
-    if tier is not None:
-        data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier]
-    data['version'] = 3
-    to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder)
-    response = HttpResponse(to_json, content_type='application/json')
-    return response
-
-
-class LocationJSONEncoder(DjangoJSONEncoder):
-    '''Base JSONEncoder extended to handle CheckLocation objects.'''
-
-    def default(self, obj):
-        if isinstance(obj, CheckLocation):
-            return {
-                'id': obj.pk,
-                'hostname': obj.hostname,
-                'source_ip': obj.source_ip,
-                'country': unicode(obj.country.name),
-                'country_code': obj.country.code,
-                'ip_version': obj.ip_version,
-            }
-        return super(LocationJSONEncoder, self).default(obj)
-
-
-def locations_json(request):
-    data = {}
-    data['version'] = 1
-    data['locations'] = list(CheckLocation.objects.all().order_by('pk'))
-    to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder)
-    response = HttpResponse(to_json, content_type='application/json')
-    return response
-
-# vim: set ts=4 sw=4 et:
diff --git a/mirrors/views/__init__.py b/mirrors/views/__init__.py
new file mode 100644
index 00000000..01e8519d
--- /dev/null
+++ b/mirrors/views/__init__.py
@@ -0,0 +1,149 @@
+from datetime import timedelta
+from itertools import groupby
+from operator import attrgetter, itemgetter
+
+from django.db import connection
+from django.http import Http404
+from django.shortcuts import get_object_or_404, render
+from django.utils.timezone import now
+from django.views.decorators.http import condition
+from django_countries.fields import Country
+
+from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog,
+        CheckLocation)
+from ..utils import get_mirror_statuses, get_mirror_errors
+
+
+def mirrors(request):
+    mirror_list = Mirror.objects.select_related().order_by('tier', 'name')
+    protos = MirrorUrl.objects.values_list(
+            'mirror_id', 'protocol__protocol').order_by(
+            'mirror_id', 'protocol__protocol').distinct()
+    countries = MirrorUrl.objects.values_list(
+            'mirror_id', 'country').order_by(
+            'mirror_id', 'country').distinct()
+
+    if not request.user.is_authenticated():
+        mirror_list = mirror_list.filter(public=True, active=True)
+        protos = protos.filter(
+                mirror__public=True, mirror__active=True, active=True)
+        countries = countries.filter(
+                mirror__public=True, mirror__active=True, active=True)
+
+    protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))}
+    countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))}
+
+    for mirror in mirror_list:
+        item_protos = protos.get(mirror.id, [])
+        mirror.protocols = [item[1] for item in item_protos]
+        mirror.country = None
+        item_countries = countries.get(mirror.id, [])
+        if len(item_countries) == 1:
+            mirror.country = Country(item_countries[0][1])
+
+    return render(request, 'mirrors/mirrors.html',
+            {'mirror_list': mirror_list})
+
+
+def mirror_details(request, name):
+    mirror = get_object_or_404(Mirror, name=name)
+    authorized = request.user.is_authenticated()
+    if not authorized and \
+            (not mirror.public or not mirror.active):
+        raise Http404
+    error_cutoff = timedelta(days=7)
+
+    status_info = get_mirror_statuses(mirror_id=mirror.id,
+            show_all=authorized)
+    checked_urls = {url for url in status_info['urls'] \
+            if url.mirror_id == mirror.id}
+    all_urls = mirror.urls.select_related('protocol')
+    if not authorized:
+        all_urls = all_urls.filter(active=True)
+    all_urls = set(all_urls)
+    # Add dummy data for URLs that we haven't checked recently
+    other_urls = all_urls.difference(checked_urls)
+    for url in other_urls:
+        for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg',
+                'duration_stddev', 'score'):
+            setattr(url, attr, None)
+    all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url'))
+
+    error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff,
+            show_all=True)
+
+    context = {
+        'mirror': mirror,
+        'urls': all_urls,
+        'cutoff': error_cutoff,
+        'error_logs': error_logs,
+    }
+    return render(request, 'mirrors/mirror_details.html', context)
+
+
+def url_details(request, name, url_id):
+    url = get_object_or_404(MirrorUrl.objects.select_related(),
+            id=url_id, mirror__name=name)
+    mirror = url.mirror
+    authorized = request.user.is_authenticated()
+    if not authorized and \
+            (not mirror.public or not mirror.active or not url.active):
+        raise Http404
+    error_cutoff = timedelta(days=7)
+    cutoff_time = now() - error_cutoff
+    logs = MirrorLog.objects.select_related('location').filter(
+            url=url, check_time__gte=cutoff_time).order_by('-check_time')
+
+    context = {
+        'url': url,
+        'logs': logs,
+    }
+    return render(request, 'mirrors/url_details.html', context)
+
+
+def status_last_modified(request, *args, **kwargs):
+    cursor = connection.cursor()
+    cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog")
+    return cursor.fetchone()[0]
+
+
+@condition(last_modified_func=status_last_modified)
+def status(request, tier=None):
+    if tier is not None:
+        tier = int(tier)
+        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+            raise Http404
+    bad_timedelta = timedelta(days=3)
+    status_info = get_mirror_statuses()
+
+    urls = status_info['urls']
+    good_urls = []
+    bad_urls = []
+    for url in urls:
+        # screen by tier if we were asked to
+        if tier is not None and url.mirror.tier != tier:
+            continue
+        # split them into good and bad lists based on delay
+        if url.completion_pct is None:
+            # skip URLs that have never been checked
+            continue
+        elif not url.delay or url.delay > bad_timedelta:
+            bad_urls.append(url)
+        else:
+            good_urls.append(url)
+
+    error_logs = get_mirror_errors()
+    if tier is not None:
+        error_logs = [log for log in error_logs
+                if log['url'].mirror.tier == tier]
+
+    context = status_info.copy()
+    context.update({
+        'good_urls': sorted(good_urls, key=attrgetter('score')),
+        'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max),
+        'error_logs': error_logs,
+        'tier': tier,
+    })
+    return render(request, 'mirrors/status.html', context)
+
+# vim: set ts=4 sw=4 et:
diff --git a/mirrors/views/api.py b/mirrors/views/api.py
new file mode 100644
index 00000000..b72585e6
--- /dev/null
+++ b/mirrors/views/api.py
@@ -0,0 +1,108 @@
+from datetime import timedelta
+import json
+
+from django.core.serializers.json import DjangoJSONEncoder
+from django.http import Http404, HttpResponse
+from django.shortcuts import get_object_or_404
+from django.utils.timezone import now
+
+from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog,
+        CheckLocation)
+from ..utils import get_mirror_statuses, DEFAULT_CUTOFF
+
+
+class MirrorStatusJSONEncoder(DjangoJSONEncoder):
+    '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl
+    serialization. The base class takes care of datetime.datetime types.'''
+    url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct',
+            'delay', 'duration_avg', 'duration_stddev', 'score')
+
+    def default(self, obj):
+        if isinstance(obj, timedelta):
+            # always returned as integer seconds
+            return obj.days * 24 * 3600 + obj.seconds
+        if isinstance(obj, MirrorUrl):
+            data = {attr: getattr(obj, attr) for attr in self.url_attributes}
+            country = obj.country
+            data['country'] = unicode(country.name)
+            data['country_code'] = country.code
+            return data
+        if isinstance(obj, MirrorProtocol):
+            return unicode(obj)
+        return super(MirrorStatusJSONEncoder, self).default(obj)
+
+
+class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder):
+    '''Adds URL check history information.'''
+    log_attributes = ('check_time', 'last_sync', 'duration', 'is_success',
+            'location_id')
+
+    def default(self, obj):
+        if isinstance(obj, MirrorUrl):
+            data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+            cutoff = now() - DEFAULT_CUTOFF
+            data['logs'] = list(obj.logs.filter(
+                    check_time__gte=cutoff).order_by('check_time'))
+            return data
+        if isinstance(obj, MirrorLog):
+            data = {attr: getattr(obj, attr) for attr in self.log_attributes}
+            data['error'] = obj.error or None
+            return data
+        return super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+
+
+class LocationJSONEncoder(DjangoJSONEncoder):
+    '''Base JSONEncoder extended to handle CheckLocation objects.'''
+
+    def default(self, obj):
+        if isinstance(obj, CheckLocation):
+            return {
+                'id': obj.pk,
+                'hostname': obj.hostname,
+                'source_ip': obj.source_ip,
+                'country': unicode(obj.country.name),
+                'country_code': obj.country.code,
+                'ip_version': obj.ip_version,
+            }
+        return super(LocationJSONEncoder, self).default(obj)
+
+
+def status_json(request, tier=None):
+    if tier is not None:
+        tier = int(tier)
+        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+            raise Http404
+    status_info = get_mirror_statuses()
+    data = status_info.copy()
+    if tier is not None:
+        data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier]
+    data['version'] = 3
+    to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder)
+    response = HttpResponse(to_json, content_type='application/json')
+    return response
+
+
+def mirror_details_json(request, name):
+    authorized = request.user.is_authenticated()
+    mirror = get_object_or_404(Mirror, name=name)
+    if not authorized and (not mirror.public or not mirror.active):
+        raise Http404
+    status_info = get_mirror_statuses(mirror_id=mirror.id,
+            show_all=authorized)
+    data = status_info.copy()
+    data['version'] = 3
+    to_json = json.dumps(data, ensure_ascii=False,
+            cls=ExtendedMirrorStatusJSONEncoder)
+    response = HttpResponse(to_json, content_type='application/json')
+    return response
+
+
+def locations_json(request):
+    data = {}
+    data['version'] = 1
+    data['locations'] = list(CheckLocation.objects.all().order_by('pk'))
+    to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder)
+    response = HttpResponse(to_json, content_type='application/json')
+    return response
+
+# vim: set ts=4 sw=4 et:
diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py
new file mode 100644
index 00000000..3c68d036
--- /dev/null
+++ b/mirrors/views/mirrorlist.py
@@ -0,0 +1,129 @@
+from operator import attrgetter, itemgetter
+
+from django import forms
+from django.db.models import Q
+from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple
+from django.shortcuts import get_object_or_404, redirect, render
+from django.views.decorators.csrf import csrf_exempt
+from django_countries import countries
+
+from ..models import MirrorUrl, MirrorProtocol
+from ..utils import get_mirror_statuses
+
+
+class MirrorlistForm(forms.Form):
+    country = forms.MultipleChoiceField(required=False,
+            widget=SelectMultiple(attrs={'size': '12'}))
+    protocol = forms.MultipleChoiceField(required=False,
+            widget=CheckboxSelectMultiple)
+    ip_version = forms.MultipleChoiceField(required=False,
+            label="IP version", choices=(('4','IPv4'), ('6','IPv6')),
+            widget=CheckboxSelectMultiple)
+    use_mirror_status = forms.BooleanField(required=False)
+
+    def __init__(self, *args, **kwargs):
+        super(MirrorlistForm, self).__init__(*args, **kwargs)
+        fields = self.fields
+        fields['country'].choices = [('all','All')] + self.get_countries()
+        fields['country'].initial = ['all']
+        protos = [(p.protocol, p.protocol) for p in
+                MirrorProtocol.objects.filter(is_download=True)]
+        initial = MirrorProtocol.objects.filter(is_download=True, default=True)
+        fields['protocol'].choices = protos
+        fields['protocol'].initial = [p.protocol for p in initial]
+        fields['ip_version'].initial = ['4']
+
+    def get_countries(self):
+        country_codes = set()
+        country_codes.update(MirrorUrl.objects.filter(active=True,
+                mirror__active=True).exclude(country='').values_list(
+                'country', flat=True).order_by().distinct())
+        code_list = [(code, countries.name(code)) for code in country_codes]
+        return sorted(code_list, key=itemgetter(1))
+
+    def as_div(self):
+        "Returns this form rendered as HTML <divs>s."
+        return self._html_output(
+            normal_row = u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
+            error_row = u'%s',
+            row_ender = '</div>',
+            help_text_html = u' <span class="helptext">%s</span>',
+            errors_on_separate_row = True)
+
+
+@csrf_exempt
+def generate_mirrorlist(request):
+    if request.method == 'POST' or len(request.GET) > 0:
+        form = MirrorlistForm(data=request.REQUEST)
+        if form.is_valid():
+            countries = form.cleaned_data['country']
+            protocols = form.cleaned_data['protocol']
+            use_status = form.cleaned_data['use_mirror_status']
+            ipv4 = '4' in form.cleaned_data['ip_version']
+            ipv6 = '6' in form.cleaned_data['ip_version']
+            return find_mirrors(request, countries, protocols,
+                    use_status, ipv4, ipv6)
+    else:
+        form = MirrorlistForm()
+
+    return render(request, 'mirrors/mirrorlist_generate.html',
+            {'mirrorlist_form': form})
+
+
+def status_filter(original_urls):
+    status_info = get_mirror_statuses()
+    scores = {u.id: u.score for u in status_info['urls']}
+    urls = []
+    for u in original_urls:
+        u.score = scores.get(u.id, None)
+        # also include mirrors that don't have an up to date score
+        # (as opposed to those that have been set with no score)
+        if (u.id not in scores) or (u.score and u.score < 100.0):
+            urls.append(u)
+    # if a url doesn't have a score, treat it as the highest possible
+    return sorted(urls, key=lambda x: x.score or 100.0)
+
+
+def find_mirrors(request, countries=None, protocols=None, use_status=False,
+        ipv4_supported=True, ipv6_supported=True):
+    if not protocols:
+        protocols = MirrorProtocol.objects.filter(is_download=True)
+    elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol:
+        # we already have a queryset, no need to query again
+        pass
+    else:
+        protocols = MirrorProtocol.objects.filter(protocol__in=protocols)
+    qset = MirrorUrl.objects.select_related().filter(
+            protocol__in=protocols, active=True,
+            mirror__public=True, mirror__active=True)
+    if countries and 'all' not in countries:
+        qset = qset.filter(country__in=countries)
+
+    ip_version = Q()
+    if ipv4_supported:
+        ip_version |= Q(has_ipv4=True)
+    if ipv6_supported:
+        ip_version |= Q(has_ipv6=True)
+    qset = qset.filter(ip_version)
+
+    if not use_status:
+        sort_key = attrgetter('country.name', 'mirror.name', 'url')
+        urls = sorted(qset, key=sort_key)
+        template = 'mirrors/mirrorlist.txt'
+    else:
+        urls = status_filter(qset)
+        template = 'mirrors/mirrorlist_status.txt'
+
+    context = {
+        'mirror_urls': urls,
+    }
+    return render(request, template, context, content_type='text/plain')
+
+
+def find_mirrors_simple(request, protocol):
+    if protocol == 'smart':
+        return redirect('mirrorlist_simple', 'http', permanent=True)
+    proto = get_object_or_404(MirrorProtocol, protocol=protocol)
+    return find_mirrors(request, protocols=[proto])
+
+# vim: set ts=4 sw=4 et: