Choose an up-to-date mirror for download URLs

Given that we collect a lot of mirror status data, we can utilize it to
ensure the download link on the website actually works and newly-added
packages have actually been mirrored out. Add a method that attempts to
use the mirror status data to determine a mirror we should redirect our
download requests to. This can change on a regular basis, and falls back
to the old method if no mirror status data is available.

Signed-off-by: Dan McGee <dan@archlinux.org>

2 files changed, 38 insertions, 9 deletions

diff --git a/mirrors/utils.py b/mirrors/utils.py
index 0f8fef84..619d5f5c 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -27,6 +27,7 @@ def annotate_url(url, delays):
         url.delay = None
         url.score = None
 
+
 @cache_function(123)
 def get_mirror_statuses(cutoff=default_cutoff):
     cutoff_time = utc_now() - cutoff
@@ -81,6 +82,7 @@ def get_mirror_statuses(cutoff=default_cutoff):
         'urls': urls,
     }
 
+
 @cache_function(117)
 def get_mirror_errors(cutoff=default_cutoff):
     cutoff_time = utc_now() - cutoff
@@ -96,4 +98,36 @@ def get_mirror_errors(cutoff=default_cutoff):
         err['country'] = err['url__country'] or err['url__mirror__country']
     return errors
 
+
+@cache_function(295)
+def get_mirror_url_for_download(cutoff=default_cutoff):
+    '''Find a good mirror URL to use for package downloads. If we have mirror
+    status data available, it is used to determine a good choice by looking at
+    the last batch of status rows.'''
+    cutoff_time = utc_now() - cutoff
+    status_data = MirrorLog.objects.filter(
+            check_time__gte=cutoff_time).aggregate(
+            Max('check_time'), Max('last_sync'))
+    if status_data:
+        min_check_time = status_data['check_time__max'] - timedelta(minutes=5)
+        min_sync_time = status_data['last_sync__max'] - timedelta(minutes=30)
+        best_logs = MirrorLog.objects.filter(is_success=True,
+                check_time__gte=min_check_time, last_sync__gte=min_sync_time,
+                url__mirror__public=True, url__mirror__active=True,
+                url__protocol__protocol__iexact='HTTP').order_by(
+                'duration')[:1]
+        if best_logs:
+            return MirrorUrl.objects.get(id=best_logs[0].url_id)
+
+    mirror_urls = MirrorUrl.objects.filter(
+            mirror__public=True, mirror__active=True,
+            protocol__protocol__iexact='HTTP')
+    # look first for an 'Any' URL, then fall back to any HTTP URL
+    filtered_urls = mirror_urls.filter(mirror__country='Any')[:1]
+    if not filtered_urls:
+        filtered_urls = mirror_urls[:1]
+    if not filtered_urls:
+        return None
+    return filtered_urls[0]
+
 # vim: set ts=4 sw=4 et:
diff --git a/packages/views/__init__.py b/packages/views/__init__.py
index 2798fff1..abc86d78 100644
--- a/packages/views/__init__.py
+++ b/packages/views/__init__.py
@@ -14,6 +14,7 @@ from django.views.generic.simple import direct_to_template
 
 from main.models import Package, PackageFile, PackageDepend, Arch, Repo
 from mirrors.models import MirrorUrl
+from mirrors.utils import get_mirror_url_for_download
 from ..models import (PackageRelation, PackageGroup, License,
         Conflict, Provision, Replacement)
 from ..utils import (get_group_info, get_differences_info,
@@ -225,21 +226,15 @@ def files_json(request, name, repo, arch):
 def download(request, name, repo, arch):
     pkg = get_object_or_404(Package,
             pkgname=name, repo__name__iexact=repo, arch__name=arch)
-    mirror_urls = MirrorUrl.objects.filter(
-            mirror__public=True, mirror__active=True,
-            protocol__protocol__iexact='HTTP')
-    # look first for an 'Any' URL, then fall back to any HTTP URL
-    filtered_urls = mirror_urls.filter(mirror__country='Any')[:1]
-    if not filtered_urls:
-        filtered_urls = mirror_urls[:1]
-    if not filtered_urls:
+    url = get_mirror_url_for_download()
+    if not url:
         raise Http404
     arch = pkg.arch.name
     if pkg.arch.agnostic:
         # grab the first non-any arch to fake the download path
         arch = Arch.objects.exclude(agnostic=True)[0].name
     values = {
-        'host': filtered_urls[0].url,
+        'host': url.url,
         'arch': arch,
         'repo': pkg.repo.name.lower(),
         'file': pkg.filename,