From 76516cae45f3d1080065608bdb8f2d086322012f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 19:45:57 -0500 Subject: Don't limit protocols returned by mirror status function If results weren't available for certain URLs, they won't show up anyway in this list, and if we start to check rsync URLs, then we want their values to come back in this status list. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 32fa3587..54de567e 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -32,11 +32,9 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=default_cutoff): cutoff_time = utc_now() - cutoff - protocols = list(MirrorProtocol.objects.filter(is_download=True)) # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, - protocol__in=protocols, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), -- cgit v1.2.3-54-g00ecf From ae1c526ffbe908322f0dd8d8805360b81ab22b0f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 20:35:50 -0500 Subject: Add ability to restrict status report to single tier This should make it easier to catch errors in our Tier 1 mirrors. Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/utils.py | 2 +- mirrors/views.py | 19 ++++++++++++++++--- templates/mirrors/status.html | 4 ++-- 4 files changed, 20 insertions(+), 6 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/urls.py b/mirrors/urls.py index f002e9d6..bb4eb969 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -4,6 +4,7 @@ (r'^$', 'mirrors', {}, 'mirror-list'), (r'^status/$', 'status', {}, 'mirror-status'), (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), + (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), ) diff --git a/mirrors/utils.py b/mirrors/utils.py index 54de567e..2014411d 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -88,7 +88,7 @@ def get_mirror_errors(cutoff=default_cutoff): is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'error').annotate( + 'url__mirror__country', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') errors = list(errors) diff --git a/mirrors/views.py b/mirrors/views.py index b0be6238..8f092be7 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,7 +13,7 @@ from django.views.generic.simple import direct_to_template from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -184,7 +184,11 @@ def mirror_details(request, name): {'mirror': mirror, 'urls': all_urls}) -def status(request): +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in TIER_CHOICES]: + raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() @@ -192,17 +196,26 @@ def status(request): good_urls = [] bad_urls = [] for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue # split them into good and bad lists based on delay if not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url__mirror__tier'] == tier] + context = status_info.copy() context.update({ 'good_urls': sorted(good_urls, key=attrgetter('score')), 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': get_mirror_errors(), + 'error_logs': error_logs, + 'tier': tier, }) return direct_to_template(request, 'mirrors/status.html', context) diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 2c350f56..472e9501 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -2,11 +2,11 @@ {% load static from staticfiles %} {% load mirror_status %} -{% block title %}Arch Linux - Mirror Status{% endblock %} +{% block title %}Arch Linux - Mirror Status{% if tier != None %} - Tier {{ tier }}{% endif %}{% endblock %} {% block content %}
-

Mirror Status

+

Mirror Status{% if tier != None %} - Tier {{ tier }}{% endif %}

This page reports the status of all known, public, and active Arch Linux mirrors. All data on this page reflects the status of the mirrors within the last {{ cutoff|hours }}. All listed times are UTC. The check script runs -- cgit v1.2.3-54-g00ecf From 0f3c894e7a0f573fa0198459150f387c3a7f23ae Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 20:38:01 -0500 Subject: Don't include StdDev on sqlite3 mirror status query Because this function isn't shipped by default, it makes more sense to just omit it completely from the query we do to build the tables on this page when in development. Substitute 0.0 for the value so the rest of the calculations and display work as expected. Signed-off-by: Dan McGee --- mirrors/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 2014411d..9aa8e0f5 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -3,7 +3,7 @@ from django.db.models import Avg, Count, Max, Min, StdDev from django_countries.fields import Country -from main.utils import cache_function, utc_now +from main.utils import cache_function, utc_now, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl @@ -40,8 +40,11 @@ def get_mirror_statuses(cutoff=default_cutoff): success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration'), - duration_stddev=StdDev('logs__duration')) + duration_avg=Avg('logs__duration')) + + vendor = database_vendor(MirrorUrl) + if vendor != 'sqlite': + urls.annotate(duration_stddev=StdDev('logs__duration')) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the @@ -70,6 +73,9 @@ def get_mirror_statuses(cutoff=default_cutoff): check_frequency = None for url in urls: + # fake the standard deviation for local testing setups + if vendor == 'sqlite': + setattr(url, 'duration_stddev', 0.0) annotate_url(url, delays) return { -- cgit v1.2.3-54-g00ecf From 9c7350650e66b5eb6228778e335a160be5ea7f98 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 21:53:35 -0500 Subject: Correctly reassign queryset with added annotation in mirror status This was a dumb oversight on my part in commit 0f3c894e7a0. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 9aa8e0f5..f2c98ee0 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -44,7 +44,7 @@ def get_mirror_statuses(cutoff=default_cutoff): vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': - urls.annotate(duration_stddev=StdDev('logs__duration')) + urls = urls.annotate(duration_stddev=StdDev('logs__duration')) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the -- cgit v1.2.3-54-g00ecf From c0bf9e20660cfae7ea8994472555bba23398b598 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:19:48 -0500 Subject: Remove custom utc_now() function, use django.utils.timezone.now() This was around from the time when we handled timezones sanely and Django did not; now that we are on 1.4 we no longer need our own code to handle this. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 9 +++++---- devel/models.py | 5 +++-- devel/views.py | 12 ++++++------ main/models.py | 5 +++-- main/utils.py | 9 ++------- mirrors/management/commands/mirrorcheck.py | 7 ++++--- mirrors/utils.py | 17 +++++++++-------- news/models.py | 11 +++++------ packages/management/commands/signoff_report.py | 8 ++++---- packages/views/flag.py | 8 ++++---- packages/views/signoff.py | 4 ++-- releng/management/commands/syncisos.py | 5 ++--- 12 files changed, 49 insertions(+), 51 deletions(-) (limited to 'mirrors/utils.py') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 2d9b68b2..e69691db 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -28,10 +28,11 @@ from django.core.management.base import BaseCommand, CommandError from django.db import connections, router, transaction from django.db.utils import IntegrityError +from django.utils.timezone import now from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update @@ -318,7 +319,7 @@ def populate_files(dbpkg, repopkg, force=False): filename=filename) pkg_files.append(pkgfile) batched_bulk_create(PackageFile, pkg_files) - dbpkg.files_last_update = utc_now() + dbpkg.files_last_update = now() dbpkg.save() @@ -388,7 +389,7 @@ def db_update(archname, reponame, pkgs, force=False): dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=utc_now()) + populate_pkg(dbpkg, pkg, timestamp=now()) Update.objects.log_update(None, dbpkg) except IntegrityError: logger.warning("Could not add package %s; " @@ -417,7 +418,7 @@ def db_update(archname, reponame, pkgs, force=False): if not force and pkg_same_version(pkg, dbpkg): continue elif not force: - timestamp = utc_now() + timestamp = now() # The odd select_for_update song and dance here are to ensure # simultaneous updates don't happen on a package, causing diff --git a/devel/models.py b/devel/models.py index fd5df00a..9b6f07a7 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,10 +4,11 @@ from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User +from django.utils.timezone import now from django_countries import CountryField from .fields import PGPKeyField -from main.utils import make_choice, utc_now +from main.utils import make_choice class UserProfile(models.Model): @@ -105,7 +106,7 @@ def set_last_modified(sender, **kwargs): signal handler.''' obj = kwargs['instance'] if hasattr(obj, 'last_modified'): - obj.last_modified = utc_now() + obj.last_modified = now() # connect signals needed to keep cache in line with reality diff --git a/devel/views.py b/devel/views.py index 78ed26f2..143b12bf 100644 --- a/devel/views.py +++ b/devel/views.py @@ -24,11 +24,11 @@ from django.views.generic.simple import direct_to_template from django.utils.encoding import force_unicode from django.utils.http import http_date +from django.utils.timezone import now from .models import UserProfile from main.models import Package, PackageFile, TodolistPkg from main.models import Arch, Repo -from main.utils import utc_now from news.models import News from packages.models import PackageRelation, Signoff, Depend from packages.utils import get_signoff_groups @@ -122,15 +122,15 @@ def clock(request): else: dev.last_action = None - now = utc_now() + current_time = now() page_dict = { 'developers': devs, - 'utc_now': now, + 'utc_now': current_time, } response = direct_to_template(request, 'devel/clock.html', page_dict) if not response.has_header('Expires'): - expire_time = now.replace(second=0, microsecond=0) + expire_time = current_time.replace(second=0, microsecond=0) expire_time += timedelta(minutes=1) expire_time = time.mktime(expire_time.timetuple()) response['Expires'] = http_date(expire_time) @@ -198,12 +198,12 @@ def report(request, report_name, username=None): if report_name == 'old': title = 'Packages last built more than one year ago' - cutoff = utc_now() - timedelta(days=365) + cutoff = now() - timedelta(days=365) packages = packages.filter( build_date__lt=cutoff).order_by('build_date') elif report_name == 'long-out-of-date': title = 'Packages marked out-of-date more than 90 days ago' - cutoff = utc_now() - timedelta(days=90) + cutoff = now() - timedelta(days=90) packages = packages.filter( flag_date__lt=cutoff).order_by('flag_date') elif report_name == 'big': diff --git a/main/models.py b/main/models.py index 04d8da8f..6c9dfe4d 100644 --- a/main/models.py +++ b/main/models.py @@ -6,9 +6,10 @@ from django.db import models from django.contrib.auth.models import User from django.contrib.sites.models import Site +from django.utils.timezone import now from .fields import PositiveBigIntegerField -from .utils import cache_function, set_created_field, utc_now +from .utils import cache_function, set_created_field class TodolistManager(models.Manager): @@ -385,7 +386,7 @@ class Meta: def set_todolist_fields(sender, **kwargs): todolist = kwargs['instance'] if not todolist.date_added: - todolist.date_added = utc_now() + todolist.date_added = now() # connect signals needed to keep cache in line with reality from main.utils import refresh_latest diff --git a/main/utils.py b/main/utils.py index 879abfb9..0b6849a4 100644 --- a/main/utils.py +++ b/main/utils.py @@ -5,10 +5,10 @@ from datetime import datetime import hashlib -from pytz import utc from django.core.cache import cache from django.db import connections, router +from django.utils.timezone import now CACHE_TIMEOUT = 1800 @@ -94,17 +94,12 @@ def retrieve_latest(sender, latest_by=None): return None -def utc_now(): - '''Returns a timezone-aware UTC date representing now.''' - return datetime.utcnow().replace(tzinfo=utc) - - def set_created_field(sender, **kwargs): '''This will set the 'created' field on any object to the current UTC time if it is unset. For use as a pre_save signal handler.''' obj = kwargs['instance'] if hasattr(obj, 'created') and not obj.created: - obj.created = utc_now() + obj.created = now() def database_vendor(model, mode='read'): diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7a133cbf..e09ea680 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -29,8 +29,9 @@ from django.core.management.base import NoArgsCommand from django.db import transaction +from django.utils.timezone import now -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -83,7 +84,7 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) try: @@ -136,7 +137,7 @@ def check_mirror_url(mirror_url, timeout): def check_rsync_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) tempdir = tempfile.mkdtemp() lastsync_path = os.path.join(tempdir, 'lastsync') diff --git a/mirrors/utils.py b/mirrors/utils.py index f2c98ee0..bf030d39 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,14 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.timezone import now from django_countries.fields import Country -from main.utils import cache_function, utc_now, database_vendor +from main.utils import cache_function, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl -default_cutoff = timedelta(hours=24) +DEFAULT_CUTOFF = timedelta(hours=24) def annotate_url(url, delays): '''Given a MirrorURL object, add a few more attributes to it regarding @@ -30,8 +31,8 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, @@ -88,8 +89,8 @@ def get_mirror_statuses(cutoff=default_cutoff): @cache_function(117) -def get_mirror_errors(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_errors(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( @@ -105,11 +106,11 @@ def get_mirror_errors(cutoff=default_cutoff): @cache_function(295) -def get_mirror_url_for_download(cutoff=default_cutoff): +def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): '''Find a good mirror URL to use for package downloads. If we have mirror status data available, it is used to determine a good choice by looking at the last batch of status rows.''' - cutoff_time = utc_now() - cutoff + cutoff_time = now() - cutoff status_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) diff --git a/news/models.py b/news/models.py index 95026e1d..2efea579 100644 --- a/news/models.py +++ b/news/models.py @@ -1,8 +1,7 @@ from django.db import models from django.contrib.auth.models import User from django.contrib.sites.models import Site - -from main.utils import utc_now +from django.utils.timezone import now class News(models.Model): @@ -29,13 +28,13 @@ class Meta: def set_news_fields(sender, **kwargs): news = kwargs['instance'] - now = utc_now() - news.last_modified = now + current_time = now() + news.last_modified = current_time if not news.postdate: - news.postdate = now + news.postdate = current_time # http://diveintomark.org/archives/2004/05/28/howto-atom-id news.guid = 'tag:%s,%s:%s' % (Site.objects.get_current(), - now.strftime('%Y-%m-%d'), news.get_absolute_url()) + current_time.strftime('%Y-%m-%d'), news.get_absolute_url()) # connect signals needed to keep cache in line with reality from main.utils import refresh_latest diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py index ddf930db..72fcbe1e 100644 --- a/packages/management/commands/signoff_report.py +++ b/packages/management/commands/signoff_report.py @@ -15,6 +15,7 @@ from django.contrib.sites.models import Site from django.db.models import Count from django.template import loader, Context +from django.utils.timezone import now from collections import namedtuple from datetime import timedelta @@ -23,7 +24,6 @@ import sys from main.models import Repo -from main.utils import utc_now from packages.models import Signoff from packages.utils import get_signoff_groups @@ -66,9 +66,9 @@ def generate_report(email, repo_name): new_hours = 24 old_days = 14 - now = utc_now() - new_cutoff = now - timedelta(hours=new_hours) - old_cutoff = now - timedelta(days=old_days) + current_time = now() + new_cutoff = current_time - timedelta(hours=new_hours) + old_cutoff = current_time - timedelta(days=old_days) if len(signoff_groups) == 0: # no need to send an email at all diff --git a/packages/views/flag.py b/packages/views/flag.py index 7fa2d508..f3db93b3 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -5,12 +5,12 @@ from django.db import transaction from django.shortcuts import get_object_or_404, redirect from django.template import loader, Context +from django.utils.timezone import now from django.views.generic.simple import direct_to_template from django.views.decorators.cache import cache_page, never_cache from ..models import FlagRequest from main.models import Package -from main.utils import utc_now class FlagForm(forms.Form): @@ -76,10 +76,10 @@ def flag(request, name, repo, arch): @transaction.commit_on_success def perform_updates(): - now = utc_now() - pkgs.update(flag_date=now) + current_time = now() + pkgs.update(flag_date=current_time) # store our flag request - flag_request = FlagRequest(created=now, + flag_request = FlagRequest(created=current_time, user_email=email, message=message, ip_address=ip_addr, pkgbase=pkg.pkgbase, version=version, repo=pkg.repo, diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 61d949fc..7aa39106 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -8,11 +8,11 @@ from django.db import transaction from django.http import HttpResponse, Http404 from django.shortcuts import get_list_or_404, redirect, render +from django.utils.timezone import now from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template from main.models import Package, Arch, Repo -from main.utils import utc_now from ..models import SignoffSpecification, Signoff from ..utils import (get_signoff_groups, approved_by_signoffs, PackageSignoffGroup) @@ -45,7 +45,7 @@ def signoff_package(request, name, repo, arch, revoke=False): package, request.user, False) except Signoff.DoesNotExist: raise Http404 - signoff.revoked = utc_now() + signoff.revoked = now() signoff.save() created = False else: diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index 62f005ff..223c771b 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -4,8 +4,8 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError +from django.utils.timezone import now -from main.utils import utc_now from releng.models import Iso @@ -54,9 +54,8 @@ def handle(self, *args, **options): existing.active = True existing.removed = None existing.save() - now = utc_now() # and then mark all other names as no longer active Iso.objects.filter(active=True).exclude(name__in=active_isos).update( - active=False, removed=now) + active=False, removed=now()) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From f0b7e73de61c03a5018ed352605e6329611448d2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 10 Oct 2012 20:17:55 -0500 Subject: Make mirror log time query a bit more efficient We don't need the full mirror log objects; we just need a very small subset of values from them here to do the required math and object building. Signed-off-by: Dan McGee --- mirrors/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index bf030d39..0a32b766 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -50,12 +50,14 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the # results here. - times = MirrorLog.objects.filter(is_success=True, last_sync__isnull=False, + times = MirrorLog.objects.values_list( + 'url_id', 'check_time', 'last_sync').filter( + is_success=True, last_sync__isnull=False, check_time__gte=cutoff_time) delays = {} - for log in times: - delay = log.check_time - log.last_sync - delays.setdefault(log.url_id, []).append(delay) + for url_id, check_time, last_sync in times: + delay = check_time - last_sync + delays.setdefault(url_id, []).append(delay) if urls: last_check = max([u.last_check for u in urls]) -- cgit v1.2.3-54-g00ecf From 86102c6e645451c03e3e576060eba7f93350bf6b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 14:19:23 -0600 Subject: Allow filtering retrieved mirror statuses by mirror_id When we don't need them all, no need to fetch them all. Let the database do the work for us, hopefully. Signed-off-by: Dan McGee --- mirrors/utils.py | 19 +++++++++++++++---- mirrors/views.py | 8 ++++---- 2 files changed, 19 insertions(+), 8 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 0a32b766..ba027c99 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -31,7 +31,7 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( @@ -43,6 +43,9 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): last_check=Max('logs__check_time'), duration_avg=Avg('logs__duration')) + if mirror_ids: + urls = urls.filter(mirror_id__in=mirror_ids) + vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': urls = urls.annotate(duration_stddev=StdDev('logs__duration')) @@ -54,6 +57,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): 'url_id', 'check_time', 'last_sync').filter( is_success=True, last_sync__isnull=False, check_time__gte=cutoff_time) + if mirror_ids: + times = times.filter(url__mirror_id__in=mirror_ids) delays = {} for url_id, check_time, last_sync in times: delay = check_time - last_sync @@ -62,8 +67,10 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): if urls: last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) - check_info = MirrorLog.objects.filter( - check_time__gte=cutoff_time).aggregate( + check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) + if mirror_ids: + check_info = check_info.filter(url__mirror_id__in=mirror_ids) + check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: check_frequency = (check_info['mx'] - check_info['mn']) \ @@ -91,7 +98,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): @cache_function(117) -def get_mirror_errors(cutoff=DEFAULT_CUTOFF): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, @@ -100,6 +107,10 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF): 'url__mirror__country', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') + + if mirror_ids: + urls = urls.filter(mirror_id__in=mirror_ids) + errors = list(errors) for err in errors: ctry_code = err['url__country'] or err['url__mirror__country'] diff --git a/mirrors/views.py b/mirrors/views.py index cbd86611..4b9721dc 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -14,7 +14,7 @@ from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog -from .utils import get_mirror_statuses, get_mirror_errors +from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF COUNTRY_LOOKUP = dict(COUNTRIES) @@ -171,7 +171,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) checked_urls = [url for url in status_info['urls'] \ if url.mirror_id == mirror.id] all_urls = mirror.urls.select_related('protocol') @@ -186,7 +186,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 # include only URLs for this particular mirror @@ -266,7 +266,7 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): def default(self, obj): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - cutoff = now() - timedelta(hours=24) + cutoff = now() - DEFAULT_CUTOFF data['logs'] = obj.logs.filter(check_time__gte=cutoff) return data if isinstance(obj, MirrorLog): -- cgit v1.2.3-54-g00ecf From e26d5722289bd2e972633891d8dac09296b0cbc4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Nov 2012 14:55:37 -0600 Subject: Mirror graph tweaking after usage with real data * Clamp y-axis minimum to 0. * Don't plot `is_success == false` values. * Ensure URLs are sorted predictably. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 5 ++++- mirrors/utils.py | 3 ++- mirrors/views.py | 2 -- 3 files changed, 6 insertions(+), 4 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 1c352a9f..decc8fb8 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -25,7 +25,7 @@ function mirror_status(chart_id, data_url) { d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) ]).nice(d3.time.hour); y.domain([ - d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.duration; }); }), + 0, d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) ]).nice(); @@ -111,6 +111,9 @@ function mirror_status(chart_id, data_url) { return { url: url.url, logs: jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } return { duration: log.duration, check_time: new Date(log.check_time) diff --git a/mirrors/utils.py b/mirrors/utils.py index ba027c99..85e4ee93 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -41,7 +41,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')) + duration_avg=Avg('logs__duration')).order_by( + 'mirror', 'url') if mirror_ids: urls = urls.filter(mirror_id__in=mirror_ids) diff --git a/mirrors/views.py b/mirrors/views.py index be01e919..5e374b4d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -189,8 +189,6 @@ def mirror_details_json(request, name): status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 - # include only URLs for this particular mirror - data['urls'] = [url for url in data['urls'] if url.mirror_id == mirror.id] to_json = json.dumps(data, ensure_ascii=False, cls=ExtendedMirrorStatusJSONEncoder) response = HttpResponse(to_json, mimetype='application/json') -- cgit v1.2.3-54-g00ecf From a2cfa7edbbed8edb1ad4d3391c6edb055c13de1b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 15:39:56 -0600 Subject: Optimize mirror status data fetching Now that we have as many mirror URLs as we do, we can do a better job fetching and aggregating this data. The prior method resulted in a rather unwieldy query being pushed down to the database with a horrendously long GROUP BY clause. Instead of trying to group by everything at once so we can retrieve mirror URL info at the same time, separate the two queries- one for getting URL performance data, one for the qualitative data. The impetus behind fixing this is the PostgreSQL slow query log in production; this currently shows up the most of any queries we run in the system. Signed-off-by: Dan McGee --- mirrors/utils.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 85e4ee93..07a7138f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -33,23 +33,26 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff - # I swear, this actually has decent performance... - urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')).order_by( - 'mirror', 'url') - - if mirror_ids: - urls = urls.filter(mirror_id__in=mirror_ids) + duration_avg=Avg('logs__duration')) vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': - urls = urls.annotate(duration_stddev=StdDev('logs__duration')) + url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) + + urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + mirror__active=True, mirror__public=True, + logs__check_time__gte=cutoff_time).order_by('mirror__id', 'url') + + if mirror_ids: + url_data = url_data.filter(mirror_id__in=mirror_ids) + urls = urls.filter(mirror_id__in=mirror_ids) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the @@ -66,6 +69,11 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): delays.setdefault(url_id, []).append(delay) if urls: + url_data = dict((item['id'], item) for item in url_data) + for url in urls: + for k, v in url_data.get(url.id, {}).items(): + if k not in ('id', 'mirror_id'): + setattr(url, k, v) last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) -- cgit v1.2.3-54-g00ecf From f0f6f7235a62186c1cae9c79036dde5d8821373d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 19 Nov 2012 08:10:21 -0600 Subject: Fix mirror URL duplication in status view We need to ensure we don't duplicate URLs in the status view, so add a distinct() call back in to the queryset when it was inadvertently dropped in commit a2cfa7edbb. This negates a lot of the performance gains we had, unfortunately, so it looks like a nested subquery might be more efficient. Disappointing the planner can't do this for us. Signed-off-by: Dan McGee --- mirrors/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 07a7138f..a62c7f05 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -48,7 +48,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).order_by('mirror__id', 'url') + logs__check_time__gte=cutoff_time).distinct().order_by( + 'mirror__id', 'url') if mirror_ids: url_data = url_data.filter(mirror_id__in=mirror_ids) -- cgit v1.2.3-54-g00ecf From bec73c7a37c07821f145dbcf11435d4f2b94a149 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 12 Jan 2013 15:56:36 -0600 Subject: Round two of mirror status query improvements This seems to generate much more performant queries at the database level than what we were previously doing, and also doesn't show duplicate rows. Signed-off-by: Dan McGee --- mirrors/utils.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index a62c7f05..1d560021 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -33,9 +33,16 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff - url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( + + valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).annotate( + logs__check_time__gte=cutoff_time).distinct() + + if mirror_ids: + valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) + + url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( + id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), @@ -47,13 +54,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( - mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).distinct().order_by( - 'mirror__id', 'url') - - if mirror_ids: - url_data = url_data.filter(mirror_id__in=mirror_ids) - urls = urls.filter(mirror_id__in=mirror_ids) + id__in=valid_urls).order_by('mirror__id', 'url') # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the -- cgit v1.2.3-54-g00ecf From 6f0ae6746baea657ee6d7c21ac0813a04f825443 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 01:00:11 -0600 Subject: Drop country column from mirror table We now always look for this information at the URL level, not the mirror level. This simplifies quite a bit of code in and around the mirror views. Signed-off-by: Dan McGee --- mirrors/admin.py | 4 +- .../0020_auto__del_field_mirror_country.py | 70 ++++++++++++++++++++++ mirrors/models.py | 7 +-- mirrors/utils.py | 7 +-- mirrors/views.py | 15 ++--- public/views.py | 2 +- templates/mirrors/mirror_details.html | 12 ++-- templates/mirrors/mirrorlist.txt | 2 +- templates/mirrors/mirrorlist_status.txt | 2 +- templates/mirrors/mirrors.html | 4 +- templates/mirrors/status_table.html | 2 +- templates/public/download.html | 2 +- 12 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 mirrors/migrations/0020_auto__del_field_mirror_country.py (limited to 'mirrors/utils.py') diff --git a/mirrors/admin.py b/mirrors/admin.py index 65fff368..eaa38391 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -62,9 +62,9 @@ class Meta: class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm - list_display = ('name', 'tier', 'country', 'active', 'public', + list_display = ('name', 'tier', 'active', 'public', 'isos', 'admin_email', 'alternate_email') - list_filter = ('tier', 'active', 'public', 'country') + list_filter = ('tier', 'active', 'public') search_fields = ('name', 'admin_email', 'alternate_email') inlines = [ MirrorUrlInlineAdmin, diff --git a/mirrors/migrations/0020_auto__del_field_mirror_country.py b/mirrors/migrations/0020_auto__del_field_mirror_country.py new file mode 100644 index 00000000..c2220a50 --- /dev/null +++ b/mirrors/migrations/0020_auto__del_field_mirror_country.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.delete_column('mirrors_mirror', 'country') + + + def backwards(self, orm): + db.add_column('mirrors_mirror', 'country', + self.gf('django_countries.fields.CountryField')(blank=True, default='', max_length=2, db_index=True), + keep_default=False) + + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 0179d5bf..ca421d13 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -17,7 +17,6 @@ class Mirror(models.Model): name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) - country = CountryField(blank=True, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) alternate_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) @@ -28,7 +27,7 @@ class Mirror(models.Model): notes = models.TextField(blank=True) class Meta: - ordering = ('country', 'name') + ordering = ('name',) def __unicode__(self): return self.name @@ -75,10 +74,6 @@ def address_families(self): def hostname(self): return urlparse(self.url).hostname - @property - def real_country(self): - return self.country or self.mirror.country - def clean(self): try: # Auto-map the protocol field by looking at the URL diff --git a/mirrors/utils.py b/mirrors/utils.py index 1d560021..3ab176b3 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -115,7 +115,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'url__mirror__tier', 'error').annotate( + 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') @@ -124,8 +124,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): errors = list(errors) for err in errors: - ctry_code = err['url__country'] or err['url__mirror__country'] - err['country'] = Country(ctry_code) + err['country'] = Country(err['url__country']) return errors @@ -152,7 +151,7 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): mirror_urls = MirrorUrl.objects.filter( mirror__public=True, mirror__active=True, protocol__default=True) # look first for a country-agnostic URL, then fall back to any HTTP URL - filtered_urls = mirror_urls.filter(mirror__country='')[:1] + filtered_urls = mirror_urls.filter(country='')[:1] if not filtered_urls: filtered_urls = mirror_urls[:1] if not filtered_urls: diff --git a/mirrors/views.py b/mirrors/views.py index d3867802..545e3557 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -42,9 +42,6 @@ def __init__(self, *args, **kwargs): def get_countries(self): country_codes = set() - country_codes.update(Mirror.objects.filter(active=True).exclude( - country='').values_list( - 'country', flat=True).order_by().distinct()) country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) @@ -81,7 +78,7 @@ def generate_mirrorlist(request): def default_protocol_filter(original_urls): - key_func = attrgetter('real_country') + key_func = attrgetter('country') sorted_urls = sorted(original_urls, key=key_func) urls = [] for _, group in groupby(sorted_urls, key=key_func): @@ -119,8 +116,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, protocol__in=protocols, mirror__public=True, mirror__active=True) if countries and 'all' not in countries: - qset = qset.filter(Q(country__in=countries) | - Q(mirror__country__in=countries)) + qset = qset.filter(country__in=countries) ip_version = Q() if ipv4_supported: @@ -135,7 +131,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = qset if not use_status: - sort_key = attrgetter('real_country.name', 'mirror.name', 'url') + sort_key = attrgetter('country.name', 'mirror.name', 'url') urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: @@ -158,7 +154,7 @@ def find_mirrors_simple(request, protocol): def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') protos = MirrorUrl.objects.values_list( 'mirror_id', 'protocol__protocol').order_by( 'mirror__id', 'protocol__protocol').distinct() @@ -254,8 +250,7 @@ def default(self, obj): return list(obj) if isinstance(obj, MirrorUrl): data = {attr: getattr(obj, attr) for attr in self.url_attributes} - # get any override on the country attribute first - country = obj.real_country + country = obj.country data['country'] = unicode(country.name) data['country_code'] = country.code return data diff --git a/public/views.py b/public/views.py index 65b0c31f..e15f5b89 100644 --- a/public/views.py +++ b/public/views.py @@ -85,7 +85,7 @@ def download(request): mirror_urls = MirrorUrl.objects.select_related('mirror').filter( protocol__default=True, mirror__public=True, mirror__active=True, mirror__isos=True) - sort_by = attrgetter('real_country.name', 'mirror.name') + sort_by = attrgetter('country.name', 'mirror.name') mirror_urls = sorted(mirror_urls, key=sort_by) context = { 'release': release, diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index 132557cd..a56123ff 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -18,10 +18,6 @@

Mirror Details: {{ mirror.name }}

Tier: {{ mirror.get_tier_display }} - - Country: - {% if mirror.country %} {% endif %}{{ mirror.country.name|default:'Worldwide' }} - Has ISOs: {{ mirror.isos|yesno|capfirst }} @@ -77,6 +73,8 @@

Available URLs

Mirror URL + Protocol + Country IPv4 IPv6 Last Sync @@ -91,6 +89,8 @@

Available URLs

{% for m_url in urls %} {% if m_url.protocol.is_download %}{{ m_url.url }}{% else %}{{ m_url.url }}{% endif %} + {{ m_url.protocol }} + {% if m_url.country %} {% endif %}{{ m_url.country.name }} {{ m_url.has_ipv4|yesno|capfirst }} {{ m_url.has_ipv6|yesno|capfirst }} {{ m_url.last_sync|date:'Y-m-d H:i'|default:'unknown' }} @@ -115,8 +115,8 @@

Mirror Status Chart

{% endblock %} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html index 1961d222..c7394de6 100644 --- a/templates/mirrors/status_table.html +++ b/templates/mirrors/status_table.html @@ -17,7 +17,7 @@ {% spaceless %} {{ m_url.url }} {{ m_url.protocol }} - {% if m_url.real_country %} {% endif %}{{ m_url.real_country.name }} + {% if m_url.country %} {% endif %}{{ m_url.country.name }} {{ m_url.completion_pct|percentage:1 }} {{ m_url.delay|duration|default:'unknown' }} {{ m_url.duration_avg|floatformat:2 }} diff --git a/templates/public/download.html b/templates/public/download.html index 3005ffb3..0c96fcef 100644 --- a/templates/public/download.html +++ b/templates/public/download.html @@ -81,7 +81,7 @@

Checksums

{% cache 600 download-mirrors %}
- {% regroup mirror_urls by real_country as grouped_urls %} + {% regroup mirror_urls by country as grouped_urls %} {% for country in grouped_urls %} {% if country.grouper %}
{{ country.grouper.name }}
{% else %}
Worldwide
{% endif %} -- cgit v1.2.3-54-g00ecf