From 76516cae45f3d1080065608bdb8f2d086322012f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 19:45:57 -0500 Subject: Don't limit protocols returned by mirror status function If results weren't available for certain URLs, they won't show up anyway in this list, and if we start to check rsync URLs, then we want their values to come back in this status list. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 32fa3587..54de567e 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -32,11 +32,9 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=default_cutoff): cutoff_time = utc_now() - cutoff - protocols = list(MirrorProtocol.objects.filter(is_download=True)) # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, - protocol__in=protocols, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), -- cgit v1.2.3-54-g00ecf From ae1c526ffbe908322f0dd8d8805360b81ab22b0f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 20:35:50 -0500 Subject: Add ability to restrict status report to single tier This should make it easier to catch errors in our Tier 1 mirrors. Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/utils.py | 2 +- mirrors/views.py | 19 ++++++++++++++++--- templates/mirrors/status.html | 4 ++-- 4 files changed, 20 insertions(+), 6 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/urls.py b/mirrors/urls.py index f002e9d6..bb4eb969 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -4,6 +4,7 @@ (r'^$', 'mirrors', {}, 'mirror-list'), (r'^status/$', 'status', {}, 'mirror-status'), (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), + (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), ) diff --git a/mirrors/utils.py b/mirrors/utils.py index 54de567e..2014411d 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -88,7 +88,7 @@ def get_mirror_errors(cutoff=default_cutoff): is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'error').annotate( + 'url__mirror__country', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') errors = list(errors) diff --git a/mirrors/views.py b/mirrors/views.py index b0be6238..8f092be7 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,7 +13,7 @@ from django.views.generic.simple import direct_to_template from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -184,7 +184,11 @@ def mirror_details(request, name): {'mirror': mirror, 'urls': all_urls}) -def status(request): +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in TIER_CHOICES]: + raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() @@ -192,17 +196,26 @@ def status(request): good_urls = [] bad_urls = [] for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue # split them into good and bad lists based on delay if not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url__mirror__tier'] == tier] + context = status_info.copy() context.update({ 'good_urls': sorted(good_urls, key=attrgetter('score')), 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': get_mirror_errors(), + 'error_logs': error_logs, + 'tier': tier, }) return direct_to_template(request, 'mirrors/status.html', context) diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 2c350f56..472e9501 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -2,11 +2,11 @@ {% load static from staticfiles %} {% load mirror_status %} -{% block title %}Arch Linux - Mirror Status{% endblock %} +{% block title %}Arch Linux - Mirror Status{% if tier != None %} - Tier {{ tier }}{% endif %}{% endblock %} {% block content %}
-

Mirror Status

+

Mirror Status{% if tier != None %} - Tier {{ tier }}{% endif %}

This page reports the status of all known, public, and active Arch Linux mirrors. All data on this page reflects the status of the mirrors within the last {{ cutoff|hours }}. All listed times are UTC. The check script runs -- cgit v1.2.3-54-g00ecf From 0f3c894e7a0f573fa0198459150f387c3a7f23ae Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 20:38:01 -0500 Subject: Don't include StdDev on sqlite3 mirror status query Because this function isn't shipped by default, it makes more sense to just omit it completely from the query we do to build the tables on this page when in development. Substitute 0.0 for the value so the rest of the calculations and display work as expected. Signed-off-by: Dan McGee --- mirrors/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 2014411d..9aa8e0f5 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -3,7 +3,7 @@ from django.db.models import Avg, Count, Max, Min, StdDev from django_countries.fields import Country -from main.utils import cache_function, utc_now +from main.utils import cache_function, utc_now, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl @@ -40,8 +40,11 @@ def get_mirror_statuses(cutoff=default_cutoff): success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration'), - duration_stddev=StdDev('logs__duration')) + duration_avg=Avg('logs__duration')) + + vendor = database_vendor(MirrorUrl) + if vendor != 'sqlite': + urls.annotate(duration_stddev=StdDev('logs__duration')) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the @@ -70,6 +73,9 @@ def get_mirror_statuses(cutoff=default_cutoff): check_frequency = None for url in urls: + # fake the standard deviation for local testing setups + if vendor == 'sqlite': + setattr(url, 'duration_stddev', 0.0) annotate_url(url, delays) return { -- cgit v1.2.3-54-g00ecf From 9c7350650e66b5eb6228778e335a160be5ea7f98 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 21:53:35 -0500 Subject: Correctly reassign queryset with added annotation in mirror status This was a dumb oversight on my part in commit 0f3c894e7a0. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 9aa8e0f5..f2c98ee0 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -44,7 +44,7 @@ def get_mirror_statuses(cutoff=default_cutoff): vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': - urls.annotate(duration_stddev=StdDev('logs__duration')) + urls = urls.annotate(duration_stddev=StdDev('logs__duration')) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the -- cgit v1.2.3-54-g00ecf From c0bf9e20660cfae7ea8994472555bba23398b598 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:19:48 -0500 Subject: Remove custom utc_now() function, use django.utils.timezone.now() This was around from the time when we handled timezones sanely and Django did not; now that we are on 1.4 we no longer need our own code to handle this. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 9 +++++---- devel/models.py | 5 +++-- devel/views.py | 12 ++++++------ main/models.py | 5 +++-- main/utils.py | 9 ++------- mirrors/management/commands/mirrorcheck.py | 7 ++++--- mirrors/utils.py | 17 +++++++++-------- news/models.py | 11 +++++------ packages/management/commands/signoff_report.py | 8 ++++---- packages/views/flag.py | 8 ++++---- packages/views/signoff.py | 4 ++-- releng/management/commands/syncisos.py | 5 ++--- 12 files changed, 49 insertions(+), 51 deletions(-) (limited to 'mirrors/utils.py') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 2d9b68b2..e69691db 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -28,10 +28,11 @@ from django.core.management.base import BaseCommand, CommandError from django.db import connections, router, transaction from django.db.utils import IntegrityError +from django.utils.timezone import now from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update @@ -318,7 +319,7 @@ def populate_files(dbpkg, repopkg, force=False): filename=filename) pkg_files.append(pkgfile) batched_bulk_create(PackageFile, pkg_files) - dbpkg.files_last_update = utc_now() + dbpkg.files_last_update = now() dbpkg.save() @@ -388,7 +389,7 @@ def db_update(archname, reponame, pkgs, force=False): dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository) try: with transaction.commit_on_success(): - populate_pkg(dbpkg, pkg, timestamp=utc_now()) + populate_pkg(dbpkg, pkg, timestamp=now()) Update.objects.log_update(None, dbpkg) except IntegrityError: logger.warning("Could not add package %s; " @@ -417,7 +418,7 @@ def db_update(archname, reponame, pkgs, force=False): if not force and pkg_same_version(pkg, dbpkg): continue elif not force: - timestamp = utc_now() + timestamp = now() # The odd select_for_update song and dance here are to ensure # simultaneous updates don't happen on a package, causing diff --git a/devel/models.py b/devel/models.py index fd5df00a..9b6f07a7 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,10 +4,11 @@ from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User +from django.utils.timezone import now from django_countries import CountryField from .fields import PGPKeyField -from main.utils import make_choice, utc_now +from main.utils import make_choice class UserProfile(models.Model): @@ -105,7 +106,7 @@ def set_last_modified(sender, **kwargs): signal handler.''' obj = kwargs['instance'] if hasattr(obj, 'last_modified'): - obj.last_modified = utc_now() + obj.last_modified = now() # connect signals needed to keep cache in line with reality diff --git a/devel/views.py b/devel/views.py index 78ed26f2..143b12bf 100644 --- a/devel/views.py +++ b/devel/views.py @@ -24,11 +24,11 @@ from django.views.generic.simple import direct_to_template from django.utils.encoding import force_unicode from django.utils.http import http_date +from django.utils.timezone import now from .models import UserProfile from main.models import Package, PackageFile, TodolistPkg from main.models import Arch, Repo -from main.utils import utc_now from news.models import News from packages.models import PackageRelation, Signoff, Depend from packages.utils import get_signoff_groups @@ -122,15 +122,15 @@ def clock(request): else: dev.last_action = None - now = utc_now() + current_time = now() page_dict = { 'developers': devs, - 'utc_now': now, + 'utc_now': current_time, } response = direct_to_template(request, 'devel/clock.html', page_dict) if not response.has_header('Expires'): - expire_time = now.replace(second=0, microsecond=0) + expire_time = current_time.replace(second=0, microsecond=0) expire_time += timedelta(minutes=1) expire_time = time.mktime(expire_time.timetuple()) response['Expires'] = http_date(expire_time) @@ -198,12 +198,12 @@ def report(request, report_name, username=None): if report_name == 'old': title = 'Packages last built more than one year ago' - cutoff = utc_now() - timedelta(days=365) + cutoff = now() - timedelta(days=365) packages = packages.filter( build_date__lt=cutoff).order_by('build_date') elif report_name == 'long-out-of-date': title = 'Packages marked out-of-date more than 90 days ago' - cutoff = utc_now() - timedelta(days=90) + cutoff = now() - timedelta(days=90) packages = packages.filter( flag_date__lt=cutoff).order_by('flag_date') elif report_name == 'big': diff --git a/main/models.py b/main/models.py index 04d8da8f..6c9dfe4d 100644 --- a/main/models.py +++ b/main/models.py @@ -6,9 +6,10 @@ from django.db import models from django.contrib.auth.models import User from django.contrib.sites.models import Site +from django.utils.timezone import now from .fields import PositiveBigIntegerField -from .utils import cache_function, set_created_field, utc_now +from .utils import cache_function, set_created_field class TodolistManager(models.Manager): @@ -385,7 +386,7 @@ class Meta: def set_todolist_fields(sender, **kwargs): todolist = kwargs['instance'] if not todolist.date_added: - todolist.date_added = utc_now() + todolist.date_added = now() # connect signals needed to keep cache in line with reality from main.utils import refresh_latest diff --git a/main/utils.py b/main/utils.py index 879abfb9..0b6849a4 100644 --- a/main/utils.py +++ b/main/utils.py @@ -5,10 +5,10 @@ from datetime import datetime import hashlib -from pytz import utc from django.core.cache import cache from django.db import connections, router +from django.utils.timezone import now CACHE_TIMEOUT = 1800 @@ -94,17 +94,12 @@ def retrieve_latest(sender, latest_by=None): return None -def utc_now(): - '''Returns a timezone-aware UTC date representing now.''' - return datetime.utcnow().replace(tzinfo=utc) - - def set_created_field(sender, **kwargs): '''This will set the 'created' field on any object to the current UTC time if it is unset. For use as a pre_save signal handler.''' obj = kwargs['instance'] if hasattr(obj, 'created') and not obj.created: - obj.created = utc_now() + obj.created = now() def database_vendor(model, mode='read'): diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7a133cbf..e09ea680 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -29,8 +29,9 @@ from django.core.management.base import NoArgsCommand from django.db import transaction +from django.utils.timezone import now -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -83,7 +84,7 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) try: @@ -136,7 +137,7 @@ def check_mirror_url(mirror_url, timeout): def check_rsync_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) tempdir = tempfile.mkdtemp() lastsync_path = os.path.join(tempdir, 'lastsync') diff --git a/mirrors/utils.py b/mirrors/utils.py index f2c98ee0..bf030d39 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,14 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.timezone import now from django_countries.fields import Country -from main.utils import cache_function, utc_now, database_vendor +from main.utils import cache_function, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl -default_cutoff = timedelta(hours=24) +DEFAULT_CUTOFF = timedelta(hours=24) def annotate_url(url, delays): '''Given a MirrorURL object, add a few more attributes to it regarding @@ -30,8 +31,8 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, @@ -88,8 +89,8 @@ def get_mirror_statuses(cutoff=default_cutoff): @cache_function(117) -def get_mirror_errors(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_errors(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( @@ -105,11 +106,11 @@ def get_mirror_errors(cutoff=default_cutoff): @cache_function(295) -def get_mirror_url_for_download(cutoff=default_cutoff): +def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): '''Find a good mirror URL to use for package downloads. If we have mirror status data available, it is used to determine a good choice by looking at the last batch of status rows.''' - cutoff_time = utc_now() - cutoff + cutoff_time = now() - cutoff status_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) diff --git a/news/models.py b/news/models.py index 95026e1d..2efea579 100644 --- a/news/models.py +++ b/news/models.py @@ -1,8 +1,7 @@ from django.db import models from django.contrib.auth.models import User from django.contrib.sites.models import Site - -from main.utils import utc_now +from django.utils.timezone import now class News(models.Model): @@ -29,13 +28,13 @@ class Meta: def set_news_fields(sender, **kwargs): news = kwargs['instance'] - now = utc_now() - news.last_modified = now + current_time = now() + news.last_modified = current_time if not news.postdate: - news.postdate = now + news.postdate = current_time # http://diveintomark.org/archives/2004/05/28/howto-atom-id news.guid = 'tag:%s,%s:%s' % (Site.objects.get_current(), - now.strftime('%Y-%m-%d'), news.get_absolute_url()) + current_time.strftime('%Y-%m-%d'), news.get_absolute_url()) # connect signals needed to keep cache in line with reality from main.utils import refresh_latest diff --git a/packages/management/commands/signoff_report.py b/packages/management/commands/signoff_report.py index ddf930db..72fcbe1e 100644 --- a/packages/management/commands/signoff_report.py +++ b/packages/management/commands/signoff_report.py @@ -15,6 +15,7 @@ from django.contrib.sites.models import Site from django.db.models import Count from django.template import loader, Context +from django.utils.timezone import now from collections import namedtuple from datetime import timedelta @@ -23,7 +24,6 @@ import sys from main.models import Repo -from main.utils import utc_now from packages.models import Signoff from packages.utils import get_signoff_groups @@ -66,9 +66,9 @@ def generate_report(email, repo_name): new_hours = 24 old_days = 14 - now = utc_now() - new_cutoff = now - timedelta(hours=new_hours) - old_cutoff = now - timedelta(days=old_days) + current_time = now() + new_cutoff = current_time - timedelta(hours=new_hours) + old_cutoff = current_time - timedelta(days=old_days) if len(signoff_groups) == 0: # no need to send an email at all diff --git a/packages/views/flag.py b/packages/views/flag.py index 7fa2d508..f3db93b3 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -5,12 +5,12 @@ from django.db import transaction from django.shortcuts import get_object_or_404, redirect from django.template import loader, Context +from django.utils.timezone import now from django.views.generic.simple import direct_to_template from django.views.decorators.cache import cache_page, never_cache from ..models import FlagRequest from main.models import Package -from main.utils import utc_now class FlagForm(forms.Form): @@ -76,10 +76,10 @@ def flag(request, name, repo, arch): @transaction.commit_on_success def perform_updates(): - now = utc_now() - pkgs.update(flag_date=now) + current_time = now() + pkgs.update(flag_date=current_time) # store our flag request - flag_request = FlagRequest(created=now, + flag_request = FlagRequest(created=current_time, user_email=email, message=message, ip_address=ip_addr, pkgbase=pkg.pkgbase, version=version, repo=pkg.repo, diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 61d949fc..7aa39106 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -8,11 +8,11 @@ from django.db import transaction from django.http import HttpResponse, Http404 from django.shortcuts import get_list_or_404, redirect, render +from django.utils.timezone import now from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template from main.models import Package, Arch, Repo -from main.utils import utc_now from ..models import SignoffSpecification, Signoff from ..utils import (get_signoff_groups, approved_by_signoffs, PackageSignoffGroup) @@ -45,7 +45,7 @@ def signoff_package(request, name, repo, arch, revoke=False): package, request.user, False) except Signoff.DoesNotExist: raise Http404 - signoff.revoked = utc_now() + signoff.revoked = now() signoff.save() created = False else: diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index 62f005ff..223c771b 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -4,8 +4,8 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError +from django.utils.timezone import now -from main.utils import utc_now from releng.models import Iso @@ -54,9 +54,8 @@ def handle(self, *args, **options): existing.active = True existing.removed = None existing.save() - now = utc_now() # and then mark all other names as no longer active Iso.objects.filter(active=True).exclude(name__in=active_isos).update( - active=False, removed=now) + active=False, removed=now()) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From f0b7e73de61c03a5018ed352605e6329611448d2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 10 Oct 2012 20:17:55 -0500 Subject: Make mirror log time query a bit more efficient We don't need the full mirror log objects; we just need a very small subset of values from them here to do the required math and object building. Signed-off-by: Dan McGee --- mirrors/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index bf030d39..0a32b766 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -50,12 +50,14 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the # results here. - times = MirrorLog.objects.filter(is_success=True, last_sync__isnull=False, + times = MirrorLog.objects.values_list( + 'url_id', 'check_time', 'last_sync').filter( + is_success=True, last_sync__isnull=False, check_time__gte=cutoff_time) delays = {} - for log in times: - delay = log.check_time - log.last_sync - delays.setdefault(log.url_id, []).append(delay) + for url_id, check_time, last_sync in times: + delay = check_time - last_sync + delays.setdefault(url_id, []).append(delay) if urls: last_check = max([u.last_check for u in urls]) -- cgit v1.2.3-54-g00ecf From 86102c6e645451c03e3e576060eba7f93350bf6b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 14:19:23 -0600 Subject: Allow filtering retrieved mirror statuses by mirror_id When we don't need them all, no need to fetch them all. Let the database do the work for us, hopefully. Signed-off-by: Dan McGee --- mirrors/utils.py | 19 +++++++++++++++---- mirrors/views.py | 8 ++++---- 2 files changed, 19 insertions(+), 8 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 0a32b766..ba027c99 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -31,7 +31,7 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( @@ -43,6 +43,9 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): last_check=Max('logs__check_time'), duration_avg=Avg('logs__duration')) + if mirror_ids: + urls = urls.filter(mirror_id__in=mirror_ids) + vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': urls = urls.annotate(duration_stddev=StdDev('logs__duration')) @@ -54,6 +57,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): 'url_id', 'check_time', 'last_sync').filter( is_success=True, last_sync__isnull=False, check_time__gte=cutoff_time) + if mirror_ids: + times = times.filter(url__mirror_id__in=mirror_ids) delays = {} for url_id, check_time, last_sync in times: delay = check_time - last_sync @@ -62,8 +67,10 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): if urls: last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) - check_info = MirrorLog.objects.filter( - check_time__gte=cutoff_time).aggregate( + check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) + if mirror_ids: + check_info = check_info.filter(url__mirror_id__in=mirror_ids) + check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: check_frequency = (check_info['mx'] - check_info['mn']) \ @@ -91,7 +98,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): @cache_function(117) -def get_mirror_errors(cutoff=DEFAULT_CUTOFF): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, @@ -100,6 +107,10 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF): 'url__mirror__country', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') + + if mirror_ids: + urls = urls.filter(mirror_id__in=mirror_ids) + errors = list(errors) for err in errors: ctry_code = err['url__country'] or err['url__mirror__country'] diff --git a/mirrors/views.py b/mirrors/views.py index cbd86611..4b9721dc 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -14,7 +14,7 @@ from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog -from .utils import get_mirror_statuses, get_mirror_errors +from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF COUNTRY_LOOKUP = dict(COUNTRIES) @@ -171,7 +171,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) checked_urls = [url for url in status_info['urls'] \ if url.mirror_id == mirror.id] all_urls = mirror.urls.select_related('protocol') @@ -186,7 +186,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 # include only URLs for this particular mirror @@ -266,7 +266,7 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): def default(self, obj): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - cutoff = now() - timedelta(hours=24) + cutoff = now() - DEFAULT_CUTOFF data['logs'] = obj.logs.filter(check_time__gte=cutoff) return data if isinstance(obj, MirrorLog): -- cgit v1.2.3-54-g00ecf From e26d5722289bd2e972633891d8dac09296b0cbc4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Nov 2012 14:55:37 -0600 Subject: Mirror graph tweaking after usage with real data * Clamp y-axis minimum to 0. * Don't plot `is_success == false` values. * Ensure URLs are sorted predictably. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 5 ++++- mirrors/utils.py | 3 ++- mirrors/views.py | 2 -- 3 files changed, 6 insertions(+), 4 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 1c352a9f..decc8fb8 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -25,7 +25,7 @@ function mirror_status(chart_id, data_url) { d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) ]).nice(d3.time.hour); y.domain([ - d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.duration; }); }), + 0, d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) ]).nice(); @@ -111,6 +111,9 @@ function mirror_status(chart_id, data_url) { return { url: url.url, logs: jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } return { duration: log.duration, check_time: new Date(log.check_time) diff --git a/mirrors/utils.py b/mirrors/utils.py index ba027c99..85e4ee93 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -41,7 +41,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')) + duration_avg=Avg('logs__duration')).order_by( + 'mirror', 'url') if mirror_ids: urls = urls.filter(mirror_id__in=mirror_ids) diff --git a/mirrors/views.py b/mirrors/views.py index be01e919..5e374b4d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -189,8 +189,6 @@ def mirror_details_json(request, name): status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 - # include only URLs for this particular mirror - data['urls'] = [url for url in data['urls'] if url.mirror_id == mirror.id] to_json = json.dumps(data, ensure_ascii=False, cls=ExtendedMirrorStatusJSONEncoder) response = HttpResponse(to_json, mimetype='application/json') -- cgit v1.2.3-54-g00ecf From a2cfa7edbbed8edb1ad4d3391c6edb055c13de1b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 15:39:56 -0600 Subject: Optimize mirror status data fetching Now that we have as many mirror URLs as we do, we can do a better job fetching and aggregating this data. The prior method resulted in a rather unwieldy query being pushed down to the database with a horrendously long GROUP BY clause. Instead of trying to group by everything at once so we can retrieve mirror URL info at the same time, separate the two queries- one for getting URL performance data, one for the qualitative data. The impetus behind fixing this is the PostgreSQL slow query log in production; this currently shows up the most of any queries we run in the system. Signed-off-by: Dan McGee --- mirrors/utils.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 85e4ee93..07a7138f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -33,23 +33,26 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff - # I swear, this actually has decent performance... - urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')).order_by( - 'mirror', 'url') - - if mirror_ids: - urls = urls.filter(mirror_id__in=mirror_ids) + duration_avg=Avg('logs__duration')) vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': - urls = urls.annotate(duration_stddev=StdDev('logs__duration')) + url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) + + urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + mirror__active=True, mirror__public=True, + logs__check_time__gte=cutoff_time).order_by('mirror__id', 'url') + + if mirror_ids: + url_data = url_data.filter(mirror_id__in=mirror_ids) + urls = urls.filter(mirror_id__in=mirror_ids) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the @@ -66,6 +69,11 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): delays.setdefault(url_id, []).append(delay) if urls: + url_data = dict((item['id'], item) for item in url_data) + for url in urls: + for k, v in url_data.get(url.id, {}).items(): + if k not in ('id', 'mirror_id'): + setattr(url, k, v) last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) -- cgit v1.2.3-54-g00ecf From f0f6f7235a62186c1cae9c79036dde5d8821373d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 19 Nov 2012 08:10:21 -0600 Subject: Fix mirror URL duplication in status view We need to ensure we don't duplicate URLs in the status view, so add a distinct() call back in to the queryset when it was inadvertently dropped in commit a2cfa7edbb. This negates a lot of the performance gains we had, unfortunately, so it looks like a nested subquery might be more efficient. Disappointing the planner can't do this for us. Signed-off-by: Dan McGee --- mirrors/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 07a7138f..a62c7f05 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -48,7 +48,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).order_by('mirror__id', 'url') + logs__check_time__gte=cutoff_time).distinct().order_by( + 'mirror__id', 'url') if mirror_ids: url_data = url_data.filter(mirror_id__in=mirror_ids) -- cgit v1.2.3-54-g00ecf From bec73c7a37c07821f145dbcf11435d4f2b94a149 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 12 Jan 2013 15:56:36 -0600 Subject: Round two of mirror status query improvements This seems to generate much more performant queries at the database level than what we were previously doing, and also doesn't show duplicate rows. Signed-off-by: Dan McGee --- mirrors/utils.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index a62c7f05..1d560021 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -33,9 +33,16 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff - url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( + + valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).annotate( + logs__check_time__gte=cutoff_time).distinct() + + if mirror_ids: + valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) + + url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( + id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), @@ -47,13 +54,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( - mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).distinct().order_by( - 'mirror__id', 'url') - - if mirror_ids: - url_data = url_data.filter(mirror_id__in=mirror_ids) - urls = urls.filter(mirror_id__in=mirror_ids) + id__in=valid_urls).order_by('mirror__id', 'url') # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the -- cgit v1.2.3-54-g00ecf From 6f0ae6746baea657ee6d7c21ac0813a04f825443 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 01:00:11 -0600 Subject: Drop country column from mirror table We now always look for this information at the URL level, not the mirror level. This simplifies quite a bit of code in and around the mirror views. Signed-off-by: Dan McGee --- mirrors/admin.py | 4 +- .../0020_auto__del_field_mirror_country.py | 70 ++++++++++++++++++++++ mirrors/models.py | 7 +-- mirrors/utils.py | 7 +-- mirrors/views.py | 15 ++--- public/views.py | 2 +- templates/mirrors/mirror_details.html | 12 ++-- templates/mirrors/mirrorlist.txt | 2 +- templates/mirrors/mirrorlist_status.txt | 2 +- templates/mirrors/mirrors.html | 4 +- templates/mirrors/status_table.html | 2 +- templates/public/download.html | 2 +- 12 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 mirrors/migrations/0020_auto__del_field_mirror_country.py (limited to 'mirrors/utils.py') diff --git a/mirrors/admin.py b/mirrors/admin.py index 65fff368..eaa38391 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -62,9 +62,9 @@ class Meta: class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm - list_display = ('name', 'tier', 'country', 'active', 'public', + list_display = ('name', 'tier', 'active', 'public', 'isos', 'admin_email', 'alternate_email') - list_filter = ('tier', 'active', 'public', 'country') + list_filter = ('tier', 'active', 'public') search_fields = ('name', 'admin_email', 'alternate_email') inlines = [ MirrorUrlInlineAdmin, diff --git a/mirrors/migrations/0020_auto__del_field_mirror_country.py b/mirrors/migrations/0020_auto__del_field_mirror_country.py new file mode 100644 index 00000000..c2220a50 --- /dev/null +++ b/mirrors/migrations/0020_auto__del_field_mirror_country.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.delete_column('mirrors_mirror', 'country') + + + def backwards(self, orm): + db.add_column('mirrors_mirror', 'country', + self.gf('django_countries.fields.CountryField')(blank=True, default='', max_length=2, db_index=True), + keep_default=False) + + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 0179d5bf..ca421d13 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -17,7 +17,6 @@ class Mirror(models.Model): name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) - country = CountryField(blank=True, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) alternate_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) @@ -28,7 +27,7 @@ class Mirror(models.Model): notes = models.TextField(blank=True) class Meta: - ordering = ('country', 'name') + ordering = ('name',) def __unicode__(self): return self.name @@ -75,10 +74,6 @@ def address_families(self): def hostname(self): return urlparse(self.url).hostname - @property - def real_country(self): - return self.country or self.mirror.country - def clean(self): try: # Auto-map the protocol field by looking at the URL diff --git a/mirrors/utils.py b/mirrors/utils.py index 1d560021..3ab176b3 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -115,7 +115,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'url__mirror__tier', 'error').annotate( + 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') @@ -124,8 +124,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): errors = list(errors) for err in errors: - ctry_code = err['url__country'] or err['url__mirror__country'] - err['country'] = Country(ctry_code) + err['country'] = Country(err['url__country']) return errors @@ -152,7 +151,7 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): mirror_urls = MirrorUrl.objects.filter( mirror__public=True, mirror__active=True, protocol__default=True) # look first for a country-agnostic URL, then fall back to any HTTP URL - filtered_urls = mirror_urls.filter(mirror__country='')[:1] + filtered_urls = mirror_urls.filter(country='')[:1] if not filtered_urls: filtered_urls = mirror_urls[:1] if not filtered_urls: diff --git a/mirrors/views.py b/mirrors/views.py index d3867802..545e3557 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -42,9 +42,6 @@ def __init__(self, *args, **kwargs): def get_countries(self): country_codes = set() - country_codes.update(Mirror.objects.filter(active=True).exclude( - country='').values_list( - 'country', flat=True).order_by().distinct()) country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) @@ -81,7 +78,7 @@ def generate_mirrorlist(request): def default_protocol_filter(original_urls): - key_func = attrgetter('real_country') + key_func = attrgetter('country') sorted_urls = sorted(original_urls, key=key_func) urls = [] for _, group in groupby(sorted_urls, key=key_func): @@ -119,8 +116,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, protocol__in=protocols, mirror__public=True, mirror__active=True) if countries and 'all' not in countries: - qset = qset.filter(Q(country__in=countries) | - Q(mirror__country__in=countries)) + qset = qset.filter(country__in=countries) ip_version = Q() if ipv4_supported: @@ -135,7 +131,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = qset if not use_status: - sort_key = attrgetter('real_country.name', 'mirror.name', 'url') + sort_key = attrgetter('country.name', 'mirror.name', 'url') urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: @@ -158,7 +154,7 @@ def find_mirrors_simple(request, protocol): def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') protos = MirrorUrl.objects.values_list( 'mirror_id', 'protocol__protocol').order_by( 'mirror__id', 'protocol__protocol').distinct() @@ -254,8 +250,7 @@ def default(self, obj): return list(obj) if isinstance(obj, MirrorUrl): data = {attr: getattr(obj, attr) for attr in self.url_attributes} - # get any override on the country attribute first - country = obj.real_country + country = obj.country data['country'] = unicode(country.name) data['country_code'] = country.code return data diff --git a/public/views.py b/public/views.py index 65b0c31f..e15f5b89 100644 --- a/public/views.py +++ b/public/views.py @@ -85,7 +85,7 @@ def download(request): mirror_urls = MirrorUrl.objects.select_related('mirror').filter( protocol__default=True, mirror__public=True, mirror__active=True, mirror__isos=True) - sort_by = attrgetter('real_country.name', 'mirror.name') + sort_by = attrgetter('country.name', 'mirror.name') mirror_urls = sorted(mirror_urls, key=sort_by) context = { 'release': release, diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index 132557cd..a56123ff 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -18,10 +18,6 @@

Mirror Details: {{ mirror.name }}

Tier: {{ mirror.get_tier_display }} - - Country: - {% if mirror.country %} {% endif %}{{ mirror.country.name|default:'Worldwide' }} - Has ISOs: {{ mirror.isos|yesno|capfirst }} @@ -77,6 +73,8 @@

Available URLs

Mirror URL + Protocol + Country IPv4 IPv6 Last Sync @@ -91,6 +89,8 @@

Available URLs

{% for m_url in urls %} {% if m_url.protocol.is_download %}{{ m_url.url }}{% else %}{{ m_url.url }}{% endif %} + {{ m_url.protocol }} + {% if m_url.country %} {% endif %}{{ m_url.country.name }} {{ m_url.has_ipv4|yesno|capfirst }} {{ m_url.has_ipv6|yesno|capfirst }} {{ m_url.last_sync|date:'Y-m-d H:i'|default:'unknown' }} @@ -115,8 +115,8 @@

Mirror Status Chart

{% endblock %} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html index 1961d222..c7394de6 100644 --- a/templates/mirrors/status_table.html +++ b/templates/mirrors/status_table.html @@ -17,7 +17,7 @@ {% spaceless %} {{ m_url.url }} {{ m_url.protocol }} - {% if m_url.real_country %} {% endif %}{{ m_url.real_country.name }} + {% if m_url.country %} {% endif %}{{ m_url.country.name }} {{ m_url.completion_pct|percentage:1 }} {{ m_url.delay|duration|default:'unknown' }} {{ m_url.duration_avg|floatformat:2 }} diff --git a/templates/public/download.html b/templates/public/download.html index 3005ffb3..0c96fcef 100644 --- a/templates/public/download.html +++ b/templates/public/download.html @@ -81,7 +81,7 @@

Checksums

{% cache 600 download-mirrors %}
- {% regroup mirror_urls by real_country as grouped_urls %} + {% regroup mirror_urls by country as grouped_urls %} {% for country in grouped_urls %} {% if country.grouper %}
{{ country.grouper.name }}
{% else %}
Worldwide
{% endif %} -- cgit v1.2.3-54-g00ecf From 2c24ee9100a9e60fec16055d6496caeda3a1d8e2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 11:33:17 -0500 Subject: Calculate average URL delay in the database Rather than doing this in the Python code and needing 12,000+ rows returned from the database, we can do it in the database and get fewer than 300 rows back. If I recall correctly, the reason this was not done originally was due to our usage of MySQL and some really bad date math/overflow stuff it did when the interval between last_sync and check_time were greater than about a week. Luckily, we have switched to using a more sane database. Signed-off-by: Dan McGee --- mirrors/utils.py | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 3ab176b3..2721e20e 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,5 +1,6 @@ from datetime import timedelta +from django.db import connection from django.db.models import Avg, Count, Max, Min, StdDev from django.utils.timezone import now from django_countries.fields import Country @@ -10,13 +11,12 @@ DEFAULT_CUTOFF = timedelta(hours=24) -def annotate_url(url, delays): +def annotate_url(url, delay): '''Given a MirrorURL object, add a few more attributes to it regarding status, including completion_pct, delay, and score.''' url.completion_pct = float(url.success_count) / url.check_count - if url.id in delays: - url_delays = delays[url.id] - url.delay = sum(url_delays, timedelta()) / len(url_delays) + if delay is not None: + url.delay = delay hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 if url.completion_pct > 0: @@ -30,6 +30,30 @@ def annotate_url(url, delays): url.score = None +def url_delays(cutoff_time, mirror_id=None): + cursor = connection.cursor() + if mirror_id is None: + sql= """ +SELECT url_id, AVG(check_time - last_sync) +FROM mirrors_mirrorlog +WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL +GROUP BY url_id +""" + cursor.execute(sql, [True, cutoff_time]) + else: + sql = """ +SELECT l.url_id, avg(check_time - last_sync) +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL +AND mirror_id = %s +GROUP BY url_id +""" + cursor.execute(sql, [True, cutoff_time, mirror_id]) + + return {url_id: delay for url_id, delay in cursor.fetchall()} + + @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff @@ -55,20 +79,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - - # The Django ORM makes it really hard to get actual average delay in the - # above query, so run a seperate query for it and we will process the - # results here. - times = MirrorLog.objects.values_list( - 'url_id', 'check_time', 'last_sync').filter( - is_success=True, last_sync__isnull=False, - check_time__gte=cutoff_time) - if mirror_ids: - times = times.filter(url__mirror_id__in=mirror_ids) - delays = {} - for url_id, check_time, last_sync in times: - delay = check_time - last_sync - delays.setdefault(url_id, []).append(delay) + delays = url_delays(cutoff_time) if urls: url_data = dict((item['id'], item) for item in url_data) @@ -97,7 +108,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): # fake the standard deviation for local testing setups if vendor == 'sqlite': setattr(url, 'duration_stddev', 0.0) - annotate_url(url, delays) + annotate_url(url, delays.get(url.id, None)) return { 'cutoff': cutoff, -- cgit v1.2.3-54-g00ecf From c588d1c85f86f5ee10a96bec679111c8675b703c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 11:38:11 -0500 Subject: Support only a single mirror ID in error/status retrieval This simplifies things and makes injecting this single mirror ID into custom SQL a whole lot easier. Signed-off-by: Dan McGee --- mirrors/utils.py | 18 +++++++++--------- mirrors/views.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 2721e20e..d18dc22f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -55,15 +55,15 @@ def url_delays(cutoff_time, mirror_id=None): @cache_function(123) -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).distinct() - if mirror_ids: - valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) + if mirror_id: + valid_urls = valid_urls.filter(mirror_id=mirror_id) url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( @@ -79,7 +79,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - delays = url_delays(cutoff_time) + delays = url_delays(cutoff_time, mirror_id) if urls: url_data = dict((item['id'], item) for item in url_data) @@ -90,8 +90,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) - if mirror_ids: - check_info = check_info.filter(url__mirror_id__in=mirror_ids) + if mirror_id: + check_info = check_info.filter(url__mirror_id=mirror_id) check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: @@ -120,7 +120,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): @cache_function(117) -def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, @@ -130,8 +130,8 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') - if mirror_ids: - urls = urls.filter(mirror_id__in=mirror_ids) + if mirror_id: + urls = urls.filter(mirror_id=mirror_id) errors = list(errors) for err in errors: diff --git a/mirrors/views.py b/mirrors/views.py index 56397633..07e28d40 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -175,7 +175,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} all_urls = set(mirror.urls.select_related('protocol')) @@ -193,7 +193,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, -- cgit v1.2.3-54-g00ecf From 213aa3a2fab6f3a56be348a067c132f568efbaff Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:09:41 -0500 Subject: Reduce mirror status query madness Move completely to custom SQL for this logic. The Django ORM just doesn't play nice with the kind of query we are looking to do, so it is easier to do using raw SQL. The biggest pain factor here is in supporting sqlite as it doesn't have nearly the capabilities in handling datetime types directly in the database, as well as having some different type conversion necessities. Signed-off-by: Dan McGee --- mirrors/utils.py | 146 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 55 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index d18dc22f..eb1211f1 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -2,6 +2,7 @@ from django.db import connection from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country @@ -11,47 +12,103 @@ DEFAULT_CUTOFF = timedelta(hours=24) -def annotate_url(url, delay): - '''Given a MirrorURL object, add a few more attributes to it regarding - status, including completion_pct, delay, and score.''' - url.completion_pct = float(url.success_count) / url.check_count - if delay is not None: - url.delay = delay - hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 - if url.completion_pct > 0: - divisor = url.completion_pct - else: - # arbitrary small value - divisor = 0.005 - url.score = (hours + url.duration_avg + url.duration_stddev) / divisor - else: - url.delay = None - url.score = None +def dictfetchall(cursor): + "Returns all rows from a cursor as a dict." + desc = cursor.description + return [ + dict(zip([col[0] for col in desc], row)) + for row in cursor.fetchall() + ] -def url_delays(cutoff_time, mirror_id=None): - cursor = connection.cursor() - if mirror_id is None: - sql= """ -SELECT url_id, AVG(check_time - last_sync) -FROM mirrors_mirrorlog -WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL -GROUP BY url_id +def status_data(cutoff_time, mirror_id=None): + if mirror_id is not None: + params = [cutoff_time, mirror_id] + mirror_where = 'AND u.mirror_id = %s' + else: + params = [cutoff_time] + mirror_where = '' + + vendor = database_vendor(MirrorUrl) + if vendor == 'sqlite': + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + 0.0 AS duration_stddev, + AVG(STRFTIME('%%s', check_time) - STRFTIME('%%s', last_sync)) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id """ - cursor.execute(sql, [True, cutoff_time]) else: sql = """ -SELECT l.url_id, avg(check_time - last_sync) +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + STDDEV(l.duration) AS duration_stddev, + AVG(check_time - last_sync) AS delay FROM mirrors_mirrorlog l JOIN mirrors_mirrorurl u ON u.id = l.url_id -WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL -AND mirror_id = %s -GROUP BY url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id """ - cursor.execute(sql, [True, cutoff_time, mirror_id]) - return {url_id: delay for url_id, delay in cursor.fetchall()} + cursor = connection.cursor() + cursor.execute(sql, params) + url_data = dictfetchall(cursor) + + # sqlite loves to return less than ideal types + if vendor == 'sqlite': + for item in url_data: + item['delay'] = timedelta(seconds=item['delay']) + item['last_sync'] = parse_datetime(item['last_sync']) + item['last_check'] = parse_datetime(item['last_check']) + + return {item['url_id']: item for item in url_data} + + +def annotate_url(url, url_data): + '''Given a MirrorURL object, add a few more attributes to it regarding + status, including completion_pct, delay, and score.''' + known_attrs = ( + ('success_count', 0), + ('check_count', 0), + ('completion_pct', None), + ('last_check', None), + ('last_sync', None), + ('delay', None), + ('score', None), + ) + for k, v in known_attrs: + setattr(url, k, v) + for k, v in url_data.items(): + if k not in ('url_id', 'mirror_id'): + setattr(url, k, v) + + if url.check_count > 0: + url.completion_pct = float(url.success_count) / url.check_count + + if url.delay is not None: + hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 + + if url.completion_pct > 0: + divisor = url.completion_pct + else: + # arbitrary small value + divisor = 0.005 + stddev = url.duration_stddev or 0.0 + url.score = (hours + url.duration_avg + stddev) / divisor @cache_function(123) @@ -65,29 +122,14 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): if mirror_id: valid_urls = valid_urls.filter(mirror_id=mirror_id) - url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( - id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( - check_count=Count('logs'), - success_count=Count('logs__duration'), - last_sync=Max('logs__last_sync'), - last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')) - - vendor = database_vendor(MirrorUrl) - if vendor != 'sqlite': - url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) - + url_data = status_data(cutoff_time, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - delays = url_delays(cutoff_time, mirror_id) if urls: - url_data = dict((item['id'], item) for item in url_data) for url in urls: - for k, v in url_data.get(url.id, {}).items(): - if k not in ('id', 'mirror_id'): - setattr(url, k, v) - last_check = max([u.last_check for u in urls]) + annotate_url(url, url_data.get(url.id, {})) + last_check = max([u.last_check for u in urls if u.last_check]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) if mirror_id: @@ -104,12 +146,6 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): num_checks = 0 check_frequency = None - for url in urls: - # fake the standard deviation for local testing setups - if vendor == 'sqlite': - setattr(url, 'duration_stddev', 0.0) - annotate_url(url, delays.get(url.id, None)) - return { 'cutoff': cutoff, 'last_check': last_check, -- cgit v1.2.3-54-g00ecf From f357a39a49a8edc713d512976a0be2a2a8ac5c4f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:21:16 -0500 Subject: Remove cache_function decorator from a few spots The benefit of these storage operations might be outweighed by the cost, especially given how infrequently these functions are called. Signed-off-by: Dan McGee --- main/models.py | 5 +---- mirrors/utils.py | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) (limited to 'mirrors/utils.py') diff --git a/main/models.py b/main/models.py index a561f4f6..89215f05 100644 --- a/main/models.py +++ b/main/models.py @@ -10,7 +10,7 @@ from django.utils.timezone import now from .fields import PositiveBigIntegerField -from .utils import cache_function, set_created_field +from .utils import set_created_field from devel.models import DeveloperKey from packages.alpm import AlpmAPI @@ -187,7 +187,6 @@ def applicable_arches(self): self._applicable_arches = list(arches) return self._applicable_arches - #@cache_function(119) def get_requiredby(self): """ Returns a list of package objects. An attempt will be made to keep this @@ -265,7 +264,6 @@ def get_requiredby(self): trimmed.append(dep) return trimmed - #@cache_function(121) def get_depends(self): """ Returns a list of dicts. Each dict contains ('dep', 'pkg', and @@ -293,7 +291,6 @@ def sort_key(val): return (sort_order.get(dep.deptype, 1000), dep.name) return sorted(deps, key=sort_key) - #@cache_function(123) def reverse_conflicts(self): """ Returns a list of packages with conflicts against this package. diff --git a/mirrors/utils.py b/mirrors/utils.py index eb1211f1..5a8bbf5d 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -111,7 +111,6 @@ def annotate_url(url, url_data): url.score = (hours + url.duration_avg + stddev) / divisor -@cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff @@ -155,7 +154,6 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): } -@cache_function(117) def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( -- cgit v1.2.3-54-g00ecf From b7b24740640e24883cd17fd683e1d465fbb343f8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 22:12:01 -0500 Subject: Various minor code cleanups and fixes Most of these were suggested by PyCharm, and include everything from little syntax issues and other bad smells to dead or bad code. Signed-off-by: Dan McGee --- devel/management/commands/pgp_import.py | 1 + devel/models.py | 1 - devel/utils.py | 2 +- devel/views.py | 2 +- main/log.py | 1 - main/migrations/0029_fill_in_repo_data.py | 1 - main/models.py | 12 +++++------- main/utils.py | 1 - mirrors/management/commands/mirrorcheck.py | 12 +++--------- mirrors/models.py | 2 +- mirrors/utils.py | 6 +++--- packages/migrations/0002_populate_package_relation.py | 2 -- packages/templatetags/package_extras.py | 4 ++-- packages/utils.py | 2 +- packages/views/display.py | 2 -- packages/views/flag.py | 3 +-- public/views.py | 1 - releng/management/commands/syncisos.py | 2 +- releng/models.py | 2 +- releng/views.py | 2 +- retro/templates/retro/index-20030330.html | 1 - sitestatic/archweb.js | 1 - todolists/utils.py | 1 - todolists/views.py | 1 - visualize/static/visualize.js | 2 +- 25 files changed, 23 insertions(+), 44 deletions(-) (limited to 'mirrors/utils.py') diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py index 10e6cfcb..b1f29d77 100644 --- a/devel/management/commands/pgp_import.py +++ b/devel/management/commands/pgp_import.py @@ -95,6 +95,7 @@ def parse_keydata(data): # parse all of the output from our successful GPG command logger.info("parsing command output") + node = None for line in data.split('\n'): parts = line.split(':') if parts[0] == 'pub': diff --git a/devel/models.py b/devel/models.py index 67de40a6..4354e0f2 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,7 +4,6 @@ from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User -from django.utils.timezone import now from django_countries import CountryField from .fields import PGPKeyField diff --git a/devel/utils.py b/devel/utils.py index e8e3a6c4..340841f5 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -131,7 +131,7 @@ def find(self, userstring): self.username_email, self.user_name) for matcher in find_methods: user = matcher(name, email) - if user != None: + if user is not None: break self.cache[userstring] = user diff --git a/devel/views.py b/devel/views.py index 61c1e568..4258ea7f 100644 --- a/devel/views.py +++ b/devel/views.py @@ -34,7 +34,7 @@ @login_required def index(request): '''the developer dashboard''' - if(request.user.is_authenticated()): + if request.user.is_authenticated(): inner_q = PackageRelation.objects.filter(user=request.user) else: inner_q = PackageRelation.objects.none() diff --git a/main/log.py b/main/log.py index 63634874..5c745cc8 100644 --- a/main/log.py +++ b/main/log.py @@ -46,7 +46,6 @@ def filter(self, record): trace = '\n'.join(traceback.format_exception(*record.exc_info)) key = md5(trace).hexdigest() - duplicate = False cache = self.cache_module.cache # Test if the cache works diff --git a/main/migrations/0029_fill_in_repo_data.py b/main/migrations/0029_fill_in_repo_data.py index 0887b28c..7da6b1c4 100644 --- a/main/migrations/0029_fill_in_repo_data.py +++ b/main/migrations/0029_fill_in_repo_data.py @@ -7,7 +7,6 @@ class Migration(DataMigration): def forwards(self, orm): - "Write your forwards methods here." orm.Repo.objects.filter(name__istartswith='community').update(bugs_project=5, svn_root='community') orm.Repo.objects.filter(name__iexact='multilib').update(bugs_project=5, svn_root='community') diff --git a/main/models.py b/main/models.py index 89215f05..24aeed89 100644 --- a/main/models.py +++ b/main/models.py @@ -7,7 +7,6 @@ from django.db.models import Q from django.contrib.auth.models import User from django.contrib.sites.models import Site -from django.utils.timezone import now from .fields import PositiveBigIntegerField from .utils import set_created_field @@ -140,7 +139,7 @@ def get_full_url(self, proto='https'): @property def signature(self): try: - data = b64decode(self.pgp_signature) + data = b64decode(self.pgp_signature.encode('utf-8')) except TypeError: return None if not data: @@ -274,7 +273,6 @@ def get_depends(self): Packages will match the testing status of this package if possible. """ deps = [] - arches = None # TODO: we can use list comprehension and an 'in' query to make this # more effective for dep in self.depends.all(): @@ -400,13 +398,13 @@ def elsewhere(self): '''attempt to locate this package anywhere else, regardless of architecture or repository. Excludes this package from the list.''' names = [self.pkgname] - if self.pkgname.startswith('lib32-'): + if self.pkgname.startswith(u'lib32-'): names.append(self.pkgname[6:]) - elif self.pkgname.endswith('-multilib'): + elif self.pkgname.endswith(u'-multilib'): names.append(self.pkgname[:-9]) else: - names.append('lib32-' + self.pkgname) - names.append(self.pkgname + '-multilib') + names.append(u'lib32-' + self.pkgname) + names.append(self.pkgname + u'-multilib') return Package.objects.normal().filter( pkgname__in=names).exclude(id=self.id).order_by( 'arch__name', 'repo__name') diff --git a/main/utils.py b/main/utils.py index 8394e5cd..9ee8db58 100644 --- a/main/utils.py +++ b/main/utils.py @@ -3,7 +3,6 @@ except ImportError: import pickle -from datetime import datetime import hashlib from django.core.cache import cache diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index d6de8f22..e7dd7b49 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -106,19 +106,13 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, location, timeout): - if location: - if location.family == socket.AF_INET6: - ipopt = '--ipv6' - elif location.family == socket.AF_INET: - ipopt = '--ipv4' - url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=now(), location=location) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) + start = time.time() try: - start = time.time() result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() @@ -147,12 +141,12 @@ def check_mirror_url(mirror_url, location, timeout): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except HTTPException as e: + except HTTPException: # e.g., BadStatusLine log.is_success = False log.error = "Exception in processing HTTP request." logger.debug("failed: %s, %s", url, log.error) - except socket.timeout as e: + except socket.timeout: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) diff --git a/mirrors/models.py b/mirrors/models.py index 791b0078..d8ac7952 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -92,7 +92,7 @@ def clean(self): families = self.address_families() self.has_ipv4 = socket.AF_INET in families self.has_ipv6 = socket.AF_INET6 in families - except socket.error as e: + except socket.error: # We don't fail in this case; we'll just set both to False self.has_ipv4 = False self.has_ipv6 = False diff --git a/mirrors/utils.py b/mirrors/utils.py index 5a8bbf5d..531cf005 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,13 @@ from datetime import timedelta from django.db import connection -from django.db.models import Avg, Count, Max, Min, StdDev +from django.db.models import Count, Max, Min from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country from main.utils import cache_function, database_vendor -from .models import MirrorLog, MirrorProtocol, MirrorUrl +from .models import MirrorLog, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) @@ -165,7 +165,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): ).order_by('-last_occurred', '-error_count') if mirror_id: - urls = urls.filter(mirror_id=mirror_id) + errors = errors.filter(url__mirror_id=mirror_id) errors = list(errors) for err in errors: diff --git a/packages/migrations/0002_populate_package_relation.py b/packages/migrations/0002_populate_package_relation.py index 738e068f..b0d32c7a 100644 --- a/packages/migrations/0002_populate_package_relation.py +++ b/packages/migrations/0002_populate_package_relation.py @@ -11,7 +11,6 @@ class Migration(DataMigration): ) def forwards(self, orm): - "Write your forwards methods here." # search by pkgbase first and insert those records qs = orm['main.Package'].objects.exclude(maintainer=None).exclude( pkgbase=None).distinct().values('pkgbase', 'maintainer_id') @@ -29,7 +28,6 @@ def forwards(self, orm): defaults={'user_id': row['maintainer_id']}) def backwards(self, orm): - "Write your backwards methods here." if not db.dry_run: orm.PackageRelation.objects.all().delete() pass diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index f14fab1e..ef0e1aea 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -53,10 +53,10 @@ def do_buildsortqs(parser, token): tagname, sortfield = token.split_contents() except ValueError: raise template.TemplateSyntaxError( - "%r tag requires a single argument" % tagname) + "%r tag requires a single argument" % token) if not (sortfield[0] == sortfield[-1] and sortfield[0] in ('"', "'")): raise template.TemplateSyntaxError( - "%r tag's argument should be in quotes" % tagname) + "%r tag's argument should be in quotes" % token) return BuildQueryStringNode(sortfield[1:-1]) diff --git a/packages/utils.py b/packages/utils.py index a4217fbd..4f3b8665 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -391,7 +391,7 @@ def signoffs_id_query(model, repos): """ cursor = connection.cursor() # query pre-process- fill in table name and placeholders for IN - repo_sql = ','.join(['%s' for r in repos]) + repo_sql = ','.join(['%s' for _ in repos]) sql = sql % (model._meta.db_table, repo_sql, repo_sql) repo_ids = [r.pk for r in repos] # repo_ids are needed twice, so double the array diff --git a/packages/views/display.py b/packages/views/display.py index 87424483..021c7ed8 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -228,8 +228,6 @@ def download(request, name, repo, arch): if pkg.arch.agnostic: # grab the first non-any arch to fake the download path arch = Arch.objects.exclude(agnostic=True)[0].name - values = { - } url = '{host}{repo}/os/{arch}/{filename}'.format(host=url.url, repo=pkg.repo.name.lower(), arch=arch, filename=pkg.filename) return redirect(url) diff --git a/packages/views/flag.py b/packages/views/flag.py index 5c76e1d5..39cdcef8 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -110,7 +110,7 @@ def perform_updates(): subject = '%s package [%s] marked out-of-date' % \ (pkg.repo.name, pkg.pkgname) for maint in maints: - if maint.userprofile.notify == True: + if maint.userprofile.notify is True: toemail.append(maint.email) if toemail: @@ -133,7 +133,6 @@ def perform_updates(): return redirect('package-flag-confirmed', name=name, repo=repo, arch=arch) else: - initial = {} form = FlagForm(authenticated=authenticated) context = { diff --git a/public/views.py b/public/views.py index 22cb8759..39273396 100644 --- a/public/views.py +++ b/public/views.py @@ -125,7 +125,6 @@ def keys(request): master_keys = MasterKey.objects.select_related('owner', 'revoker', 'owner__userprofile', 'revoker__userprofile').filter( revoked__isnull=True) - master_key_ids = frozenset(key.pgp_key[-16:] for key in master_keys) sig_counts = PGPSignature.objects.filter(not_expired, valid=True, signee__in=user_key_ids).order_by().values_list('signer').annotate( diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index c9f61964..f182cc33 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -20,7 +20,7 @@ def handle_starttag(self, tag, attrs): if tag == 'a': for name, value in attrs: if name == "href": - if value != '../' and self.url_re.search(value) != None: + if value != '../' and self.url_re.search(value) is not None: self.hyperlinks.append(value[:-1]) def parse(self, url): diff --git a/releng/models.py b/releng/models.py index b95f7d52..5ee2f325 100644 --- a/releng/models.py +++ b/releng/models.py @@ -160,7 +160,7 @@ def info_html(self): def torrent(self): try: - data = b64decode(self.torrent_data) + data = b64decode(self.torrent_data.encode('utf-8')) except TypeError: return None if not data: diff --git a/releng/views.py b/releng/views.py index ad4b07d1..b1c76a4a 100644 --- a/releng/views.py +++ b/releng/views.py @@ -231,7 +231,7 @@ def release_torrent(request, version): release = get_object_or_404(Release, version=version) if not release.torrent_data: raise Http404 - data = b64decode(release.torrent_data) + data = b64decode(release.torrent_data.encode('utf-8')) response = HttpResponse(data, content_type='application/x-bittorrent') # TODO: this is duplicated from Release.iso_url() filename = 'archlinux-%s-dual.iso.torrent' % release.version diff --git a/retro/templates/retro/index-20030330.html b/retro/templates/retro/index-20030330.html index 449731af..51cc8ba3 100644 --- a/retro/templates/retro/index-20030330.html +++ b/retro/templates/retro/index-20030330.html @@ -232,7 +232,6 @@
[ Older News ]

-


diff --git a/sitestatic/archweb.js b/sitestatic/archweb.js index dda22d9e..aa225f5f 100644 --- a/sitestatic/archweb.js +++ b/sitestatic/archweb.js @@ -146,7 +146,6 @@ if (typeof $ !== 'undefined' && typeof $.tablesorter !== 'undefined') { (function($) { $.fn.enableCheckboxRangeSelection = function() { var lastCheckbox = null, - lastElement = null, spec = this; spec.unbind("click.checkboxrange"); diff --git a/todolists/utils.py b/todolists/utils.py index 51a75a3c..7b98c887 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -1,5 +1,4 @@ from django.db import connections, router -from django.db.models import Count from .models import Todolist, TodolistPackage from packages.models import Package diff --git a/todolists/views.py b/todolists/views.py index 7636d38e..d5b39934 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -9,7 +9,6 @@ from django.views.decorators.cache import never_cache from django.views.generic import DeleteView from django.template import Context, loader -from django.template.defaultfilters import slugify from django.utils.timezone import now from main.models import Package, Repo diff --git a/visualize/static/visualize.js b/visualize/static/visualize.js index 7e240d44..5004fe6c 100644 --- a/visualize/static/visualize.js +++ b/visualize/static/visualize.js @@ -55,7 +55,7 @@ function packages_treemap(chart_id, orderings, default_order) { var nodes = d3_div.data([json]).selectAll("div") .data(treemap.nodes, key_func); /* start out new nodes in the center of the picture area */ - var w_center = jq_div.width() / 2; + var w_center = jq_div.width() / 2, h_center = jq_div.height() / 2; nodes.enter().append("div") .attr("class", "treemap-cell") -- cgit v1.2.3-54-g00ecf From 6de0cfbd23aae69036439db817cc26740d8796cd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 11:13:49 -0500 Subject: Fix some None issues with sqlite3 and mirror status If certain attributes came back from the database as NULL, we had issues parsing them. Pass None/NULL straight through rather than trying to type-convert. Signed-off-by: Dan McGee --- mirrors/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 531cf005..ba45da5f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -71,8 +71,10 @@ def status_data(cutoff_time, mirror_id=None): # sqlite loves to return less than ideal types if vendor == 'sqlite': for item in url_data: - item['delay'] = timedelta(seconds=item['delay']) - item['last_sync'] = parse_datetime(item['last_sync']) + if item['delay'] is not None: + item['delay'] = timedelta(seconds=item['delay']) + if item['last_sync'] is not None: + item['last_sync'] = parse_datetime(item['last_sync']) item['last_check'] = parse_datetime(item['last_check']) return {item['url_id']: item for item in url_data} -- cgit v1.2.3-54-g00ecf