From 78a553e558217640c0efa233ac7a7037be3f34b5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 12 Apr 2011 18:36:20 -0500 Subject: Add optional country override for individual mirror URLs This allows a named top-level mirror to have geographically distributed URLs, e.g. kernel.org and the geo-DNS setup. Signed-off-by: Dan McGee --- mirrors/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/utils.py b/mirrors/utils.py index 124b66e6..2d88f125 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -82,10 +82,13 @@ def get_mirror_errors(cutoff=default_cutoff): errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( - 'url__url', 'url__protocol__protocol', 'url__mirror__country', - 'error').annotate( + 'url__url', 'url__country', 'url__protocol__protocol', + 'url__mirror__country', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') - return list(errors) + errors = list(errors) + for err in errors: + err['country'] = err['url__country'] or err['url__mirror__country'] + return errors # vim: set ts=4 sw=4 et: -- cgit v1.2.3-54-g00ecf From 1b91de94357a60ad372abe4213d1fa52f5fc9c9e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 18 Apr 2011 15:10:20 -0500 Subject: mirrors: pylint discovered cleanups Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 6 ++--- mirrors/management/commands/mirrorresolv.py | 2 +- mirrors/utils.py | 42 ++++++++++++++++------------- mirrors/views.py | 7 +++-- 4 files changed, 30 insertions(+), 27 deletions(-) (limited to 'mirrors/utils.py') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 44971c4f..ea43d558 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -13,7 +13,7 @@ from django.db import transaction from collections import deque -from datetime import datetime, timedelta +from datetime import datetime import logging import re import socket @@ -134,8 +134,8 @@ def __init__(self, work, num_threads=10): @transaction.commit_on_success def run(self): logger.debug("starting threads") - for t in self.threads: - t.start() + for thread in self.threads: + thread.start() logger.debug("joining on all threads") self.tasks.join() logger.debug("processing log entries") diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 8a628bd4..4e812f2d 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -49,6 +49,6 @@ def resolve_mirrors(): mirrorurl.has_ipv4, mirrorurl.has_ipv6) mirrorurl.save(force_update=True) except socket.error, e: - logger.warn("error resolving %s: %s", hostname, e) + logger.warn("error resolving %s: %s", mirrorurl.hostname, e) # vim: set ts=4 sw=4 et: diff --git a/mirrors/utils.py b/mirrors/utils.py index 2d88f125..686ec581 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -7,6 +7,25 @@ default_cutoff = datetime.timedelta(hours=24) +def annotate_url(url, delays): + '''Given a MirrorURL object, add a few more attributes to it regarding + status, including completion_pct, delay, and score.''' + url.completion_pct = float(url.success_count) / url.check_count + if url.id in delays: + url_delays = delays[url.id] + url.delay = sum(url_delays, datetime.timedelta()) / len(url_delays) + hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 + + if url.completion_pct > 0: + divisor = url.completion_pct + else: + # arbitrary small value + divisor = 0.005 + url.score = (hours + url.duration_avg + url.duration_stddev) / divisor + else: + url.delay = None + url.score = None + @cache_function(300) def get_mirror_statuses(cutoff=default_cutoff): cutoff_time = datetime.datetime.utcnow() - cutoff @@ -31,8 +50,8 @@ def get_mirror_statuses(cutoff=default_cutoff): check_time__gte=cutoff_time) delays = {} for log in times: - d = log.check_time - log.last_sync - delays.setdefault(log.url_id, []).append(d) + delay = log.check_time - log.last_sync + delays.setdefault(log.url_id, []).append(delay) if urls: last_check = max([u.last_check for u in urls]) @@ -44,29 +63,14 @@ def get_mirror_statuses(cutoff=default_cutoff): check_frequency = (check_info['mx'] - check_info['mn']) \ / (num_checks - 1) else: - check_frequency = None; + check_frequency = None else: last_check = None num_checks = 0 check_frequency = None for url in urls: - url.completion_pct = float(url.success_count) / url.check_count - if url.id in delays: - url_delays = delays[url.id] - d = sum(url_delays, datetime.timedelta()) / len(url_delays) - url.delay = d - hours = d.days * 24.0 + d.seconds / 3600.0 - - if url.completion_pct > 0: - divisor = url.completion_pct - else: - # arbitrary small value - divisor = 0.005 - url.score = (hours + url.duration_avg + url.duration_stddev) / divisor - else: - url.delay = None - url.score = None + annotate_url(url, delays) return { 'cutoff': cutoff, diff --git a/mirrors/views.py b/mirrors/views.py index 69592146..f03a2e8a 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,6 +1,5 @@ from django import forms from django.core.serializers.json import DjangoJSONEncoder -from django.db.models import Avg, Count, Max, Min, StdDev from django.db.models import Q from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 @@ -96,11 +95,11 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, mimetype='text/plain') def mirrors(request): - mirrors = Mirror.objects.select_related().order_by('tier', 'country') + mirror_list = Mirror.objects.select_related().order_by('tier', 'country') if not request.user.is_authenticated(): - mirrors = mirrors.filter(public=True, active=True) + mirror_list = mirror_list.filter(public=True, active=True) return direct_to_template(request, 'mirrors/mirrors.html', - {'mirror_list': mirrors}) + {'mirror_list': mirror_list}) def mirror_details(request, name): mirror = get_object_or_404(Mirror, name=name) -- cgit v1.2.3-54-g00ecf