summaryrefslogtreecommitdiff
path: root/mirrors/utils.py
diff options
context:
space:
mode:
Diffstat (limited to 'mirrors/utils.py')
-rw-r--r--mirrors/utils.py163
1 files changed, 105 insertions, 58 deletions
diff --git a/mirrors/utils.py b/mirrors/utils.py
index 3ab176b3..ba45da5f 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -1,22 +1,107 @@
from datetime import timedelta
-from django.db.models import Avg, Count, Max, Min, StdDev
+from django.db import connection
+from django.db.models import Count, Max, Min
+from django.utils.dateparse import parse_datetime
from django.utils.timezone import now
from django_countries.fields import Country
from main.utils import cache_function, database_vendor
-from .models import MirrorLog, MirrorProtocol, MirrorUrl
+from .models import MirrorLog, MirrorUrl
DEFAULT_CUTOFF = timedelta(hours=24)
-def annotate_url(url, delays):
+
+def dictfetchall(cursor):
+ "Returns all rows from a cursor as a dict."
+ desc = cursor.description
+ return [
+ dict(zip([col[0] for col in desc], row))
+ for row in cursor.fetchall()
+ ]
+
+
+def status_data(cutoff_time, mirror_id=None):
+ if mirror_id is not None:
+ params = [cutoff_time, mirror_id]
+ mirror_where = 'AND u.mirror_id = %s'
+ else:
+ params = [cutoff_time]
+ mirror_where = ''
+
+ vendor = database_vendor(MirrorUrl)
+ if vendor == 'sqlite':
+ sql = """
+SELECT l.url_id, u.mirror_id,
+ COUNT(l.id) AS check_count,
+ COUNT(l.duration) AS success_count,
+ MAX(l.last_sync) AS last_sync,
+ MAX(l.check_time) AS last_check,
+ AVG(l.duration) AS duration_avg,
+ 0.0 AS duration_stddev,
+ AVG(STRFTIME('%%s', check_time) - STRFTIME('%%s', last_sync)) AS delay
+FROM mirrors_mirrorlog l
+JOIN mirrors_mirrorurl u ON u.id = l.url_id
+WHERE l.check_time >= %s
+""" + mirror_where + """
+GROUP BY l.url_id, u.mirror_id
+"""
+ else:
+ sql = """
+SELECT l.url_id, u.mirror_id,
+ COUNT(l.id) AS check_count,
+ COUNT(l.duration) AS success_count,
+ MAX(l.last_sync) AS last_sync,
+ MAX(l.check_time) AS last_check,
+ AVG(l.duration) AS duration_avg,
+ STDDEV(l.duration) AS duration_stddev,
+ AVG(check_time - last_sync) AS delay
+FROM mirrors_mirrorlog l
+JOIN mirrors_mirrorurl u ON u.id = l.url_id
+WHERE l.check_time >= %s
+""" + mirror_where + """
+GROUP BY l.url_id, u.mirror_id
+"""
+
+ cursor = connection.cursor()
+ cursor.execute(sql, params)
+ url_data = dictfetchall(cursor)
+
+ # sqlite loves to return less than ideal types
+ if vendor == 'sqlite':
+ for item in url_data:
+ if item['delay'] is not None:
+ item['delay'] = timedelta(seconds=item['delay'])
+ if item['last_sync'] is not None:
+ item['last_sync'] = parse_datetime(item['last_sync'])
+ item['last_check'] = parse_datetime(item['last_check'])
+
+ return {item['url_id']: item for item in url_data}
+
+
+def annotate_url(url, url_data):
'''Given a MirrorURL object, add a few more attributes to it regarding
status, including completion_pct, delay, and score.'''
- url.completion_pct = float(url.success_count) / url.check_count
- if url.id in delays:
- url_delays = delays[url.id]
- url.delay = sum(url_delays, timedelta()) / len(url_delays)
+ known_attrs = (
+ ('success_count', 0),
+ ('check_count', 0),
+ ('completion_pct', None),
+ ('last_check', None),
+ ('last_sync', None),
+ ('delay', None),
+ ('score', None),
+ )
+ for k, v in known_attrs:
+ setattr(url, k, v)
+ for k, v in url_data.items():
+ if k not in ('url_id', 'mirror_id'):
+ setattr(url, k, v)
+
+ if url.check_count > 0:
+ url.completion_pct = float(url.success_count) / url.check_count
+
+ if url.delay is not None:
hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0
if url.completion_pct > 0:
@@ -24,63 +109,32 @@ def annotate_url(url, delays):
else:
# arbitrary small value
divisor = 0.005
- url.score = (hours + url.duration_avg + url.duration_stddev) / divisor
- else:
- url.delay = None
- url.score = None
+ stddev = url.duration_stddev or 0.0
+ url.score = (hours + url.duration_avg + stddev) / divisor
-@cache_function(123)
-def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None):
+def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None):
cutoff_time = now() - cutoff
valid_urls = MirrorUrl.objects.filter(
mirror__active=True, mirror__public=True,
logs__check_time__gte=cutoff_time).distinct()
- if mirror_ids:
- valid_urls = valid_urls.filter(mirror_id__in=mirror_ids)
-
- url_data = MirrorUrl.objects.values('id', 'mirror_id').filter(
- id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate(
- check_count=Count('logs'),
- success_count=Count('logs__duration'),
- last_sync=Max('logs__last_sync'),
- last_check=Max('logs__check_time'),
- duration_avg=Avg('logs__duration'))
-
- vendor = database_vendor(MirrorUrl)
- if vendor != 'sqlite':
- url_data = url_data.annotate(duration_stddev=StdDev('logs__duration'))
+ if mirror_id:
+ valid_urls = valid_urls.filter(mirror_id=mirror_id)
+ url_data = status_data(cutoff_time, mirror_id)
urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter(
id__in=valid_urls).order_by('mirror__id', 'url')
- # The Django ORM makes it really hard to get actual average delay in the
- # above query, so run a seperate query for it and we will process the
- # results here.
- times = MirrorLog.objects.values_list(
- 'url_id', 'check_time', 'last_sync').filter(
- is_success=True, last_sync__isnull=False,
- check_time__gte=cutoff_time)
- if mirror_ids:
- times = times.filter(url__mirror_id__in=mirror_ids)
- delays = {}
- for url_id, check_time, last_sync in times:
- delay = check_time - last_sync
- delays.setdefault(url_id, []).append(delay)
-
if urls:
- url_data = dict((item['id'], item) for item in url_data)
for url in urls:
- for k, v in url_data.get(url.id, {}).items():
- if k not in ('id', 'mirror_id'):
- setattr(url, k, v)
- last_check = max([u.last_check for u in urls])
+ annotate_url(url, url_data.get(url.id, {}))
+ last_check = max([u.last_check for u in urls if u.last_check])
num_checks = max([u.check_count for u in urls])
check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time)
- if mirror_ids:
- check_info = check_info.filter(url__mirror_id__in=mirror_ids)
+ if mirror_id:
+ check_info = check_info.filter(url__mirror_id=mirror_id)
check_info = check_info.aggregate(
mn=Min('check_time'), mx=Max('check_time'))
if num_checks > 1:
@@ -93,12 +147,6 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None):
num_checks = 0
check_frequency = None
- for url in urls:
- # fake the standard deviation for local testing setups
- if vendor == 'sqlite':
- setattr(url, 'duration_stddev', 0.0)
- annotate_url(url, delays)
-
return {
'cutoff': cutoff,
'last_check': last_check,
@@ -108,8 +156,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None):
}
-@cache_function(117)
-def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None):
+def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None):
cutoff_time = now() - cutoff
errors = MirrorLog.objects.filter(
is_success=False, check_time__gte=cutoff_time,
@@ -119,8 +166,8 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None):
error_count=Count('error'), last_occurred=Max('check_time')
).order_by('-last_occurred', '-error_count')
- if mirror_ids:
- urls = urls.filter(mirror_id__in=mirror_ids)
+ if mirror_id:
+ errors = errors.filter(url__mirror_id=mirror_id)
errors = list(errors)
for err in errors:
generated by cgit v1.2.3-54-g00ecf (git 2.45.2) at 2024-11-22 12:28:53 +0000