summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLuke Shumaker <lukeshu@sbcglobal.net>2015-04-13 21:10:46 -0400
committerLuke Shumaker <lukeshu@sbcglobal.net>2015-04-13 21:10:46 -0400
commit05a44e86fbb656e5a4f6228c28b61df740549b34 (patch)
tree4d272046b88cd07f82d48da8a2390d785ace96c5
parentc6eb3b7b3c77d1b70126e0755c9ee66de88d5395 (diff)
parent67b87ed4c0abd593e2833b3aa6b969c2e2451b60 (diff)
Merge branch 'archweb-generic' into master-nomake
# Conflicts: # local_settings.py.example
-rw-r--r--local_settings.py.example4
-rw-r--r--mirrors/utils.py69
-rw-r--r--mirrors/views.py19
-rw-r--r--releng/views.py5
-rw-r--r--requirements.txt2
-rw-r--r--requirements_prod.txt2
6 files changed, 42 insertions, 59 deletions
diff --git a/local_settings.py.example b/local_settings.py.example
index 3e33aa7e..44e38459 100644
--- a/local_settings.py.example
+++ b/local_settings.py.example
@@ -1,5 +1,3 @@
-### Django settings for Parabola GNU/Linux-libre project.
-
## Debug settings
DEBUG = False # If you are running without another HTTP server, must be true
TEMPLATE_DEBUG = False
@@ -55,8 +53,6 @@ CACHES = {
#'LOCATION': '127.0.0.1:11211',
}
}
-CACHE_MIDDLEWARE_KEY_PREFIX = 'arch'
-CACHE_MIDDLEWARE_SECONDS = 300
## Use secure session cookies? Make this true if you want all
## logged-in actions to take place over HTTPS only. If developing
diff --git a/mirrors/utils.py b/mirrors/utils.py
index fe18cd6a..51daf50e 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -21,9 +21,7 @@ def dictfetchall(cursor):
for row in cursor.fetchall()
]
-@cache_function(178)
-def status_data(cutoff=DEFAULT_CUTOFF, mirror_id=None):
- cutoff_time = now() - cutoff
+def status_data(cutoff_time, mirror_id=None):
if mirror_id is not None:
params = [cutoff_time, mirror_id]
mirror_where = 'AND u.mirror_id = %s'
@@ -84,19 +82,16 @@ GROUP BY l.url_id, u.mirror_id
def annotate_url(url, url_data):
'''Given a MirrorURL object, add a few more attributes to it regarding
status, including completion_pct, delay, and score.'''
- known_attrs = (
- ('success_count', 0),
- ('check_count', 0),
- ('completion_pct', None),
- ('duration_avg', None),
- ('duration_stddev', None),
- ('last_check', None),
- ('last_sync', None),
- ('delay', None),
- ('score', None),
- )
- for k, v in known_attrs:
- setattr(url, k, v)
+ # set up some sane default values in case we are missing anything
+ url.success_count = 0
+ url.check_count = 0
+ url.completion_pct = None
+ url.duration_avg = None
+ url.duration_stddev = None
+ url.last_check = None
+ url.last_sync = None
+ url.delay = None
+ url.score = None
for k, v in url_data.items():
if k not in ('url_id', 'mirror_id'):
setattr(url, k, v)
@@ -107,7 +102,7 @@ def annotate_url(url, url_data):
if url.delay is not None:
hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0
- if url.completion_pct > 0:
+ if url.completion_pct > 0.0:
divisor = url.completion_pct
else:
# arbitrary small value
@@ -115,28 +110,26 @@ def annotate_url(url, url_data):
stddev = url.duration_stddev or 0.0
url.score = (hours + url.duration_avg + stddev) / divisor
+ return url
+
+@cache_function(178)
def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False):
cutoff_time = now() - cutoff
- valid_urls = MirrorUrl.objects.filter(
- logs__check_time__gte=cutoff_time).distinct()
-
+ urls = MirrorUrl.objects.select_related(
+ 'mirror', 'protocol').order_by('mirror__id', 'url')
if mirror_id:
- valid_urls = valid_urls.filter(mirror_id=mirror_id)
+ urls = urls.filter(mirror_id=mirror_id)
if not show_all:
- valid_urls = valid_urls.filter(active=True, mirror__active=True,
+ urls = urls.filter(active=True, mirror__active=True,
mirror__public=True)
- url_data = status_data(cutoff, mirror_id)
- urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter(
- id__in=valid_urls).order_by('mirror__id', 'url')
-
if urls:
- for url in urls:
- annotate_url(url, url_data.get(url.id, {}))
- last_check = max([u.last_check for u in urls if u.last_check])
- num_checks = max([u.check_count for u in urls])
+ url_data = status_data(cutoff_time, mirror_id)
+ urls = [annotate_url(url, url_data.get(url.id, {})) for url in urls]
+ last_check = max([u.last_check for u in urls if u.last_check] or [None])
+ num_checks = max(u.check_count for u in urls)
check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time)
if mirror_id:
check_info = check_info.filter(url__mirror_id=mirror_id)
@@ -195,23 +188,21 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF):
if status_data['check_time__max'] is not None:
min_check_time = status_data['check_time__max'] - timedelta(minutes=5)
min_sync_time = status_data['last_sync__max'] - timedelta(minutes=20)
- best_logs = MirrorLog.objects.filter(is_success=True,
+ best_logs = MirrorLog.objects.select_related('url').filter(
+ is_success=True,
check_time__gte=min_check_time, last_sync__gte=min_sync_time,
url__active=True,
url__mirror__public=True, url__mirror__active=True,
url__protocol__default=True).order_by(
'duration')[:1]
if best_logs:
- return MirrorUrl.objects.get(id=best_logs[0].url_id)
+ return best_logs[0].url
mirror_urls = MirrorUrl.objects.filter(active=True,
- mirror__public=True, mirror__active=True, protocol__default=True)
- # look first for a country-agnostic URL, then fall back to any HTTP URL
- filtered_urls = mirror_urls.filter(country='')[:1]
- if not filtered_urls:
- filtered_urls = mirror_urls[:1]
- if not filtered_urls:
+ mirror__public=True, mirror__active=True,
+ protocol__default=True)[:1]
+ if not mirror_urls:
return None
- return filtered_urls[0]
+ return mirror_urls[0]
# vim: set ts=4 sw=4 et:
diff --git a/mirrors/views.py b/mirrors/views.py
index 55c40c4d..c2736da8 100644
--- a/mirrors/views.py
+++ b/mirrors/views.py
@@ -11,6 +11,7 @@ from django.db.models import Q
from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404, redirect, render
from django.utils.timezone import now
+from django.views.decorators.cache import cache_page
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import condition
from django_countries.data import COUNTRIES
@@ -245,7 +246,10 @@ def status(request, tier=None):
if tier is not None and url.mirror.tier != tier:
continue
# split them into good and bad lists based on delay
- if not url.delay or url.delay > bad_timedelta:
+ if url.completion_pct is None:
+ # skip URLs that have never been checked
+ continue
+ elif not url.delay or url.delay > bad_timedelta:
bad_urls.append(url)
else:
good_urls.append(url)
@@ -275,9 +279,6 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder):
if isinstance(obj, timedelta):
# always returned as integer seconds
return obj.days * 24 * 3600 + obj.seconds
- if hasattr(obj, '__iter__'):
- # mainly for queryset serialization
- return list(obj)
if isinstance(obj, MirrorUrl):
data = {attr: getattr(obj, attr) for attr in self.url_attributes}
country = obj.country
@@ -298,14 +299,15 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder):
if isinstance(obj, MirrorUrl):
data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
cutoff = now() - DEFAULT_CUTOFF
- data['logs'] = obj.logs.filter(
- check_time__gte=cutoff).order_by('check_time')
+ data['logs'] = list(obj.logs.filter(
+ check_time__gte=cutoff).order_by('check_time'))
return data
if isinstance(obj, MirrorLog):
return {attr: getattr(obj, attr) for attr in self.log_attributes}
return super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+@cache_page(67)
@condition(last_modified_func=status_last_modified)
def status_json(request, tier=None):
if tier is not None:
@@ -326,9 +328,6 @@ class LocationJSONEncoder(DjangoJSONEncoder):
'''Base JSONEncoder extended to handle CheckLocation objects.'''
def default(self, obj):
- if hasattr(obj, '__iter__'):
- # mainly for queryset serialization
- return list(obj)
if isinstance(obj, CheckLocation):
return {
'id': obj.pk,
@@ -344,7 +343,7 @@ class LocationJSONEncoder(DjangoJSONEncoder):
def locations_json(request):
data = {}
data['version'] = 1
- data['locations'] = CheckLocation.objects.all().order_by('pk')
+ data['locations'] = list(CheckLocation.objects.all().order_by('pk'))
to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder)
response = HttpResponse(to_json, content_type='application/json')
return response
diff --git a/releng/views.py b/releng/views.py
index af25b966..0fb55b29 100644
--- a/releng/views.py
+++ b/releng/views.py
@@ -247,9 +247,6 @@ class ReleaseJSONEncoder(DjangoJSONEncoder):
'created', 'md5_sum', 'sha1_sum')
def default(self, obj):
- if hasattr(obj, '__iter__'):
- # mainly for queryset serialization
- return list(obj)
if isinstance(obj, Release):
data = {attr: getattr(obj, attr) or None
for attr in self.release_attributes}
@@ -276,7 +273,7 @@ def releases_json(request):
data = {
'version': 1,
- 'releases': releases,
+ 'releases': list(releases),
'latest_version': latest_version,
}
to_json = json.dumps(data, ensure_ascii=False, cls=ReleaseJSONEncoder)
diff --git a/requirements.txt b/requirements.txt
index e2b49aba..5d544e3c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,11 +1,11 @@
-e git+git://github.com/fredj/cssmin.git@master#egg=cssmin
--e git+git://github.com/SmileyChris/django-countries.git@a2c924074dbe2f0b9b3059bf70064aeadf5643ed#egg=django-countries
Django==1.7
IPy==0.81
Jinja2==2.7.3
Markdown==2.4.1
bencode==1.0
django-jinja==1.0.4
+django_countries==3.0
jsmin==2.0.11
pgpdump==1.5
pytz>=2014.7
diff --git a/requirements_prod.txt b/requirements_prod.txt
index 020a5ac4..e3bc0ae2 100644
--- a/requirements_prod.txt
+++ b/requirements_prod.txt
@@ -1,11 +1,11 @@
-e git+git://github.com/fredj/cssmin.git@master#egg=cssmin
--e git+git://github.com/SmileyChris/django-countries.git@a2c924074dbe2f0b9b3059bf70064aeadf5643ed#egg=django-countries
Django==1.7
IPy==0.81
Jinja2==2.7.3
Markdown==2.4.1
bencode==1.0
django-jinja==1.0.4
+django_countries==3.0
jsmin==2.0.11
pgpdump==1.5
psycopg2==2.5.4