summaryrefslogtreecommitdiff
path: root/mirrors
diff options
context:
space:
mode:
authorNicolás Reynolds <fauno@kiwwwi.com.ar>2010-12-31 15:42:55 -0300
committerNicolás Reynolds <fauno@kiwwwi.com.ar>2010-12-31 15:42:55 -0300
commitc9178286885d60741eb0e696c3e345e3b0dfb9e2 (patch)
tree7738211d84337e897acfd2263be0b33352becae0 /mirrors
parentebcd492d39cd0525b768a48558e50041aa0961e3 (diff)
parent84897adcdc698678f2ee8d2292cfe4271c6335e4 (diff)
Merge git://projects.archlinux.org/archweb
Conflicts: templates/public/index.html urls.py
Diffstat (limited to 'mirrors')
-rw-r--r--mirrors/management/commands/mirrorcheck.py36
-rw-r--r--mirrors/models.py2
-rw-r--r--mirrors/utils.py2
-rw-r--r--mirrors/views.py39
4 files changed, 61 insertions, 18 deletions
diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py
index bab79219..d7aad4e2 100644
--- a/mirrors/management/commands/mirrorcheck.py
+++ b/mirrors/management/commands/mirrorcheck.py
@@ -10,7 +10,6 @@ Usage: ./manage.py mirrorcheck
"""
from django.core.management.base import NoArgsCommand
-from django.db.models import Q
from datetime import datetime, timedelta
import logging
@@ -51,10 +50,10 @@ class Command(NoArgsCommand):
return check_current_mirrors()
-def parse_rfc3339_datetime(time):
+def parse_rfc3339_datetime(time_string):
# '2010-09-02 11:05:06+02:00'
m = re.match('^(\d{4})-(\d{2})-(\d{2}) '
- '(\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time)
+ '(\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time_string)
if m:
vals = m.groups()
parsed = datetime(int(vals[0]), int(vals[1]), int(vals[2]),
@@ -123,46 +122,55 @@ def check_mirror_url(mirror_url):
log.error = "Connection timed out."
logger.debug("failed: %s, %s" % (url, log.error))
- log.save()
return log
-def mirror_url_worker(queue):
+def mirror_url_worker(work, output):
while True:
try:
- item = queue.get(block=False)
+ item = work.get(block=False)
try:
- check_mirror_url(item)
+ log = check_mirror_url(item)
+ output.put(log)
finally:
- queue.task_done()
+ work.task_done()
except Empty:
return 0
class MirrorCheckPool(object):
def __init__(self, work, num_threads=10):
self.tasks = Queue()
- for i in work:
+ self.logs = Queue()
+ for i in list(work):
self.tasks.put(i)
self.threads = []
for i in range(num_threads):
- thread = Thread(target=mirror_url_worker, args=(self.tasks,))
+ thread = Thread(target=mirror_url_worker,
+ args=(self.tasks, self.logs))
thread.daemon = True
self.threads.append(thread)
- def run_and_join(self):
+ def run(self):
logger.debug("starting threads")
for t in self.threads:
t.start()
logger.debug("joining on all threads")
self.tasks.join()
+ logger.debug("processing log entries")
+ try:
+ while True:
+ log = self.logs.get(block=False)
+ log.save()
+ self.logs.task_done()
+ except Empty:
+ logger.debug("all log items saved to database")
def check_current_mirrors():
urls = MirrorUrl.objects.filter(
- Q(protocol__protocol__iexact='HTTP') |
- Q(protocol__protocol__iexact='FTP'),
+ protocol__is_download=True,
mirror__active=True, mirror__public=True)
pool = MirrorCheckPool(urls)
- pool.run_and_join()
+ pool.run()
return 0
# For lack of a better place to put it, here is a query to get latest check
diff --git a/mirrors/models.py b/mirrors/models.py
index f1286026..e070b1cd 100644
--- a/mirrors/models.py
+++ b/mirrors/models.py
@@ -80,7 +80,7 @@ class MirrorLog(models.Model):
error = models.CharField(max_length=255, blank=True, default='')
def __unicode__(self):
- return "Check of %s at %s" % (url.url, check_time)
+ return "Check of %s at %s" % (self.url.url, self.check_time)
class Meta:
verbose_name = 'Mirror Check Log'
diff --git a/mirrors/utils.py b/mirrors/utils.py
index c64d53c5..124b66e6 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -51,7 +51,7 @@ def get_mirror_statuses(cutoff=default_cutoff):
check_frequency = None
for url in urls:
- url.completion_pct = float(url.success_count) / num_checks
+ url.completion_pct = float(url.success_count) / url.check_count
if url.id in delays:
url_delays = delays[url.id]
d = sum(url_delays, datetime.timedelta()) / len(url_delays)
diff --git a/mirrors/views.py b/mirrors/views.py
index fb7d3361..a2b94de8 100644
--- a/mirrors/views.py
+++ b/mirrors/views.py
@@ -1,10 +1,12 @@
from django import forms
+from django.core.serializers.json import DjangoJSONEncoder
from django.db.models import Avg, Count, Max, Min, StdDev
from django.db.models import Q
-from django.http import Http404
+from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404
from django.views.decorators.csrf import csrf_exempt
from django.views.generic.simple import direct_to_template
+from django.utils import simplejson
from main.utils import make_choice
from .models import Mirror, MirrorUrl, MirrorProtocol
@@ -34,7 +36,7 @@ class MirrorlistForm(forms.Form):
@csrf_exempt
def generate_mirrorlist(request):
- if request.REQUEST.get('country', ''):
+ if request.method == 'POST' or len(request.GET) > 0:
form = MirrorlistForm(data=request.REQUEST)
if form.is_valid():
countries = form.cleaned_data['country']
@@ -129,4 +131,37 @@ def status(request):
})
return direct_to_template(request, 'mirrors/status.html', context)
+class MirrorStatusJSONEncoder(DjangoJSONEncoder):
+ '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl
+ serialization. The base class takes care of datetime.datetime types.'''
+ url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct',
+ 'delay', 'duration_avg', 'duration_stddev', 'score']
+
+ def default(self, obj):
+ if isinstance(obj, datetime.timedelta):
+ # always returned as integer seconds
+ return obj.days * 24 * 3600 + obj.seconds
+ if hasattr(obj, '__iter__'):
+ # mainly for queryset serialization
+ return list(obj)
+ if isinstance(obj, MirrorUrl):
+ data = {}
+ for attr in self.url_attributes:
+ data[attr] = getattr(obj, attr)
+ # separate because it isn't on the URL directly
+ data['country'] = obj.mirror.country
+ return data
+ if isinstance(obj, MirrorProtocol):
+ return unicode(obj)
+ return super(MirrorStatusJSONEncoder, self).default(obj)
+
+def status_json(request):
+ status_info = get_mirror_statuses()
+ data = status_info.copy()
+ data['version'] = 1
+ to_json = simplejson.dumps(data, ensure_ascii=False,
+ cls=MirrorStatusJSONEncoder)
+ response = HttpResponse(to_json, mimetype='application/json')
+ return response
+
# vim: set ts=4 sw=4 et: