diff options
26 files changed, 501 insertions, 456 deletions
diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 8c17c78f..1f16a375 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -238,7 +238,7 @@ class MirrorCheckPool(object): for url in list(urls): self.tasks.put(url) self.threads = [] - for i in range(num_threads): + for _ in range(num_threads): thread = Thread(target=mirror_url_worker, args=(self.tasks, self.logs, location, timeout)) thread.daemon = True diff --git a/mirrors/urls.py b/mirrors/urls.py index b1054380..fc510fbb 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -1,15 +1,18 @@ from django.conf.urls import patterns -urlpatterns = patterns('mirrors.views', - (r'^$', 'mirrors', {}, 'mirror-list'), - (r'^status/$', 'status', {}, 'mirror-status'), - (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), - (r'^status/tier/(?P<tier>\d+)/$', 'status', {}, 'mirror-status-tier'), - (r'^status/tier/(?P<tier>\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), - (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), - (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'), - (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'), - (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', 'url_details'), +from .views import mirrors, status, mirror_details, url_details +from .views.api import status_json, mirror_details_json, locations_json + +urlpatterns = patterns('', + (r'^$', mirrors, {}, 'mirror-list'), + (r'^status/$', status, {}, 'mirror-status'), + (r'^status/json/$', status_json, {}, 'mirror-status-json'), + (r'^status/tier/(?P<tier>\d+)/$', status, {}, 'mirror-status-tier'), + (r'^status/tier/(?P<tier>\d+)/json/$', status_json, {}, 'mirror-status-tier-json'), + (r'^locations/json/$', locations_json, {}, 'mirror-locations-json'), + (r'^(?P<name>[\.\-\w]+)/$', mirror_details), + (r'^(?P<name>[\.\-\w]+)/json/$', mirror_details_json), + (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', url_details), ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index bba54ec9..a64656a9 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -1,7 +1,7 @@ from django.conf.urls import patterns -urlpatterns = patterns('mirrors.views', +urlpatterns = patterns('mirrors.views.mirrorlist', (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), (r'^all/$', 'find_mirrors', {'countries': ['all']}), (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple', diff --git a/mirrors/utils.py b/mirrors/utils.py index 930adb8d..7c2f5d17 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -4,7 +4,6 @@ from django.db import connection from django.db.models import Count, Max, Min from django.utils.dateparse import parse_datetime from django.utils.timezone import now -from django_countries.fields import Country from main.utils import cache_function, database_vendor from .models import MirrorLog, MirrorUrl @@ -159,9 +158,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, - url__mirror__public=True).values( - 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__tier', 'error').annotate( + url__mirror__public=True).values('url__id', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') @@ -172,8 +169,11 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): url__mirror__public=True) errors = list(errors) + to_fetch = [err['url__id'] for err in errors] + urls = MirrorUrl.objects.select_related( + 'mirror', 'protocol').in_bulk(to_fetch) for err in errors: - err['country'] = Country(err['url__country'], flag_url='') + err['url'] = urls[err['url__id']] return errors @@ -183,12 +183,12 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): status data available, it is used to determine a good choice by looking at the last batch of status rows.''' cutoff_time = now() - cutoff - status_data = MirrorLog.objects.filter( + log_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) - if status_data['check_time__max'] is not None: - min_check_time = status_data['check_time__max'] - timedelta(minutes=5) - min_sync_time = status_data['last_sync__max'] - timedelta(minutes=20) + if log_data['check_time__max'] is not None: + min_check_time = log_data['check_time__max'] - timedelta(minutes=5) + min_sync_time = log_data['last_sync__max'] - timedelta(minutes=20) best_logs = MirrorLog.objects.select_related('url').filter( is_success=True, check_time__gte=min_check_time, last_sync__gte=min_sync_time, diff --git a/mirrors/views.py b/mirrors/views.py deleted file mode 100644 index 65fa0123..00000000 --- a/mirrors/views.py +++ /dev/null @@ -1,368 +0,0 @@ -from datetime import timedelta -from itertools import groupby -import json -from operator import attrgetter, itemgetter - -from django import forms -from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple -from django.core.serializers.json import DjangoJSONEncoder -from django.db import connection -from django.db.models import Q -from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404, redirect, render -from django.utils.timezone import now -from django.views.decorators.cache import cache_page -from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import condition -from django_countries import countries -from django_countries.fields import Country - -from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, - CheckLocation) -from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF - - -class MirrorlistForm(forms.Form): - country = forms.MultipleChoiceField(required=False, - widget=SelectMultiple(attrs={'size': '12'})) - protocol = forms.MultipleChoiceField(required=False, - widget=CheckboxSelectMultiple) - ip_version = forms.MultipleChoiceField(required=False, - label="IP version", choices=(('4','IPv4'), ('6','IPv6')), - widget=CheckboxSelectMultiple) - use_mirror_status = forms.BooleanField(required=False) - - def __init__(self, *args, **kwargs): - super(MirrorlistForm, self).__init__(*args, **kwargs) - fields = self.fields - fields['country'].choices = [('all','All')] + self.get_countries() - fields['country'].initial = ['all'] - protos = [(p.protocol, p.protocol) for p in - MirrorProtocol.objects.filter(is_download=True)] - initial = MirrorProtocol.objects.filter(is_download=True, default=True) - fields['protocol'].choices = protos - fields['protocol'].initial = [p.protocol for p in initial] - fields['ip_version'].initial = ['4'] - - def get_countries(self): - country_codes = set() - country_codes.update(MirrorUrl.objects.filter(active=True, - mirror__active=True).exclude(country='').values_list( - 'country', flat=True).order_by().distinct()) - code_list = [(code, countries.name(code)) for code in country_codes] - return sorted(code_list, key=itemgetter(1)) - - def as_div(self): - "Returns this form rendered as HTML <divs>s." - return self._html_output( - normal_row = u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>', - error_row = u'%s', - row_ender = '</div>', - help_text_html = u' <span class="helptext">%s</span>', - errors_on_separate_row = True) - - -@csrf_exempt -def generate_mirrorlist(request): - if request.method == 'POST' or len(request.GET) > 0: - form = MirrorlistForm(data=request.REQUEST) - if form.is_valid(): - countries = form.cleaned_data['country'] - protocols = form.cleaned_data['protocol'] - use_status = form.cleaned_data['use_mirror_status'] - ipv4 = '4' in form.cleaned_data['ip_version'] - ipv6 = '6' in form.cleaned_data['ip_version'] - return find_mirrors(request, countries, protocols, - use_status, ipv4, ipv6) - else: - form = MirrorlistForm() - - return render(request, 'mirrors/mirrorlist_generate.html', - {'mirrorlist_form': form}) - - -def status_filter(original_urls): - status_info = get_mirror_statuses() - scores = {u.id: u.score for u in status_info['urls']} - urls = [] - for u in original_urls: - u.score = scores.get(u.id, None) - # also include mirrors that don't have an up to date score - # (as opposed to those that have been set with no score) - if (u.id not in scores) or (u.score and u.score < 100.0): - urls.append(u) - # if a url doesn't have a score, treat it as the highest possible - return sorted(urls, key=lambda x: x.score or 100.0) - - -def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True): - if not protocols: - protocols = MirrorProtocol.objects.filter(is_download=True) - elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: - # we already have a queryset, no need to query again - pass - else: - protocols = MirrorProtocol.objects.filter(protocol__in=protocols) - qset = MirrorUrl.objects.select_related().filter( - protocol__in=protocols, active=True, - mirror__public=True, mirror__active=True) - if countries and 'all' not in countries: - qset = qset.filter(country__in=countries) - - ip_version = Q() - if ipv4_supported: - ip_version |= Q(has_ipv4=True) - if ipv6_supported: - ip_version |= Q(has_ipv6=True) - qset = qset.filter(ip_version) - - if not use_status: - sort_key = attrgetter('country.name', 'mirror.name', 'url') - urls = sorted(qset, key=sort_key) - template = 'mirrors/mirrorlist.txt' - else: - urls = status_filter(qset) - template = 'mirrors/mirrorlist_status.txt' - - context = { - 'mirror_urls': urls, - } - return render(request, template, context, content_type='text/plain') - - -def find_mirrors_simple(request, protocol): - if protocol == 'smart': - return redirect('mirrorlist_simple', 'http', permanent=True) - proto = get_object_or_404(MirrorProtocol, protocol=protocol) - return find_mirrors(request, protocols=[proto]) - - -def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'name') - protos = MirrorUrl.objects.values_list( - 'mirror_id', 'protocol__protocol').order_by( - 'mirror_id', 'protocol__protocol').distinct() - countries = MirrorUrl.objects.values_list( - 'mirror_id', 'country').order_by( - 'mirror_id', 'country').distinct() - - if not request.user.is_authenticated(): - mirror_list = mirror_list.filter(public=True, active=True) - protos = protos.filter( - mirror__public=True, mirror__active=True, active=True) - countries = countries.filter( - mirror__public=True, mirror__active=True, active=True) - - protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} - countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))} - - for mirror in mirror_list: - item_protos = protos.get(mirror.id, []) - mirror.protocols = [item[1] for item in item_protos] - mirror.country = None - item_countries = countries.get(mirror.id, []) - if len(item_countries) == 1: - mirror.country = Country(item_countries[0][1]) - - return render(request, 'mirrors/mirrors.html', - {'mirror_list': mirror_list}) - - -def mirror_details(request, name): - mirror = get_object_or_404(Mirror, name=name) - authorized = request.user.is_authenticated() - if not authorized and \ - (not mirror.public or not mirror.active): - raise Http404 - error_cutoff = timedelta(days=7) - - status_info = get_mirror_statuses(mirror_id=mirror.id, - show_all=authorized) - checked_urls = {url for url in status_info['urls'] \ - if url.mirror_id == mirror.id} - all_urls = mirror.urls.select_related('protocol') - if not authorized: - all_urls = all_urls.filter(active=True) - all_urls = set(all_urls) - # Add dummy data for URLs that we haven't checked recently - other_urls = all_urls.difference(checked_urls) - for url in other_urls: - for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', - 'duration_stddev', 'score'): - setattr(url, attr, None) - all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) - - error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, - show_all=True) - - context = { - 'mirror': mirror, - 'urls': all_urls, - 'cutoff': error_cutoff, - 'error_logs': error_logs, - } - return render(request, 'mirrors/mirror_details.html', context) - - -def mirror_details_json(request, name): - authorized = request.user.is_authenticated() - mirror = get_object_or_404(Mirror, name=name) - if not authorized and (not mirror.public or not mirror.active): - raise Http404 - status_info = get_mirror_statuses(mirror_id=mirror.id, - show_all=authorized) - data = status_info.copy() - data['version'] = 3 - to_json = json.dumps(data, ensure_ascii=False, - cls=ExtendedMirrorStatusJSONEncoder) - response = HttpResponse(to_json, content_type='application/json') - return response - - -def url_details(request, name, url_id): - url = get_object_or_404(MirrorUrl.objects.select_related(), - id=url_id, mirror__name=name) - mirror = url.mirror - authorized = request.user.is_authenticated() - if not authorized and \ - (not mirror.public or not mirror.active or not url.active): - raise Http404 - error_cutoff = timedelta(days=7) - cutoff_time = now() - error_cutoff - logs = MirrorLog.objects.select_related('location').filter( - url=url, check_time__gte=cutoff_time).order_by('-check_time') - - context = { - 'url': url, - 'logs': logs, - } - return render(request, 'mirrors/url_details.html', context) - - -def status_last_modified(request, *args, **kwargs): - cursor = connection.cursor() - cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog") - return cursor.fetchone()[0] - - -@condition(last_modified_func=status_last_modified) -def status(request, tier=None): - if tier is not None: - tier = int(tier) - if tier not in [t[0] for t in Mirror.TIER_CHOICES]: - raise Http404 - bad_timedelta = timedelta(days=3) - status_info = get_mirror_statuses() - - urls = status_info['urls'] - good_urls = [] - bad_urls = [] - for url in urls: - # screen by tier if we were asked to - if tier is not None and url.mirror.tier != tier: - continue - # split them into good and bad lists based on delay - if url.completion_pct is None: - # skip URLs that have never been checked - continue - elif not url.delay or url.delay > bad_timedelta: - bad_urls.append(url) - else: - good_urls.append(url) - - error_logs = get_mirror_errors() - if tier is not None: - error_logs = [log for log in error_logs - if log['url__mirror__tier'] == tier] - - context = status_info.copy() - context.update({ - 'good_urls': sorted(good_urls, key=attrgetter('score')), - 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': error_logs, - 'tier': tier, - }) - return render(request, 'mirrors/status.html', context) - - -class MirrorStatusJSONEncoder(DjangoJSONEncoder): - '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl - serialization. The base class takes care of datetime.datetime types.''' - url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', - 'delay', 'duration_avg', 'duration_stddev', 'score') - - def default(self, obj): - if isinstance(obj, timedelta): - # always returned as integer seconds - return obj.days * 24 * 3600 + obj.seconds - if isinstance(obj, MirrorUrl): - data = {attr: getattr(obj, attr) for attr in self.url_attributes} - country = obj.country - data['country'] = unicode(country.name) - data['country_code'] = country.code - return data - if isinstance(obj, MirrorProtocol): - return unicode(obj) - return super(MirrorStatusJSONEncoder, self).default(obj) - - -class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): - '''Adds URL check history information.''' - log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', - 'location_id') - - def default(self, obj): - if isinstance(obj, MirrorUrl): - data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - cutoff = now() - DEFAULT_CUTOFF - data['logs'] = list(obj.logs.filter( - check_time__gte=cutoff).order_by('check_time')) - return data - if isinstance(obj, MirrorLog): - return {attr: getattr(obj, attr) for attr in self.log_attributes} - return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - - -@cache_page(67) -@condition(last_modified_func=status_last_modified) -def status_json(request, tier=None): - if tier is not None: - tier = int(tier) - if tier not in [t[0] for t in Mirror.TIER_CHOICES]: - raise Http404 - status_info = get_mirror_statuses() - data = status_info.copy() - if tier is not None: - data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] - data['version'] = 3 - to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) - response = HttpResponse(to_json, content_type='application/json') - return response - - -class LocationJSONEncoder(DjangoJSONEncoder): - '''Base JSONEncoder extended to handle CheckLocation objects.''' - - def default(self, obj): - if isinstance(obj, CheckLocation): - return { - 'id': obj.pk, - 'hostname': obj.hostname, - 'source_ip': obj.source_ip, - 'country': unicode(obj.country.name), - 'country_code': obj.country.code, - 'ip_version': obj.ip_version, - } - return super(LocationJSONEncoder, self).default(obj) - - -def locations_json(request): - data = {} - data['version'] = 1 - data['locations'] = list(CheckLocation.objects.all().order_by('pk')) - to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) - response = HttpResponse(to_json, content_type='application/json') - return response - -# vim: set ts=4 sw=4 et: diff --git a/mirrors/views/__init__.py b/mirrors/views/__init__.py new file mode 100644 index 00000000..01e8519d --- /dev/null +++ b/mirrors/views/__init__.py @@ -0,0 +1,149 @@ +from datetime import timedelta +from itertools import groupby +from operator import attrgetter, itemgetter + +from django.db import connection +from django.http import Http404 +from django.shortcuts import get_object_or_404, render +from django.utils.timezone import now +from django.views.decorators.http import condition +from django_countries.fields import Country + +from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) +from ..utils import get_mirror_statuses, get_mirror_errors + + +def mirrors(request): + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror_id', 'protocol__protocol').distinct() + countries = MirrorUrl.objects.values_list( + 'mirror_id', 'country').order_by( + 'mirror_id', 'country').distinct() + + if not request.user.is_authenticated(): + mirror_list = mirror_list.filter(public=True, active=True) + protos = protos.filter( + mirror__public=True, mirror__active=True, active=True) + countries = countries.filter( + mirror__public=True, mirror__active=True, active=True) + + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} + countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))} + + for mirror in mirror_list: + item_protos = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in item_protos] + mirror.country = None + item_countries = countries.get(mirror.id, []) + if len(item_countries) == 1: + mirror.country = Country(item_countries[0][1]) + + return render(request, 'mirrors/mirrors.html', + {'mirror_list': mirror_list}) + + +def mirror_details(request, name): + mirror = get_object_or_404(Mirror, name=name) + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active): + raise Http404 + error_cutoff = timedelta(days=7) + + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = mirror.urls.select_related('protocol') + if not authorized: + all_urls = all_urls.filter(active=True) + all_urls = set(all_urls) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) + + error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, + show_all=True) + + context = { + 'mirror': mirror, + 'urls': all_urls, + 'cutoff': error_cutoff, + 'error_logs': error_logs, + } + return render(request, 'mirrors/mirror_details.html', context) + + +def url_details(request, name, url_id): + url = get_object_or_404(MirrorUrl.objects.select_related(), + id=url_id, mirror__name=name) + mirror = url.mirror + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active or not url.active): + raise Http404 + error_cutoff = timedelta(days=7) + cutoff_time = now() - error_cutoff + logs = MirrorLog.objects.select_related('location').filter( + url=url, check_time__gte=cutoff_time).order_by('-check_time') + + context = { + 'url': url, + 'logs': logs, + } + return render(request, 'mirrors/url_details.html', context) + + +def status_last_modified(request, *args, **kwargs): + cursor = connection.cursor() + cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog") + return cursor.fetchone()[0] + + +@condition(last_modified_func=status_last_modified) +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + bad_timedelta = timedelta(days=3) + status_info = get_mirror_statuses() + + urls = status_info['urls'] + good_urls = [] + bad_urls = [] + for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue + # split them into good and bad lists based on delay + if url.completion_pct is None: + # skip URLs that have never been checked + continue + elif not url.delay or url.delay > bad_timedelta: + bad_urls.append(url) + else: + good_urls.append(url) + + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url'].mirror.tier == tier] + + context = status_info.copy() + context.update({ + 'good_urls': sorted(good_urls, key=attrgetter('score')), + 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), + 'error_logs': error_logs, + 'tier': tier, + }) + return render(request, 'mirrors/status.html', context) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/views/api.py b/mirrors/views/api.py new file mode 100644 index 00000000..b72585e6 --- /dev/null +++ b/mirrors/views/api.py @@ -0,0 +1,108 @@ +from datetime import timedelta +import json + +from django.core.serializers.json import DjangoJSONEncoder +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404 +from django.utils.timezone import now + +from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) +from ..utils import get_mirror_statuses, DEFAULT_CUTOFF + + +class MirrorStatusJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl + serialization. The base class takes care of datetime.datetime types.''' + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') + + def default(self, obj): + if isinstance(obj, timedelta): + # always returned as integer seconds + return obj.days * 24 * 3600 + obj.seconds + if isinstance(obj, MirrorUrl): + data = {attr: getattr(obj, attr) for attr in self.url_attributes} + country = obj.country + data['country'] = unicode(country.name) + data['country_code'] = country.code + return data + if isinstance(obj, MirrorProtocol): + return unicode(obj) + return super(MirrorStatusJSONEncoder, self).default(obj) + + +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - DEFAULT_CUTOFF + data['logs'] = list(obj.logs.filter( + check_time__gte=cutoff).order_by('check_time')) + return data + if isinstance(obj, MirrorLog): + data = {attr: getattr(obj, attr) for attr in self.log_attributes} + data['error'] = obj.error or None + return data + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if isinstance(obj, CheckLocation): + return { + 'id': obj.pk, + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + status_info = get_mirror_statuses() + data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def mirror_details_json(request, name): + authorized = request.user.is_authenticated() + mirror = get_object_or_404(Mirror, name=name) + if not authorized and (not mirror.public or not mirror.active): + raise Http404 + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) + data = status_info.copy() + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = list(CheckLocation.objects.all().order_by('pk')) + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py new file mode 100644 index 00000000..3c68d036 --- /dev/null +++ b/mirrors/views/mirrorlist.py @@ -0,0 +1,129 @@ +from operator import attrgetter, itemgetter + +from django import forms +from django.db.models import Q +from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple +from django.shortcuts import get_object_or_404, redirect, render +from django.views.decorators.csrf import csrf_exempt +from django_countries import countries + +from ..models import MirrorUrl, MirrorProtocol +from ..utils import get_mirror_statuses + + +class MirrorlistForm(forms.Form): + country = forms.MultipleChoiceField(required=False, + widget=SelectMultiple(attrs={'size': '12'})) + protocol = forms.MultipleChoiceField(required=False, + widget=CheckboxSelectMultiple) + ip_version = forms.MultipleChoiceField(required=False, + label="IP version", choices=(('4','IPv4'), ('6','IPv6')), + widget=CheckboxSelectMultiple) + use_mirror_status = forms.BooleanField(required=False) + + def __init__(self, *args, **kwargs): + super(MirrorlistForm, self).__init__(*args, **kwargs) + fields = self.fields + fields['country'].choices = [('all','All')] + self.get_countries() + fields['country'].initial = ['all'] + protos = [(p.protocol, p.protocol) for p in + MirrorProtocol.objects.filter(is_download=True)] + initial = MirrorProtocol.objects.filter(is_download=True, default=True) + fields['protocol'].choices = protos + fields['protocol'].initial = [p.protocol for p in initial] + fields['ip_version'].initial = ['4'] + + def get_countries(self): + country_codes = set() + country_codes.update(MirrorUrl.objects.filter(active=True, + mirror__active=True).exclude(country='').values_list( + 'country', flat=True).order_by().distinct()) + code_list = [(code, countries.name(code)) for code in country_codes] + return sorted(code_list, key=itemgetter(1)) + + def as_div(self): + "Returns this form rendered as HTML <divs>s." + return self._html_output( + normal_row = u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>', + error_row = u'%s', + row_ender = '</div>', + help_text_html = u' <span class="helptext">%s</span>', + errors_on_separate_row = True) + + +@csrf_exempt +def generate_mirrorlist(request): + if request.method == 'POST' or len(request.GET) > 0: + form = MirrorlistForm(data=request.REQUEST) + if form.is_valid(): + countries = form.cleaned_data['country'] + protocols = form.cleaned_data['protocol'] + use_status = form.cleaned_data['use_mirror_status'] + ipv4 = '4' in form.cleaned_data['ip_version'] + ipv6 = '6' in form.cleaned_data['ip_version'] + return find_mirrors(request, countries, protocols, + use_status, ipv4, ipv6) + else: + form = MirrorlistForm() + + return render(request, 'mirrors/mirrorlist_generate.html', + {'mirrorlist_form': form}) + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = {u.id: u.score for u in status_info['urls']} + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + +def find_mirrors(request, countries=None, protocols=None, use_status=False, + ipv4_supported=True, ipv6_supported=True): + if not protocols: + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) + qset = MirrorUrl.objects.select_related().filter( + protocol__in=protocols, active=True, + mirror__public=True, mirror__active=True) + if countries and 'all' not in countries: + qset = qset.filter(country__in=countries) + + ip_version = Q() + if ipv4_supported: + ip_version |= Q(has_ipv4=True) + if ipv6_supported: + ip_version |= Q(has_ipv6=True) + qset = qset.filter(ip_version) + + if not use_status: + sort_key = attrgetter('country.name', 'mirror.name', 'url') + urls = sorted(qset, key=sort_key) + template = 'mirrors/mirrorlist.txt' + else: + urls = status_filter(qset) + template = 'mirrors/mirrorlist_status.txt' + + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') + + +def find_mirrors_simple(request, protocol): + if protocol == 'smart': + return redirect('mirrorlist_simple', 'http', permanent=True) + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) + +# vim: set ts=4 sw=4 et: diff --git a/newrelic.ini b/newrelic.ini index 98cca5b6..72158dc4 100644 --- a/newrelic.ini +++ b/newrelic.ini @@ -54,7 +54,7 @@ monitor_mode = true # write out a log file, it is also possible to say "stderr" and # output to standard error output. This would normally result in # output appearing in your web server log. -log_file = /tmp/newrelic-python-agent.log +#log_file = /tmp/newrelic-python-agent.log # Sets the level of detail of messages sent to the log file, if # a log file location has been provided. Possible values, in @@ -68,14 +68,27 @@ log_file = /tmp/newrelic-python-agent.log log_level = info # The Python Agent communicates with the New Relic service using -# HTTP by default. If you want to communicate via HTTPS to -# increase security, then turn on SSL by setting this value to -# true. Note, this will result in increased CPU overhead to -# perform the encryption involved in SSL communication, but this -# work is done asynchronously to the threads that process your -# application code, so it should not impact response times. +# SSL by default. Note that this does result in an increase in +# CPU overhead, over and above what would occur for a non SSL +# connection, to perform the encryption involved in the SSL +# communication. This work is though done in a distinct thread +# to those handling your web requests, so it should not impact +# response times. You can if you wish revert to using a non SSL +# connection, but this will result in information being sent +# over a plain socket connection and will not be as secure. ssl = true +# High Security Mode enforces certain security settings, and +# prevents them from being overridden, so that no sensitive data +# is sent to New Relic. Enabling High Security Mode means that +# SSL is turned on, request parameters are not collected, and SQL +# can not be sent to New Relic in its raw form. To activate High +# Security Mode, it must be set to 'true' in this local .ini +# configuration file AND be set to 'true' in the server-side +# configuration in the New Relic user interface. For details, see +# https://docs.newrelic.com/docs/subscriptions/high-security +high_security = false + # The Python Agent will attempt to connect directly to the New # Relic service. If there is an intermediate firewall between # your host and the New Relic service that requires you to use a @@ -83,7 +96,12 @@ ssl = true # "proxy_port" settings to the required values for the HTTP # proxy. The "proxy_user" and "proxy_pass" settings should # additionally be set if proxy authentication is implemented by -# the HTTP proxy. +# the HTTP proxy. The "proxy_scheme" setting dictates what +# protocol scheme is used in talking to the HTTP proxy. This +# would normally always be set as "http" which will result in the +# agent then using a SSL tunnel through the HTTP proxy for end to +# end encryption. +# proxy_scheme = http # proxy_host = hostname # proxy_port = 8080 # proxy_user = @@ -153,13 +171,13 @@ error_collector.enabled = true # To stop specific errors from reporting to the UI, set this to # a space separated list of the Python exception type names to # ignore. The exception name should be of the form 'module:class'. -error_collector.ignore_errors = django.http.response:Http404 +error_collector.ignore_errors = # Browser monitoring is the Real User Monitoring feature of the UI. # For those Python web frameworks that are supported, this # setting enables the auto-insertion of the browser monitoring # JavaScript fragments. -browser_monitoring.auto_instrument = true +browser_monitoring.auto_instrument = false # A thread profiling session can be scheduled via the UI when # this option is enabled. The thread profiler will periodically diff --git a/packages/views/search.py b/packages/views/search.py index e4cd0423..6e892251 100644 --- a/packages/views/search.py +++ b/packages/views/search.py @@ -42,11 +42,13 @@ class PackageSearchForm(forms.Form): people = User.objects.filter( is_active=True, userprofile__id__in=profile_ids).order_by( 'first_name', 'last_name') - people = [('', 'All'), ('orphan', 'Orphan')] + \ + maintainers = [('', 'All'), ('orphan', 'Orphan')] + \ + [(p.username, p.get_full_name()) for p in people] + packagers = [('', 'All'), ('unknown', 'Unknown')] + \ [(p.username, p.get_full_name()) for p in people] - self.fields['maintainer'].choices = people - self.fields['packager'].choices = people + self.fields['maintainer'].choices = maintainers + self.fields['packager'].choices = packagers def exact_matches(self): # only do exact match search if 'q' is sole parameter diff --git a/requirements.txt b/requirements.txt index 54cf90f0..7b8c4822 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.3 +Django==1.7.7 IPy==0.81 Jinja2==2.7.3 -Markdown==2.5.2 +Markdown==2.6.1 MarkupSafe==0.23 bencode==1.0 -django-jinja==1.0.5 -django_countries==3.0.2 -jsmin==2.1.0 +django-countries==3.3 +django-jinja==1.3.2 +jsmin==2.1.1 pgpdump==1.5 -pytz>=2014.10 +pytz>=2015.2 diff --git a/requirements_prod.txt b/requirements_prod.txt index ac528ef9..3396ebdb 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,15 +1,15 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.3 +Django==1.7.7 IPy==0.81 Jinja2==2.7.3 -Markdown==2.5.2 +Markdown==2.6.1 MarkupSafe==0.23 bencode==1.0 -django-jinja==1.0.5 -django_countries==3.0.2 -jsmin==2.1.0 +django-countries==3.3 +django-jinja==1.3.2 +jsmin==2.1.1 pgpdump==1.5 -psycopg2==2.5.4 +psycopg2==2.6 pyinotify==0.9.5 -python-memcached==1.53 -pytz>=2014.10 +python-memcached==1.54 +pytz>=2015.2 diff --git a/settings.py b/settings.py index 3d0319c6..b3b27323 100644 --- a/settings.py +++ b/settings.py @@ -152,11 +152,10 @@ LOGGING = { 'handlers': ['mail_admins'], 'level': 'ERROR', 'propagate': True, - }, + } }, } - ## Server used for linking to PGP keysearch results PGP_SERVER = 'pgp.mit.edu:11371' @@ -198,9 +197,10 @@ BRANDING_WIKINAME = 'ArchWiki' BRANDING_EMAIL = 'Arch Website Notification <nobody@archlinux.org>' BRANDING_OSEARCH_TAGS = 'linux archlinux package software' -# Country name overrides for display purposes +# Shorten some names just a bit COUNTRIES_OVERRIDE = { - 'MK': 'Macedonia', + 'GB': 'United Kingdom', + 'US': 'United States', } ## Import local settings diff --git a/sitemaps.py b/sitemaps.py index 03ad9254..7746ecab 100644 --- a/sitemaps.py +++ b/sitemaps.py @@ -37,12 +37,17 @@ class PackagesSitemap(Sitemap): class PackageFilesSitemap(PackagesSitemap): changefreq = "weekly" priority = "0.1" + # we fixed a bug on the package files page on this day, force modification + lastmod_min = datetime(2015, 4, 12).replace(tzinfo=utc) def location(self, obj): return PackagesSitemap.location(self, obj) + 'files/' def lastmod(self, obj): - return obj.files_last_update + update = obj.files_last_update + if update is None: + return None + return max(update, self.lastmod_min) class PackageGroupsSitemap(Sitemap): diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index cd46e4c0..cc3b9742 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -401,6 +401,10 @@ ul.errorlist { color: red; } +form ul.errorlist { + margin: 0.5em 0; +} + /* JS sorting via tablesorter */ table th.tablesorter-header { padding-right: 20px; diff --git a/templates/mirrors/error_table.html.jinja b/templates/mirrors/error_table.html.jinja index 52f68135..132aae63 100644 --- a/templates/mirrors/error_table.html.jinja +++ b/templates/mirrors/error_table.html.jinja @@ -7,16 +7,18 @@ <th>Error Message</th> <th>Last Occurred</th> <th>Occurrences (last {{ cutoff|hours }})</th> + <th></th> </tr> </thead> <tbody> {% for log in error_logs %}<tr class="{{ loop.cycle('odd', 'even') }}"> - <td>{{ log.url__url }}</td> - <td>{{ log.url__protocol__protocol }}</td> - <td class="country">{{ country_flag(log.country) }}{{ log.country.name }}</td> + <td>{{ log.url.url }}</td> + <td>{{ log.url.protocol.protocol }}</td> + <td class="country">{{ country_flag(log.url.country) }}{{ log.url.country.name }}</td> <td class="wrap">{{ log.error|linebreaksbr }}</td> <td>{{ log.last_occurred|date('Y-m-d H:i') }}</td> <td>{{ log.error_count }}</td> + <td><a href="{{ log.url.get_absolute_url() }}">details</a></td> </tr>{% endfor %} </tbody> </table> diff --git a/templates/mirrors/url_details_logs.html.jinja b/templates/mirrors/url_details_logs.html.jinja index 58f179d8..51f54931 100644 --- a/templates/mirrors/url_details_logs.html.jinja +++ b/templates/mirrors/url_details_logs.html.jinja @@ -14,8 +14,8 @@ <tbody> {% for log in logs %}<tr class="{{ loop.cycle('odd', 'even') }}"> <td>{{ log.check_time|date('Y-m-d H:i') }}</td> - <td class="country">{{ country_flag(log.location.country) }}{{ log.location.country.name }}</td> - <td>{{ log.location.source_ip }}</td> + <td class="country">{% if log.location %}{{ country_flag(log.location.country) }}{{ log.location.country.name }}{% else %}Unknown{% endif %}</td> + <td>{% if log.location %}{{ log.location.source_ip }}{% else %}Unknown{% endif %}</td> <td>{{ log.last_sync|date('Y-m-d H:i') }}</td> <td>{{ log.delay|duration }}</td> <td>{{ log.duration|floatvalue }}</td> diff --git a/templates/packages/details.html.jinja b/templates/packages/details.html.jinja index af4cbe24..f5b08805 100644 --- a/templates/packages/details.html.jinja +++ b/templates/packages/details.html.jinja @@ -103,7 +103,7 @@ {% endif %} <tr> <th>Description:</th> - <td class="wrap" itemprop="description">{{ pkg.pkgdesc|default("") }}</td> + <td class="wrap" itemprop="description">{{ pkg.pkgdesc|default("", true) }}</td> </tr><tr> <th>Upstream URL:</th> <td>{% if pkg.url %}<a itemprop="url" href="{{ pkg.url }}" @@ -186,7 +186,7 @@ {% if user.is_authenticated() %}{% with flag_request = pkg.flag_request() %}{% if flag_request %}<tr> <th>Last Flag Request:</th> <td class="wrap">From {{ flag_request.who() }} on {{ flag_request.created|date }}:<br/> - <div class="userdata">{{ flag_request.message|linebreaksbr|default("{no message}") }}</div></td> + <div class="userdata">{{ flag_request.message|linebreaksbr|default("{no message}", true) }}</div></td> </tr>{% endif %}{% endwith %}{% endif %} </table> </div> diff --git a/templates/packages/details_depend.html.jinja b/templates/packages/details_depend.html.jinja index 404793b6..a2d3a010 100644 --- a/templates/packages/details_depend.html.jinja +++ b/templates/packages/details_depend.html.jinja @@ -1,8 +1,8 @@ {% import 'packages/details_link.html.jinja' as details %}<li>{% if depend.pkg == None %} -{% if depend.providers %}{{ depend.dep.name }}{{ depend.dep.comparison|default("") }}{{ depend.dep.version|default("") }} <span class="virtual-dep">({% for pkg in depend.providers %}{{ details.details_link(pkg) }}{% if not loop.last %}, {% endif %}{% endfor %})</span> -{% else %}{{ depend.dep.name }}{{ depend.dep.comparison|default("") }}{{ depend.dep.version|default("") }} <span class="virtual-dep">(virtual)</span> +{% if depend.providers %}{{ depend.dep.name }}{{ depend.dep.comparison|default("", true) }}{{ depend.dep.version|default("", true) }} <span class="virtual-dep">({% for pkg in depend.providers %}{{ details.details_link(pkg) }}{% if not loop.last %}, {% endif %}{% endfor %})</span> +{% else %}{{ depend.dep.name }}{{ depend.dep.comparison|default("", true) }}{{ depend.dep.version|default("", true) }} <span class="virtual-dep">(virtual)</span> {% endif %}{% else %} -{{ details.details_link(depend.pkg) }}{{ depend.dep.comparison|default("") }}{{ depend.dep.version|default("") }} +{{ details.details_link(depend.pkg) }}{{ depend.dep.comparison|default("", true) }}{{ depend.dep.version|default("", true) }} {% if depend.pkg.repo.testing %} <span class="testing-dep"> (testing)</span> {% endif %}{% if depend.pkg.repo.staging %} <span class="staging-dep"> (staging)</span> {% endif %}{% endif %} diff --git a/templates/packages/details_relatedto.html.jinja b/templates/packages/details_relatedto.html.jinja index 955fdd37..818224de 100644 --- a/templates/packages/details_relatedto.html.jinja +++ b/templates/packages/details_relatedto.html.jinja @@ -1,3 +1,3 @@ {% import 'packages/details_link.html.jinja' as details %}{% for related in all_related %}{% with best_satisfier = related.get_best_satisfier() %} -<span class="related">{% if best_satisfier == None %}{{ related.name }}{% else %}{{ details.details_link(best_satisfier) }}{% endif %}{{ related.comparison|default("") }}{{ related.version|default("") }}{% if not loop.last %}, {% endif %}</span> +<span class="related">{% if best_satisfier == None %}{{ related.name }}{% else %}{{ details.details_link(best_satisfier) }}{% endif %}{{ related.comparison|default('', true) }}{{ related.version|default('', true) }}{% if not loop.last %}, {% endif %}</span> {% endwith %}{% endfor %} diff --git a/templates/packages/files_list.html.jinja b/templates/packages/files_list.html.jinja index c8fc3b1a..ab3e1210 100644 --- a/templates/packages/files_list.html.jinja +++ b/templates/packages/files_list.html.jinja @@ -6,7 +6,7 @@ of the package; it may be out of date.</p> {% if files|length %} <ul> {% for file in files %} -<li class="{% if file.is_directory %}d{% else %}f{% endif %}">{{ file.directory }}{{ file.filename|default('') }}</li>{% endfor %} +<li class="{% if file.is_directory %}d{% else %}f{% endif %}">{{ file.directory }}{{ file.filename|default('', true) }}</li>{% endfor %} </ul> {% else %} <p class="message">Package has no files.</p> diff --git a/templates/public/art.html b/templates/public/art.html index 89f1f81e..40fd9bad 100644 --- a/templates/public/art.html +++ b/templates/public/art.html @@ -68,14 +68,14 @@ <li><strong>archlinux-themes-slim</strong> - SLiM login themes</li> </ul> - <p>Alternatively, you can <a href="ftp://ftp.archlinux.org/other/artwork/" - title="Browse the FTP archives">download the source files via FTP</a>.</p> + <p>Alternatively, you can <a href="https://sources.archlinux.org/other/artwork/" + title="Browse the artwork source">download the source files</a>.</p> <h3>Former Logos</h3> <p>Arch has gone through a few generations of branding and what follows are some of our past logos. Although these images are no longer used frequently, - they remain subject to license restrictions. Email + they remain subject to license restrictions. Email <strong>trademarks@archlinux.org</strong> with any questions.</p> <h4>Original Ribbon Series</h4> diff --git a/templates/registration/login.html b/templates/registration/login.html index c722527d..ff360de3 100644 --- a/templates/registration/login.html +++ b/templates/registration/login.html @@ -5,21 +5,15 @@ {% block content %} <div id="dev-login" class="box"> - <h2>Developer Login</h2> - {% if form.has_errors %} - <p class="login-error">Your username and password didn't match. Please try again.</p> - {% endif %} - <form id="dev-login-form" method="post">{% csrf_token %} <fieldset> - <legend>Enter login credentials</legend> + <legend>Please enter your credentials to login.</legend> {{ form.as_p }} - <p><label></label> <input type="submit" value="Login" /></p> + <p><label></label><input type="submit" value="Login"/></p> </fieldset> </form> - </div> {% endblock %} diff --git a/templates/registration/logout.html b/templates/registration/logout.html index 71daa2e8..5c296c5d 100644 --- a/templates/registration/logout.html +++ b/templates/registration/logout.html @@ -1,11 +1,12 @@ {% extends "base.html" %} + {% block title %}{{ BRANDING_DISTRONAME }} - Logout successful{% endblock %} {% block content %} <div id="dev-logout" class="box"> <h2>Developer Logout</h2> - <p>Logout was successful.<p> + <p>Logout was successful. + <a href='{% url 'login' %}'>Click here to login again.</a></p> </div> {% endblock %} - diff --git a/todolists/views.py b/todolists/views.py index db6f20f0..a0b56e25 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -1,4 +1,5 @@ import json +from operator import attrgetter from django import forms from django.http import HttpResponse @@ -224,8 +225,9 @@ def send_todolist_emails(todo_list, new_packages): maint_packages.setdefault(maint, []).append(todo_package) for maint, packages in maint_packages.iteritems(): + packages = sorted(packages, key=attrgetter('pkgname', 'arch')) ctx = Context({ - 'todo_packages': sorted(packages), + 'todo_packages': packages, 'todolist': todo_list, }) template = loader.get_template('todolists/email_notification.txt') @@ -92,12 +92,10 @@ urlpatterns += patterns('', 'news-sitemap'), ) -# Authentication / Admin +# Authentication urlpatterns += patterns('django.contrib.auth.views', - (r'^login/$', 'login', { - 'template_name': 'registration/login.html'}), - (r'^logout/$', 'logout', { - 'template_name': 'registration/logout.html'}), + (r'^login/$', 'login', {'template_name': 'registration/login.html'}, 'login'), + (r'^logout/$', 'logout', {'template_name': 'registration/logout.html'}, 'logout'), ) # Redirects for older known pages we see in the logs @@ -119,10 +117,8 @@ legacy_urls = ( ('^docs/en/guide/install/arch-install-guide.html', 'https://wiki.archlinux.org/index.php/Installation_guide'), - ('^docs/en/', - 'https://wiki.archlinux.org/'), - ('^docs/', - 'https://wiki.archlinux.org/'), + ('^docs/en/', 'https://wiki.archlinux.org/'), + ('^docs/', 'https://wiki.archlinux.org/'), ) urlpatterns += [url(old_url, RedirectView.as_view(url=new_url)) |