from datetime import timedelta from itertools import groupby import json from operator import attrgetter, itemgetter from django import forms from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db import connection from django.db.models import Q from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.utils.timezone import now from django.views.decorators.cache import cache_page from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import condition from django_countries import countries from django_countries.fields import Country from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, CheckLocation) from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False, widget=SelectMultiple(attrs={'size': '12'})) protocol = forms.MultipleChoiceField(required=False, widget=CheckboxSelectMultiple) ip_version = forms.MultipleChoiceField(required=False, label="IP version", choices=(('4','IPv4'), ('6','IPv6')), widget=CheckboxSelectMultiple) use_mirror_status = forms.BooleanField(required=False) def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) fields = self.fields fields['country'].choices = [('all','All')] + self.get_countries() fields['country'].initial = ['all'] protos = [(p.protocol, p.protocol) for p in MirrorProtocol.objects.filter(is_download=True)] initial = MirrorProtocol.objects.filter(is_download=True, default=True) fields['protocol'].choices = protos fields['protocol'].initial = [p.protocol for p in initial] fields['ip_version'].initial = ['4'] def get_countries(self): country_codes = set() country_codes.update(MirrorUrl.objects.filter(active=True, mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) code_list = [(code, countries.name(code)) for code in country_codes] return sorted(code_list, key=itemgetter(1)) def as_div(self): "Returns this form rendered as HTML s." return self._html_output( normal_row = u'%(label)s %(field)s%(help_text)s', error_row = u'%s', row_ender = '', help_text_html = u' %s', errors_on_separate_row = True) @csrf_exempt def generate_mirrorlist(request): if request.method == 'POST' or len(request.GET) > 0: form = MirrorlistForm(data=request.REQUEST) if form.is_valid(): countries = form.cleaned_data['country'] protocols = form.cleaned_data['protocol'] use_status = form.cleaned_data['use_mirror_status'] ipv4 = '4' in form.cleaned_data['ip_version'] ipv6 = '6' in form.cleaned_data['ip_version'] return find_mirrors(request, countries, protocols, use_status, ipv4, ipv6) else: form = MirrorlistForm() return render(request, 'mirrors/mirrorlist_generate.html', {'mirrorlist_form': form}) def status_filter(original_urls): status_info = get_mirror_statuses() scores = {u.id: u.score for u in status_info['urls']} urls = [] for u in original_urls: u.score = scores.get(u.id, None) # also include mirrors that don't have an up to date score # (as opposed to those that have been set with no score) if (u.id not in scores) or (u.score and u.score < 100.0): urls.append(u) # if a url doesn't have a score, treat it as the highest possible return sorted(urls, key=lambda x: x.score or 100.0) def find_mirrors(request, countries=None, protocols=None, use_status=False, ipv4_supported=True, ipv6_supported=True): if not protocols: protocols = MirrorProtocol.objects.filter(is_download=True) elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: # we already have a queryset, no need to query again pass else: protocols = MirrorProtocol.objects.filter(protocol__in=protocols) qset = MirrorUrl.objects.select_related().filter( protocol__in=protocols, active=True, mirror__public=True, mirror__active=True) if countries and 'all' not in countries: qset = qset.filter(country__in=countries) ip_version = Q() if ipv4_supported: ip_version |= Q(has_ipv4=True) if ipv6_supported: ip_version |= Q(has_ipv6=True) qset = qset.filter(ip_version) if not use_status: sort_key = attrgetter('country.name', 'mirror.name', 'url') urls = sorted(qset, key=sort_key) template = 'mirrors/mirrorlist.txt' else: urls = status_filter(qset) template = 'mirrors/mirrorlist_status.txt' context = { 'mirror_urls': urls, } return render(request, template, context, content_type='text/plain') def find_mirrors_simple(request, protocol): if protocol == 'smart': return redirect('mirrorlist_simple', 'http', permanent=True) proto = get_object_or_404(MirrorProtocol, protocol=protocol) return find_mirrors(request, protocols=[proto]) def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'name') protos = MirrorUrl.objects.values_list( 'mirror_id', 'protocol__protocol').order_by( 'mirror_id', 'protocol__protocol').distinct() countries = MirrorUrl.objects.values_list( 'mirror_id', 'country').order_by( 'mirror_id', 'country').distinct() if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) protos = protos.filter( mirror__public=True, mirror__active=True, active=True) countries = countries.filter( mirror__public=True, mirror__active=True, active=True) protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))} for mirror in mirror_list: item_protos = protos.get(mirror.id, []) mirror.protocols = [item[1] for item in item_protos] mirror.country = None item_countries = countries.get(mirror.id, []) if len(item_countries) == 1: mirror.country = Country(item_countries[0][1]) return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) def mirror_details(request, name): mirror = get_object_or_404(Mirror, name=name) authorized = request.user.is_authenticated() if not authorized and \ (not mirror.public or not mirror.active): raise Http404 error_cutoff = timedelta(days=7) status_info = get_mirror_statuses(mirror_id=mirror.id, show_all=authorized) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} all_urls = mirror.urls.select_related('protocol') if not authorized: all_urls = all_urls.filter(active=True) all_urls = set(all_urls) # Add dummy data for URLs that we haven't checked recently other_urls = all_urls.difference(checked_urls) for url in other_urls: for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', 'duration_stddev', 'score'): setattr(url, attr, None) all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, show_all=True) context = { 'mirror': mirror, 'urls': all_urls, 'cutoff': error_cutoff, 'error_logs': error_logs, } return render(request, 'mirrors/mirror_details.html', context) def mirror_details_json(request, name): authorized = request.user.is_authenticated() mirror = get_object_or_404(Mirror, name=name) if not authorized and (not mirror.public or not mirror.active): raise Http404 status_info = get_mirror_statuses(mirror_id=mirror.id, show_all=authorized) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=ExtendedMirrorStatusJSONEncoder) response = HttpResponse(to_json, content_type='application/json') return response def url_details(request, name, url_id): url = get_object_or_404(MirrorUrl.objects.select_related(), id=url_id, mirror__name=name) mirror = url.mirror authorized = request.user.is_authenticated() if not authorized and \ (not mirror.public or not mirror.active or not url.active): raise Http404 error_cutoff = timedelta(days=7) cutoff_time = now() - error_cutoff logs = MirrorLog.objects.select_related('location').filter( url=url, check_time__gte=cutoff_time).order_by('-check_time') context = { 'url': url, 'logs': logs, } return render(request, 'mirrors/url_details.html', context) def status_last_modified(request, *args, **kwargs): cursor = connection.cursor() cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog") return cursor.fetchone()[0] @condition(last_modified_func=status_last_modified) def status(request, tier=None): if tier is not None: tier = int(tier) if tier not in [t[0] for t in Mirror.TIER_CHOICES]: raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() urls = status_info['urls'] good_urls = [] bad_urls = [] for url in urls: # screen by tier if we were asked to if tier is not None and url.mirror.tier != tier: continue # split them into good and bad lists based on delay if url.completion_pct is None: # skip URLs that have never been checked continue elif not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) error_logs = get_mirror_errors() if tier is not None: error_logs = [log for log in error_logs if log['url__mirror__tier'] == tier] context = status_info.copy() context.update({ 'good_urls': sorted(good_urls, key=attrgetter('score')), 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), 'error_logs': error_logs, 'tier': tier, }) return render(request, 'mirrors/status.html', context) class MirrorStatusJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl serialization. The base class takes care of datetime.datetime types.''' url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', 'delay', 'duration_avg', 'duration_stddev', 'score') def default(self, obj): if isinstance(obj, timedelta): # always returned as integer seconds return obj.days * 24 * 3600 + obj.seconds if isinstance(obj, MirrorUrl): data = {attr: getattr(obj, attr) for attr in self.url_attributes} country = obj.country data['country'] = unicode(country.name) data['country_code'] = country.code return data if isinstance(obj, MirrorProtocol): return unicode(obj) return super(MirrorStatusJSONEncoder, self).default(obj) class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): '''Adds URL check history information.''' log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', 'location_id') def default(self, obj): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) cutoff = now() - DEFAULT_CUTOFF data['logs'] = list(obj.logs.filter( check_time__gte=cutoff).order_by('check_time')) return data if isinstance(obj, MirrorLog): return {attr: getattr(obj, attr) for attr in self.log_attributes} return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) @cache_page(67) @condition(last_modified_func=status_last_modified) def status_json(request, tier=None): if tier is not None: tier = int(tier) if tier not in [t[0] for t in Mirror.TIER_CHOICES]: raise Http404 status_info = get_mirror_statuses() data = status_info.copy() if tier is not None: data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) response = HttpResponse(to_json, content_type='application/json') return response class LocationJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle CheckLocation objects.''' def default(self, obj): if isinstance(obj, CheckLocation): return { 'id': obj.pk, 'hostname': obj.hostname, 'source_ip': obj.source_ip, 'country': unicode(obj.country.name), 'country_code': obj.country.code, 'ip_version': obj.ip_version, } return super(LocationJSONEncoder, self).default(obj) def locations_json(request): data = {} data['version'] = 1 data['locations'] = list(CheckLocation.objects.all().order_by('pk')) to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) response = HttpResponse(to_json, content_type='application/json') return response # vim: set ts=4 sw=4 et: