From 5a31b868715dbd1a4e7fd87f0f9f04e2c8395391 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 13 Jan 2015 18:17:27 -0600 Subject: Slight tweaks to login and logout pages Turns out has_errors hasn't worked since Django pre-1.0, wow. Remove that old code and clean up some other small things while we're in there. Signed-off-by: Dan McGee --- sitestatic/archweb.css | 4 ++++ templates/registration/login.html | 10 ++-------- templates/registration/logout.html | 5 +++-- urls.py | 14 +++++--------- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index cd46e4c0..cc3b9742 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -401,6 +401,10 @@ ul.errorlist { color: red; } +form ul.errorlist { + margin: 0.5em 0; +} + /* JS sorting via tablesorter */ table th.tablesorter-header { padding-right: 20px; diff --git a/templates/registration/login.html b/templates/registration/login.html index 82f74f58..b5894319 100644 --- a/templates/registration/login.html +++ b/templates/registration/login.html @@ -5,21 +5,15 @@ {% block content %}
-

Developer Login

- {% if form.has_errors %} - - {% endif %} -
{% csrf_token %}
- Enter login credentials + Please enter your credentials to login. {{ form.as_p }} -

+

-
{% endblock %} diff --git a/templates/registration/logout.html b/templates/registration/logout.html index 50b3574b..84bbff94 100644 --- a/templates/registration/logout.html +++ b/templates/registration/logout.html @@ -1,11 +1,12 @@ {% extends "base.html" %} + {% block title %}Arch Linux - Logout successful{% endblock %} {% block content %}

Developer Logout

-

Logout was successful.

+

Logout was successful. + Click here to login again.

{% endblock %} - diff --git a/urls.py b/urls.py index 36e3fef1..76271ece 100644 --- a/urls.py +++ b/urls.py @@ -92,12 +92,10 @@ urlpatterns += patterns('', 'news-sitemap'), ) -# Authentication / Admin +# Authentication urlpatterns += patterns('django.contrib.auth.views', - (r'^login/$', 'login', { - 'template_name': 'registration/login.html'}), - (r'^logout/$', 'logout', { - 'template_name': 'registration/logout.html'}), + (r'^login/$', 'login', {'template_name': 'registration/login.html'}, 'login'), + (r'^logout/$', 'logout', {'template_name': 'registration/logout.html'}, 'logout'), ) # Redirects for older known pages we see in the logs @@ -119,10 +117,8 @@ legacy_urls = ( ('^docs/en/guide/install/arch-install-guide.html', 'https://wiki.archlinux.org/index.php/Installation_guide'), - ('^docs/en/', - 'https://wiki.archlinux.org/'), - ('^docs/', - 'https://wiki.archlinux.org/'), + ('^docs/en/', 'https://wiki.archlinux.org/'), + ('^docs/', 'https://wiki.archlinux.org/'), ) urlpatterns += [url(old_url, RedirectView.as_view(url=new_url)) -- cgit v1.2.3 From 047dc3d6348d0d30bdd828a9cc2a29e0d5946eca Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 19 Jan 2015 09:01:45 -0600 Subject: Update to latest django-countries Signed-off-by: Dan McGee --- requirements.txt | 2 +- requirements_prod.txt | 2 +- settings.py | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 54cf90f0..72e8d080 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Markdown==2.5.2 MarkupSafe==0.23 bencode==1.0 django-jinja==1.0.5 -django_countries==3.0.2 +django_countries==3.1.1 jsmin==2.1.0 pgpdump==1.5 pytz>=2014.10 diff --git a/requirements_prod.txt b/requirements_prod.txt index ac528ef9..42d82ceb 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -6,7 +6,7 @@ Markdown==2.5.2 MarkupSafe==0.23 bencode==1.0 django-jinja==1.0.5 -django_countries==3.0.2 +django_countries==3.1.1 jsmin==2.1.0 pgpdump==1.5 psycopg2==2.5.4 diff --git a/settings.py b/settings.py index 60fe71bc..3f0f8e69 100644 --- a/settings.py +++ b/settings.py @@ -155,7 +155,6 @@ LOGGING = { }, } - ## Server used for linking to PGP keysearch results PGP_SERVER = 'pgp.mit.edu:11371' @@ -175,11 +174,6 @@ TORRENT_TRACKERS = ( 'http://tracker.archlinux.org:6969/announce', ) -# Country name overrides for display purposes -COUNTRIES_OVERRIDE = { - 'MK': 'Macedonia', -} - ## Import local settings from local_settings import * -- cgit v1.2.3 From 94d1c96ac9d3e5787f4e2c118646bcc496ef1146 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 29 Jan 2015 20:16:37 -0600 Subject: Ensure packager=unknown query works as expected I broke this in commit dca00e7aab, whoops! Signed-off-by: Dan McGee --- packages/views/search.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/views/search.py b/packages/views/search.py index e4cd0423..6e892251 100644 --- a/packages/views/search.py +++ b/packages/views/search.py @@ -42,11 +42,13 @@ class PackageSearchForm(forms.Form): people = User.objects.filter( is_active=True, userprofile__id__in=profile_ids).order_by( 'first_name', 'last_name') - people = [('', 'All'), ('orphan', 'Orphan')] + \ + maintainers = [('', 'All'), ('orphan', 'Orphan')] + \ + [(p.username, p.get_full_name()) for p in people] + packagers = [('', 'All'), ('unknown', 'Unknown')] + \ [(p.username, p.get_full_name()) for p in people] - self.fields['maintainer'].choices = people - self.fields['packager'].choices = people + self.fields['maintainer'].choices = maintainers + self.fields['packager'].choices = packagers def exact_matches(self): # only do exact match search if 'q' is sole parameter -- cgit v1.2.3 From a52b06c9c20475d6ba79a64239b17c5b4ffc44fe Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 29 Jan 2015 20:19:29 -0600 Subject: Requirements version updates Signed-off-by: Dan McGee --- requirements.txt | 6 +++--- requirements_prod.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 72e8d080..59e25720 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.3 +Django==1.7.4 IPy==0.81 Jinja2==2.7.3 Markdown==2.5.2 MarkupSafe==0.23 bencode==1.0 -django-jinja==1.0.5 -django_countries==3.1.1 +django-countries==3.1.1 +django-jinja==1.1.0 jsmin==2.1.0 pgpdump==1.5 pytz>=2014.10 diff --git a/requirements_prod.txt b/requirements_prod.txt index 42d82ceb..d6481a2e 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,12 +1,12 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.3 +Django==1.7.4 IPy==0.81 Jinja2==2.7.3 Markdown==2.5.2 MarkupSafe==0.23 bencode==1.0 -django-jinja==1.0.5 -django_countries==3.1.1 +django-countries==3.1.1 +django-jinja==1.1.0 jsmin==2.1.0 pgpdump==1.5 psycopg2==2.5.4 -- cgit v1.2.3 From ba6ae553c21374b1faf401712ac9d717e27ebe7d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 3 Feb 2015 21:22:53 -0600 Subject: Sort packages correctly in the todolist notification emails Just using sorted() here is bogus without a key function; the default sort order is by the return value from id() in Python since we have no __lt__ definition on the Package model. Add an explicit sort key. Signed-off-by: Dan McGee --- todolists/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/todolists/views.py b/todolists/views.py index c781a562..75c3d2d6 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -1,4 +1,5 @@ import json +from operator import attrgetter from django import forms from django.http import HttpResponse @@ -223,8 +224,9 @@ def send_todolist_emails(todo_list, new_packages): maint_packages.setdefault(maint, []).append(todo_package) for maint, packages in maint_packages.iteritems(): + packages = sorted(packages, key=attrgetter('pkgname', 'arch')) ctx = Context({ - 'todo_packages': sorted(packages), + 'todo_packages': packages, 'todolist': todo_list, }) template = loader.get_template('todolists/email_notification.txt') -- cgit v1.2.3 From f3214b4b03cc7cc8c4317aae8c8f3353c3f01049 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 15 Feb 2015 13:09:55 -0600 Subject: Requirements updates for various packages Signed-off-by: Dan McGee --- requirements.txt | 4 ++-- requirements_prod.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 59e25720..fdca7bee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Markdown==2.5.2 MarkupSafe==0.23 bencode==1.0 django-countries==3.1.1 -django-jinja==1.1.0 -jsmin==2.1.0 +django-jinja==1.1.1 +jsmin==2.1.1 pgpdump==1.5 pytz>=2014.10 diff --git a/requirements_prod.txt b/requirements_prod.txt index d6481a2e..edb94707 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -6,10 +6,10 @@ Markdown==2.5.2 MarkupSafe==0.23 bencode==1.0 django-countries==3.1.1 -django-jinja==1.1.0 -jsmin==2.1.0 +django-jinja==1.1.1 +jsmin==2.1.1 pgpdump==1.5 -psycopg2==2.5.4 +psycopg2==2.6 pyinotify==0.9.5 python-memcached==1.53 pytz>=2014.10 -- cgit v1.2.3 From 8afa3cc2df4d3a53eec44b1b977b0b50b096617d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 26 Feb 2015 18:34:21 -0600 Subject: Add details links to third mirror status table Signed-off-by: Dan McGee --- mirrors/utils.py | 9 +++++---- mirrors/views.py | 2 +- templates/mirrors/error_table.html.jinja | 8 +++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/mirrors/utils.py b/mirrors/utils.py index 930adb8d..533cd452 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -159,9 +159,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, - url__mirror__public=True).values( - 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__tier', 'error').annotate( + url__mirror__public=True).values('url__id', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') @@ -172,8 +170,11 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): url__mirror__public=True) errors = list(errors) + to_fetch = [err['url__id'] for err in errors] + urls = MirrorUrl.objects.select_related( + 'mirror', 'protocol').in_bulk(to_fetch) for err in errors: - err['country'] = Country(err['url__country'], flag_url='') + err['url'] = urls[err['url__id']] return errors diff --git a/mirrors/views.py b/mirrors/views.py index 65fa0123..9c01bc22 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -274,7 +274,7 @@ def status(request, tier=None): error_logs = get_mirror_errors() if tier is not None: error_logs = [log for log in error_logs - if log['url__mirror__tier'] == tier] + if log['url'].mirror.tier == tier] context = status_info.copy() context.update({ diff --git a/templates/mirrors/error_table.html.jinja b/templates/mirrors/error_table.html.jinja index 52f68135..132aae63 100644 --- a/templates/mirrors/error_table.html.jinja +++ b/templates/mirrors/error_table.html.jinja @@ -7,16 +7,18 @@ Error Message Last Occurred Occurrences (last {{ cutoff|hours }}) + {% for log in error_logs %} - {{ log.url__url }} - {{ log.url__protocol__protocol }} - {{ country_flag(log.country) }}{{ log.country.name }} + {{ log.url.url }} + {{ log.url.protocol.protocol }} + {{ country_flag(log.url.country) }}{{ log.url.country.name }} {{ log.error|linebreaksbr }} {{ log.last_occurred|date('Y-m-d H:i') }} {{ log.error_count }} + details {% endfor %} -- cgit v1.2.3 From 1c0f41ff1a150f72aa323f1f3ebc8179e3ec9ac6 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 26 Feb 2015 18:40:57 -0600 Subject: Bump some package requirements Signed-off-by: Dan McGee --- requirements.txt | 4 ++-- requirements_prod.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index fdca7bee..8e8dd0fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,10 @@ Django==1.7.4 IPy==0.81 Jinja2==2.7.3 -Markdown==2.5.2 +Markdown==2.6 MarkupSafe==0.23 bencode==1.0 -django-countries==3.1.1 +django-countries==3.2 django-jinja==1.1.1 jsmin==2.1.1 pgpdump==1.5 diff --git a/requirements_prod.txt b/requirements_prod.txt index edb94707..ce655fb5 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -2,10 +2,10 @@ Django==1.7.4 IPy==0.81 Jinja2==2.7.3 -Markdown==2.5.2 +Markdown==2.6 MarkupSafe==0.23 bencode==1.0 -django-countries==3.1.1 +django-countries==3.2 django-jinja==1.1.1 jsmin==2.1.1 pgpdump==1.5 -- cgit v1.2.3 From 17b6ce186c6e793417c73e955bfc01b4f4b96864 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 26 Feb 2015 19:16:44 -0600 Subject: Move mirrors views into subdirectory We'll start splitting these up as we did in packages/. Signed-off-by: Dan McGee --- mirrors/views.py | 368 ---------------------------------------------- mirrors/views/__init__.py | 368 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 368 insertions(+), 368 deletions(-) delete mode 100644 mirrors/views.py create mode 100644 mirrors/views/__init__.py diff --git a/mirrors/views.py b/mirrors/views.py deleted file mode 100644 index 9c01bc22..00000000 --- a/mirrors/views.py +++ /dev/null @@ -1,368 +0,0 @@ -from datetime import timedelta -from itertools import groupby -import json -from operator import attrgetter, itemgetter - -from django import forms -from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple -from django.core.serializers.json import DjangoJSONEncoder -from django.db import connection -from django.db.models import Q -from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404, redirect, render -from django.utils.timezone import now -from django.views.decorators.cache import cache_page -from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import condition -from django_countries import countries -from django_countries.fields import Country - -from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, - CheckLocation) -from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF - - -class MirrorlistForm(forms.Form): - country = forms.MultipleChoiceField(required=False, - widget=SelectMultiple(attrs={'size': '12'})) - protocol = forms.MultipleChoiceField(required=False, - widget=CheckboxSelectMultiple) - ip_version = forms.MultipleChoiceField(required=False, - label="IP version", choices=(('4','IPv4'), ('6','IPv6')), - widget=CheckboxSelectMultiple) - use_mirror_status = forms.BooleanField(required=False) - - def __init__(self, *args, **kwargs): - super(MirrorlistForm, self).__init__(*args, **kwargs) - fields = self.fields - fields['country'].choices = [('all','All')] + self.get_countries() - fields['country'].initial = ['all'] - protos = [(p.protocol, p.protocol) for p in - MirrorProtocol.objects.filter(is_download=True)] - initial = MirrorProtocol.objects.filter(is_download=True, default=True) - fields['protocol'].choices = protos - fields['protocol'].initial = [p.protocol for p in initial] - fields['ip_version'].initial = ['4'] - - def get_countries(self): - country_codes = set() - country_codes.update(MirrorUrl.objects.filter(active=True, - mirror__active=True).exclude(country='').values_list( - 'country', flat=True).order_by().distinct()) - code_list = [(code, countries.name(code)) for code in country_codes] - return sorted(code_list, key=itemgetter(1)) - - def as_div(self): - "Returns this form rendered as HTML s." - return self._html_output( - normal_row = u'%(label)s %(field)s%(help_text)s', - error_row = u'%s', - row_ender = '', - help_text_html = u' %s', - errors_on_separate_row = True) - - -@csrf_exempt -def generate_mirrorlist(request): - if request.method == 'POST' or len(request.GET) > 0: - form = MirrorlistForm(data=request.REQUEST) - if form.is_valid(): - countries = form.cleaned_data['country'] - protocols = form.cleaned_data['protocol'] - use_status = form.cleaned_data['use_mirror_status'] - ipv4 = '4' in form.cleaned_data['ip_version'] - ipv6 = '6' in form.cleaned_data['ip_version'] - return find_mirrors(request, countries, protocols, - use_status, ipv4, ipv6) - else: - form = MirrorlistForm() - - return render(request, 'mirrors/mirrorlist_generate.html', - {'mirrorlist_form': form}) - - -def status_filter(original_urls): - status_info = get_mirror_statuses() - scores = {u.id: u.score for u in status_info['urls']} - urls = [] - for u in original_urls: - u.score = scores.get(u.id, None) - # also include mirrors that don't have an up to date score - # (as opposed to those that have been set with no score) - if (u.id not in scores) or (u.score and u.score < 100.0): - urls.append(u) - # if a url doesn't have a score, treat it as the highest possible - return sorted(urls, key=lambda x: x.score or 100.0) - - -def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True): - if not protocols: - protocols = MirrorProtocol.objects.filter(is_download=True) - elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: - # we already have a queryset, no need to query again - pass - else: - protocols = MirrorProtocol.objects.filter(protocol__in=protocols) - qset = MirrorUrl.objects.select_related().filter( - protocol__in=protocols, active=True, - mirror__public=True, mirror__active=True) - if countries and 'all' not in countries: - qset = qset.filter(country__in=countries) - - ip_version = Q() - if ipv4_supported: - ip_version |= Q(has_ipv4=True) - if ipv6_supported: - ip_version |= Q(has_ipv6=True) - qset = qset.filter(ip_version) - - if not use_status: - sort_key = attrgetter('country.name', 'mirror.name', 'url') - urls = sorted(qset, key=sort_key) - template = 'mirrors/mirrorlist.txt' - else: - urls = status_filter(qset) - template = 'mirrors/mirrorlist_status.txt' - - context = { - 'mirror_urls': urls, - } - return render(request, template, context, content_type='text/plain') - - -def find_mirrors_simple(request, protocol): - if protocol == 'smart': - return redirect('mirrorlist_simple', 'http', permanent=True) - proto = get_object_or_404(MirrorProtocol, protocol=protocol) - return find_mirrors(request, protocols=[proto]) - - -def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'name') - protos = MirrorUrl.objects.values_list( - 'mirror_id', 'protocol__protocol').order_by( - 'mirror_id', 'protocol__protocol').distinct() - countries = MirrorUrl.objects.values_list( - 'mirror_id', 'country').order_by( - 'mirror_id', 'country').distinct() - - if not request.user.is_authenticated(): - mirror_list = mirror_list.filter(public=True, active=True) - protos = protos.filter( - mirror__public=True, mirror__active=True, active=True) - countries = countries.filter( - mirror__public=True, mirror__active=True, active=True) - - protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} - countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))} - - for mirror in mirror_list: - item_protos = protos.get(mirror.id, []) - mirror.protocols = [item[1] for item in item_protos] - mirror.country = None - item_countries = countries.get(mirror.id, []) - if len(item_countries) == 1: - mirror.country = Country(item_countries[0][1]) - - return render(request, 'mirrors/mirrors.html', - {'mirror_list': mirror_list}) - - -def mirror_details(request, name): - mirror = get_object_or_404(Mirror, name=name) - authorized = request.user.is_authenticated() - if not authorized and \ - (not mirror.public or not mirror.active): - raise Http404 - error_cutoff = timedelta(days=7) - - status_info = get_mirror_statuses(mirror_id=mirror.id, - show_all=authorized) - checked_urls = {url for url in status_info['urls'] \ - if url.mirror_id == mirror.id} - all_urls = mirror.urls.select_related('protocol') - if not authorized: - all_urls = all_urls.filter(active=True) - all_urls = set(all_urls) - # Add dummy data for URLs that we haven't checked recently - other_urls = all_urls.difference(checked_urls) - for url in other_urls: - for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', - 'duration_stddev', 'score'): - setattr(url, attr, None) - all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) - - error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, - show_all=True) - - context = { - 'mirror': mirror, - 'urls': all_urls, - 'cutoff': error_cutoff, - 'error_logs': error_logs, - } - return render(request, 'mirrors/mirror_details.html', context) - - -def mirror_details_json(request, name): - authorized = request.user.is_authenticated() - mirror = get_object_or_404(Mirror, name=name) - if not authorized and (not mirror.public or not mirror.active): - raise Http404 - status_info = get_mirror_statuses(mirror_id=mirror.id, - show_all=authorized) - data = status_info.copy() - data['version'] = 3 - to_json = json.dumps(data, ensure_ascii=False, - cls=ExtendedMirrorStatusJSONEncoder) - response = HttpResponse(to_json, content_type='application/json') - return response - - -def url_details(request, name, url_id): - url = get_object_or_404(MirrorUrl.objects.select_related(), - id=url_id, mirror__name=name) - mirror = url.mirror - authorized = request.user.is_authenticated() - if not authorized and \ - (not mirror.public or not mirror.active or not url.active): - raise Http404 - error_cutoff = timedelta(days=7) - cutoff_time = now() - error_cutoff - logs = MirrorLog.objects.select_related('location').filter( - url=url, check_time__gte=cutoff_time).order_by('-check_time') - - context = { - 'url': url, - 'logs': logs, - } - return render(request, 'mirrors/url_details.html', context) - - -def status_last_modified(request, *args, **kwargs): - cursor = connection.cursor() - cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog") - return cursor.fetchone()[0] - - -@condition(last_modified_func=status_last_modified) -def status(request, tier=None): - if tier is not None: - tier = int(tier) - if tier not in [t[0] for t in Mirror.TIER_CHOICES]: - raise Http404 - bad_timedelta = timedelta(days=3) - status_info = get_mirror_statuses() - - urls = status_info['urls'] - good_urls = [] - bad_urls = [] - for url in urls: - # screen by tier if we were asked to - if tier is not None and url.mirror.tier != tier: - continue - # split them into good and bad lists based on delay - if url.completion_pct is None: - # skip URLs that have never been checked - continue - elif not url.delay or url.delay > bad_timedelta: - bad_urls.append(url) - else: - good_urls.append(url) - - error_logs = get_mirror_errors() - if tier is not None: - error_logs = [log for log in error_logs - if log['url'].mirror.tier == tier] - - context = status_info.copy() - context.update({ - 'good_urls': sorted(good_urls, key=attrgetter('score')), - 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': error_logs, - 'tier': tier, - }) - return render(request, 'mirrors/status.html', context) - - -class MirrorStatusJSONEncoder(DjangoJSONEncoder): - '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl - serialization. The base class takes care of datetime.datetime types.''' - url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', - 'delay', 'duration_avg', 'duration_stddev', 'score') - - def default(self, obj): - if isinstance(obj, timedelta): - # always returned as integer seconds - return obj.days * 24 * 3600 + obj.seconds - if isinstance(obj, MirrorUrl): - data = {attr: getattr(obj, attr) for attr in self.url_attributes} - country = obj.country - data['country'] = unicode(country.name) - data['country_code'] = country.code - return data - if isinstance(obj, MirrorProtocol): - return unicode(obj) - return super(MirrorStatusJSONEncoder, self).default(obj) - - -class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): - '''Adds URL check history information.''' - log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', - 'location_id') - - def default(self, obj): - if isinstance(obj, MirrorUrl): - data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - cutoff = now() - DEFAULT_CUTOFF - data['logs'] = list(obj.logs.filter( - check_time__gte=cutoff).order_by('check_time')) - return data - if isinstance(obj, MirrorLog): - return {attr: getattr(obj, attr) for attr in self.log_attributes} - return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - - -@cache_page(67) -@condition(last_modified_func=status_last_modified) -def status_json(request, tier=None): - if tier is not None: - tier = int(tier) - if tier not in [t[0] for t in Mirror.TIER_CHOICES]: - raise Http404 - status_info = get_mirror_statuses() - data = status_info.copy() - if tier is not None: - data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] - data['version'] = 3 - to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) - response = HttpResponse(to_json, content_type='application/json') - return response - - -class LocationJSONEncoder(DjangoJSONEncoder): - '''Base JSONEncoder extended to handle CheckLocation objects.''' - - def default(self, obj): - if isinstance(obj, CheckLocation): - return { - 'id': obj.pk, - 'hostname': obj.hostname, - 'source_ip': obj.source_ip, - 'country': unicode(obj.country.name), - 'country_code': obj.country.code, - 'ip_version': obj.ip_version, - } - return super(LocationJSONEncoder, self).default(obj) - - -def locations_json(request): - data = {} - data['version'] = 1 - data['locations'] = list(CheckLocation.objects.all().order_by('pk')) - to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) - response = HttpResponse(to_json, content_type='application/json') - return response - -# vim: set ts=4 sw=4 et: diff --git a/mirrors/views/__init__.py b/mirrors/views/__init__.py new file mode 100644 index 00000000..54c427b3 --- /dev/null +++ b/mirrors/views/__init__.py @@ -0,0 +1,368 @@ +from datetime import timedelta +from itertools import groupby +import json +from operator import attrgetter, itemgetter + +from django import forms +from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple +from django.core.serializers.json import DjangoJSONEncoder +from django.db import connection +from django.db.models import Q +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.utils.timezone import now +from django.views.decorators.cache import cache_page +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import condition +from django_countries import countries +from django_countries.fields import Country + +from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) +from ..utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF + + +class MirrorlistForm(forms.Form): + country = forms.MultipleChoiceField(required=False, + widget=SelectMultiple(attrs={'size': '12'})) + protocol = forms.MultipleChoiceField(required=False, + widget=CheckboxSelectMultiple) + ip_version = forms.MultipleChoiceField(required=False, + label="IP version", choices=(('4','IPv4'), ('6','IPv6')), + widget=CheckboxSelectMultiple) + use_mirror_status = forms.BooleanField(required=False) + + def __init__(self, *args, **kwargs): + super(MirrorlistForm, self).__init__(*args, **kwargs) + fields = self.fields + fields['country'].choices = [('all','All')] + self.get_countries() + fields['country'].initial = ['all'] + protos = [(p.protocol, p.protocol) for p in + MirrorProtocol.objects.filter(is_download=True)] + initial = MirrorProtocol.objects.filter(is_download=True, default=True) + fields['protocol'].choices = protos + fields['protocol'].initial = [p.protocol for p in initial] + fields['ip_version'].initial = ['4'] + + def get_countries(self): + country_codes = set() + country_codes.update(MirrorUrl.objects.filter(active=True, + mirror__active=True).exclude(country='').values_list( + 'country', flat=True).order_by().distinct()) + code_list = [(code, countries.name(code)) for code in country_codes] + return sorted(code_list, key=itemgetter(1)) + + def as_div(self): + "Returns this form rendered as HTML s." + return self._html_output( + normal_row = u'%(label)s %(field)s%(help_text)s', + error_row = u'%s', + row_ender = '', + help_text_html = u' %s', + errors_on_separate_row = True) + + +@csrf_exempt +def generate_mirrorlist(request): + if request.method == 'POST' or len(request.GET) > 0: + form = MirrorlistForm(data=request.REQUEST) + if form.is_valid(): + countries = form.cleaned_data['country'] + protocols = form.cleaned_data['protocol'] + use_status = form.cleaned_data['use_mirror_status'] + ipv4 = '4' in form.cleaned_data['ip_version'] + ipv6 = '6' in form.cleaned_data['ip_version'] + return find_mirrors(request, countries, protocols, + use_status, ipv4, ipv6) + else: + form = MirrorlistForm() + + return render(request, 'mirrors/mirrorlist_generate.html', + {'mirrorlist_form': form}) + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = {u.id: u.score for u in status_info['urls']} + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + +def find_mirrors(request, countries=None, protocols=None, use_status=False, + ipv4_supported=True, ipv6_supported=True): + if not protocols: + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) + qset = MirrorUrl.objects.select_related().filter( + protocol__in=protocols, active=True, + mirror__public=True, mirror__active=True) + if countries and 'all' not in countries: + qset = qset.filter(country__in=countries) + + ip_version = Q() + if ipv4_supported: + ip_version |= Q(has_ipv4=True) + if ipv6_supported: + ip_version |= Q(has_ipv6=True) + qset = qset.filter(ip_version) + + if not use_status: + sort_key = attrgetter('country.name', 'mirror.name', 'url') + urls = sorted(qset, key=sort_key) + template = 'mirrors/mirrorlist.txt' + else: + urls = status_filter(qset) + template = 'mirrors/mirrorlist_status.txt' + + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') + + +def find_mirrors_simple(request, protocol): + if protocol == 'smart': + return redirect('mirrorlist_simple', 'http', permanent=True) + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) + + +def mirrors(request): + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror_id', 'protocol__protocol').distinct() + countries = MirrorUrl.objects.values_list( + 'mirror_id', 'country').order_by( + 'mirror_id', 'country').distinct() + + if not request.user.is_authenticated(): + mirror_list = mirror_list.filter(public=True, active=True) + protos = protos.filter( + mirror__public=True, mirror__active=True, active=True) + countries = countries.filter( + mirror__public=True, mirror__active=True, active=True) + + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} + countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))} + + for mirror in mirror_list: + item_protos = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in item_protos] + mirror.country = None + item_countries = countries.get(mirror.id, []) + if len(item_countries) == 1: + mirror.country = Country(item_countries[0][1]) + + return render(request, 'mirrors/mirrors.html', + {'mirror_list': mirror_list}) + + +def mirror_details(request, name): + mirror = get_object_or_404(Mirror, name=name) + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active): + raise Http404 + error_cutoff = timedelta(days=7) + + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = mirror.urls.select_related('protocol') + if not authorized: + all_urls = all_urls.filter(active=True) + all_urls = set(all_urls) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) + + error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, + show_all=True) + + context = { + 'mirror': mirror, + 'urls': all_urls, + 'cutoff': error_cutoff, + 'error_logs': error_logs, + } + return render(request, 'mirrors/mirror_details.html', context) + + +def mirror_details_json(request, name): + authorized = request.user.is_authenticated() + mirror = get_object_or_404(Mirror, name=name) + if not authorized and (not mirror.public or not mirror.active): + raise Http404 + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) + data = status_info.copy() + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def url_details(request, name, url_id): + url = get_object_or_404(MirrorUrl.objects.select_related(), + id=url_id, mirror__name=name) + mirror = url.mirror + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active or not url.active): + raise Http404 + error_cutoff = timedelta(days=7) + cutoff_time = now() - error_cutoff + logs = MirrorLog.objects.select_related('location').filter( + url=url, check_time__gte=cutoff_time).order_by('-check_time') + + context = { + 'url': url, + 'logs': logs, + } + return render(request, 'mirrors/url_details.html', context) + + +def status_last_modified(request, *args, **kwargs): + cursor = connection.cursor() + cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog") + return cursor.fetchone()[0] + + +@condition(last_modified_func=status_last_modified) +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + bad_timedelta = timedelta(days=3) + status_info = get_mirror_statuses() + + urls = status_info['urls'] + good_urls = [] + bad_urls = [] + for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue + # split them into good and bad lists based on delay + if url.completion_pct is None: + # skip URLs that have never been checked + continue + elif not url.delay or url.delay > bad_timedelta: + bad_urls.append(url) + else: + good_urls.append(url) + + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url'].mirror.tier == tier] + + context = status_info.copy() + context.update({ + 'good_urls': sorted(good_urls, key=attrgetter('score')), + 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), + 'error_logs': error_logs, + 'tier': tier, + }) + return render(request, 'mirrors/status.html', context) + + +class MirrorStatusJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl + serialization. The base class takes care of datetime.datetime types.''' + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') + + def default(self, obj): + if isinstance(obj, timedelta): + # always returned as integer seconds + return obj.days * 24 * 3600 + obj.seconds + if isinstance(obj, MirrorUrl): + data = {attr: getattr(obj, attr) for attr in self.url_attributes} + country = obj.country + data['country'] = unicode(country.name) + data['country_code'] = country.code + return data + if isinstance(obj, MirrorProtocol): + return unicode(obj) + return super(MirrorStatusJSONEncoder, self).default(obj) + + +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - DEFAULT_CUTOFF + data['logs'] = list(obj.logs.filter( + check_time__gte=cutoff).order_by('check_time')) + return data + if isinstance(obj, MirrorLog): + return {attr: getattr(obj, attr) for attr in self.log_attributes} + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + +@cache_page(67) +@condition(last_modified_func=status_last_modified) +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + status_info = get_mirror_statuses() + data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if isinstance(obj, CheckLocation): + return { + 'id': obj.pk, + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = list(CheckLocation.objects.all().order_by('pk')) + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3 From cd69fd8aee59e09e2b6f01ad63ad7ac0c5c3cd16 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 26 Feb 2015 19:36:49 -0600 Subject: Pylint suggested cleanups Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 2 +- mirrors/utils.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 8c17c78f..1f16a375 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -238,7 +238,7 @@ class MirrorCheckPool(object): for url in list(urls): self.tasks.put(url) self.threads = [] - for i in range(num_threads): + for _ in range(num_threads): thread = Thread(target=mirror_url_worker, args=(self.tasks, self.logs, location, timeout)) thread.daemon = True diff --git a/mirrors/utils.py b/mirrors/utils.py index 533cd452..7c2f5d17 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -4,7 +4,6 @@ from django.db import connection from django.db.models import Count, Max, Min from django.utils.dateparse import parse_datetime from django.utils.timezone import now -from django_countries.fields import Country from main.utils import cache_function, database_vendor from .models import MirrorLog, MirrorUrl @@ -184,12 +183,12 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): status data available, it is used to determine a good choice by looking at the last batch of status rows.''' cutoff_time = now() - cutoff - status_data = MirrorLog.objects.filter( + log_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) - if status_data['check_time__max'] is not None: - min_check_time = status_data['check_time__max'] - timedelta(minutes=5) - min_sync_time = status_data['last_sync__max'] - timedelta(minutes=20) + if log_data['check_time__max'] is not None: + min_check_time = log_data['check_time__max'] - timedelta(minutes=5) + min_sync_time = log_data['last_sync__max'] - timedelta(minutes=20) best_logs = MirrorLog.objects.select_related('url').filter( is_success=True, check_time__gte=min_check_time, last_sync__gte=min_sync_time, -- cgit v1.2.3 From 5ec5b1a70245fa4c0bec4026de14ea54cec00353 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 26 Feb 2015 19:37:23 -0600 Subject: Break out mirrorlist views Signed-off-by: Dan McGee --- mirrors/urls_mirrorlist.py | 2 +- mirrors/views/__init__.py | 123 +----------------------------------------- mirrors/views/mirrorlist.py | 129 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 123 deletions(-) create mode 100644 mirrors/views/mirrorlist.py diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index bba54ec9..a64656a9 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -1,7 +1,7 @@ from django.conf.urls import patterns -urlpatterns = patterns('mirrors.views', +urlpatterns = patterns('mirrors.views.mirrorlist', (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), (r'^all/$', 'find_mirrors', {'countries': ['all']}), (r'^all/(?P[A-z]+)/$', 'find_mirrors_simple', diff --git a/mirrors/views/__init__.py b/mirrors/views/__init__.py index 54c427b3..d84b0b10 100644 --- a/mirrors/views/__init__.py +++ b/mirrors/views/__init__.py @@ -3,18 +3,13 @@ from itertools import groupby import json from operator import attrgetter, itemgetter -from django import forms -from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db import connection -from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404, redirect, render +from django.shortcuts import get_object_or_404, render from django.utils.timezone import now from django.views.decorators.cache import cache_page -from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import condition -from django_countries import countries from django_countries.fields import Country from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, @@ -22,122 +17,6 @@ from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, from ..utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF -class MirrorlistForm(forms.Form): - country = forms.MultipleChoiceField(required=False, - widget=SelectMultiple(attrs={'size': '12'})) - protocol = forms.MultipleChoiceField(required=False, - widget=CheckboxSelectMultiple) - ip_version = forms.MultipleChoiceField(required=False, - label="IP version", choices=(('4','IPv4'), ('6','IPv6')), - widget=CheckboxSelectMultiple) - use_mirror_status = forms.BooleanField(required=False) - - def __init__(self, *args, **kwargs): - super(MirrorlistForm, self).__init__(*args, **kwargs) - fields = self.fields - fields['country'].choices = [('all','All')] + self.get_countries() - fields['country'].initial = ['all'] - protos = [(p.protocol, p.protocol) for p in - MirrorProtocol.objects.filter(is_download=True)] - initial = MirrorProtocol.objects.filter(is_download=True, default=True) - fields['protocol'].choices = protos - fields['protocol'].initial = [p.protocol for p in initial] - fields['ip_version'].initial = ['4'] - - def get_countries(self): - country_codes = set() - country_codes.update(MirrorUrl.objects.filter(active=True, - mirror__active=True).exclude(country='').values_list( - 'country', flat=True).order_by().distinct()) - code_list = [(code, countries.name(code)) for code in country_codes] - return sorted(code_list, key=itemgetter(1)) - - def as_div(self): - "Returns this form rendered as HTML s." - return self._html_output( - normal_row = u'%(label)s %(field)s%(help_text)s', - error_row = u'%s', - row_ender = '', - help_text_html = u' %s', - errors_on_separate_row = True) - - -@csrf_exempt -def generate_mirrorlist(request): - if request.method == 'POST' or len(request.GET) > 0: - form = MirrorlistForm(data=request.REQUEST) - if form.is_valid(): - countries = form.cleaned_data['country'] - protocols = form.cleaned_data['protocol'] - use_status = form.cleaned_data['use_mirror_status'] - ipv4 = '4' in form.cleaned_data['ip_version'] - ipv6 = '6' in form.cleaned_data['ip_version'] - return find_mirrors(request, countries, protocols, - use_status, ipv4, ipv6) - else: - form = MirrorlistForm() - - return render(request, 'mirrors/mirrorlist_generate.html', - {'mirrorlist_form': form}) - - -def status_filter(original_urls): - status_info = get_mirror_statuses() - scores = {u.id: u.score for u in status_info['urls']} - urls = [] - for u in original_urls: - u.score = scores.get(u.id, None) - # also include mirrors that don't have an up to date score - # (as opposed to those that have been set with no score) - if (u.id not in scores) or (u.score and u.score < 100.0): - urls.append(u) - # if a url doesn't have a score, treat it as the highest possible - return sorted(urls, key=lambda x: x.score or 100.0) - - -def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True): - if not protocols: - protocols = MirrorProtocol.objects.filter(is_download=True) - elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: - # we already have a queryset, no need to query again - pass - else: - protocols = MirrorProtocol.objects.filter(protocol__in=protocols) - qset = MirrorUrl.objects.select_related().filter( - protocol__in=protocols, active=True, - mirror__public=True, mirror__active=True) - if countries and 'all' not in countries: - qset = qset.filter(country__in=countries) - - ip_version = Q() - if ipv4_supported: - ip_version |= Q(has_ipv4=True) - if ipv6_supported: - ip_version |= Q(has_ipv6=True) - qset = qset.filter(ip_version) - - if not use_status: - sort_key = attrgetter('country.name', 'mirror.name', 'url') - urls = sorted(qset, key=sort_key) - template = 'mirrors/mirrorlist.txt' - else: - urls = status_filter(qset) - template = 'mirrors/mirrorlist_status.txt' - - context = { - 'mirror_urls': urls, - } - return render(request, template, context, content_type='text/plain') - - -def find_mirrors_simple(request, protocol): - if protocol == 'smart': - return redirect('mirrorlist_simple', 'http', permanent=True) - proto = get_object_or_404(MirrorProtocol, protocol=protocol) - return find_mirrors(request, protocols=[proto]) - - def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'name') protos = MirrorUrl.objects.values_list( diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py new file mode 100644 index 00000000..3c68d036 --- /dev/null +++ b/mirrors/views/mirrorlist.py @@ -0,0 +1,129 @@ +from operator import attrgetter, itemgetter + +from django import forms +from django.db.models import Q +from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple +from django.shortcuts import get_object_or_404, redirect, render +from django.views.decorators.csrf import csrf_exempt +from django_countries import countries + +from ..models import MirrorUrl, MirrorProtocol +from ..utils import get_mirror_statuses + + +class MirrorlistForm(forms.Form): + country = forms.MultipleChoiceField(required=False, + widget=SelectMultiple(attrs={'size': '12'})) + protocol = forms.MultipleChoiceField(required=False, + widget=CheckboxSelectMultiple) + ip_version = forms.MultipleChoiceField(required=False, + label="IP version", choices=(('4','IPv4'), ('6','IPv6')), + widget=CheckboxSelectMultiple) + use_mirror_status = forms.BooleanField(required=False) + + def __init__(self, *args, **kwargs): + super(MirrorlistForm, self).__init__(*args, **kwargs) + fields = self.fields + fields['country'].choices = [('all','All')] + self.get_countries() + fields['country'].initial = ['all'] + protos = [(p.protocol, p.protocol) for p in + MirrorProtocol.objects.filter(is_download=True)] + initial = MirrorProtocol.objects.filter(is_download=True, default=True) + fields['protocol'].choices = protos + fields['protocol'].initial = [p.protocol for p in initial] + fields['ip_version'].initial = ['4'] + + def get_countries(self): + country_codes = set() + country_codes.update(MirrorUrl.objects.filter(active=True, + mirror__active=True).exclude(country='').values_list( + 'country', flat=True).order_by().distinct()) + code_list = [(code, countries.name(code)) for code in country_codes] + return sorted(code_list, key=itemgetter(1)) + + def as_div(self): + "Returns this form rendered as HTML s." + return self._html_output( + normal_row = u'%(label)s %(field)s%(help_text)s', + error_row = u'%s', + row_ender = '', + help_text_html = u' %s', + errors_on_separate_row = True) + + +@csrf_exempt +def generate_mirrorlist(request): + if request.method == 'POST' or len(request.GET) > 0: + form = MirrorlistForm(data=request.REQUEST) + if form.is_valid(): + countries = form.cleaned_data['country'] + protocols = form.cleaned_data['protocol'] + use_status = form.cleaned_data['use_mirror_status'] + ipv4 = '4' in form.cleaned_data['ip_version'] + ipv6 = '6' in form.cleaned_data['ip_version'] + return find_mirrors(request, countries, protocols, + use_status, ipv4, ipv6) + else: + form = MirrorlistForm() + + return render(request, 'mirrors/mirrorlist_generate.html', + {'mirrorlist_form': form}) + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = {u.id: u.score for u in status_info['urls']} + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + +def find_mirrors(request, countries=None, protocols=None, use_status=False, + ipv4_supported=True, ipv6_supported=True): + if not protocols: + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) + qset = MirrorUrl.objects.select_related().filter( + protocol__in=protocols, active=True, + mirror__public=True, mirror__active=True) + if countries and 'all' not in countries: + qset = qset.filter(country__in=countries) + + ip_version = Q() + if ipv4_supported: + ip_version |= Q(has_ipv4=True) + if ipv6_supported: + ip_version |= Q(has_ipv6=True) + qset = qset.filter(ip_version) + + if not use_status: + sort_key = attrgetter('country.name', 'mirror.name', 'url') + urls = sorted(qset, key=sort_key) + template = 'mirrors/mirrorlist.txt' + else: + urls = status_filter(qset) + template = 'mirrors/mirrorlist_status.txt' + + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') + + +def find_mirrors_simple(request, protocol): + if protocol == 'smart': + return redirect('mirrorlist_simple', 'http', permanent=True) + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3 From e739d440e91ff58e2efa59f8b142a7f71fa1e77f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 26 Feb 2015 19:53:44 -0600 Subject: Break out mirror JSON API views Signed-off-by: Dan McGee --- mirrors/urls.py | 9 ++-- mirrors/views/__init__.py | 102 +------------------------------------------- mirrors/views/api.py | 106 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 103 deletions(-) create mode 100644 mirrors/views/api.py diff --git a/mirrors/urls.py b/mirrors/urls.py index b1054380..a41bfdc3 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -3,13 +3,16 @@ from django.conf.urls import patterns urlpatterns = patterns('mirrors.views', (r'^$', 'mirrors', {}, 'mirror-list'), (r'^status/$', 'status', {}, 'mirror-status'), - (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), + (r'^(?P[\.\-\w]+)/$', 'mirror_details'), + (r'^(?P[\.\-\w]+)/(?P\d+)/$', 'url_details'), +) + +urlpatterns += patterns('mirrors.views.api', + (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), - (r'^(?P[\.\-\w]+)/$', 'mirror_details'), (r'^(?P[\.\-\w]+)/json/$', 'mirror_details_json'), - (r'^(?P[\.\-\w]+)/(?P\d+)/$', 'url_details'), ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/views/__init__.py b/mirrors/views/__init__.py index d84b0b10..01e8519d 100644 --- a/mirrors/views/__init__.py +++ b/mirrors/views/__init__.py @@ -1,20 +1,17 @@ from datetime import timedelta from itertools import groupby -import json from operator import attrgetter, itemgetter -from django.core.serializers.json import DjangoJSONEncoder from django.db import connection -from django.http import Http404, HttpResponse +from django.http import Http404 from django.shortcuts import get_object_or_404, render from django.utils.timezone import now -from django.views.decorators.cache import cache_page from django.views.decorators.http import condition from django_countries.fields import Country from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, CheckLocation) -from ..utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF +from ..utils import get_mirror_statuses, get_mirror_errors def mirrors(request): @@ -84,21 +81,6 @@ def mirror_details(request, name): return render(request, 'mirrors/mirror_details.html', context) -def mirror_details_json(request, name): - authorized = request.user.is_authenticated() - mirror = get_object_or_404(Mirror, name=name) - if not authorized and (not mirror.public or not mirror.active): - raise Http404 - status_info = get_mirror_statuses(mirror_id=mirror.id, - show_all=authorized) - data = status_info.copy() - data['version'] = 3 - to_json = json.dumps(data, ensure_ascii=False, - cls=ExtendedMirrorStatusJSONEncoder) - response = HttpResponse(to_json, content_type='application/json') - return response - - def url_details(request, name, url_id): url = get_object_or_404(MirrorUrl.objects.select_related(), id=url_id, mirror__name=name) @@ -164,84 +146,4 @@ def status(request, tier=None): }) return render(request, 'mirrors/status.html', context) - -class MirrorStatusJSONEncoder(DjangoJSONEncoder): - '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl - serialization. The base class takes care of datetime.datetime types.''' - url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', - 'delay', 'duration_avg', 'duration_stddev', 'score') - - def default(self, obj): - if isinstance(obj, timedelta): - # always returned as integer seconds - return obj.days * 24 * 3600 + obj.seconds - if isinstance(obj, MirrorUrl): - data = {attr: getattr(obj, attr) for attr in self.url_attributes} - country = obj.country - data['country'] = unicode(country.name) - data['country_code'] = country.code - return data - if isinstance(obj, MirrorProtocol): - return unicode(obj) - return super(MirrorStatusJSONEncoder, self).default(obj) - - -class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): - '''Adds URL check history information.''' - log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', - 'location_id') - - def default(self, obj): - if isinstance(obj, MirrorUrl): - data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - cutoff = now() - DEFAULT_CUTOFF - data['logs'] = list(obj.logs.filter( - check_time__gte=cutoff).order_by('check_time')) - return data - if isinstance(obj, MirrorLog): - return {attr: getattr(obj, attr) for attr in self.log_attributes} - return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - - -@cache_page(67) -@condition(last_modified_func=status_last_modified) -def status_json(request, tier=None): - if tier is not None: - tier = int(tier) - if tier not in [t[0] for t in Mirror.TIER_CHOICES]: - raise Http404 - status_info = get_mirror_statuses() - data = status_info.copy() - if tier is not None: - data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] - data['version'] = 3 - to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) - response = HttpResponse(to_json, content_type='application/json') - return response - - -class LocationJSONEncoder(DjangoJSONEncoder): - '''Base JSONEncoder extended to handle CheckLocation objects.''' - - def default(self, obj): - if isinstance(obj, CheckLocation): - return { - 'id': obj.pk, - 'hostname': obj.hostname, - 'source_ip': obj.source_ip, - 'country': unicode(obj.country.name), - 'country_code': obj.country.code, - 'ip_version': obj.ip_version, - } - return super(LocationJSONEncoder, self).default(obj) - - -def locations_json(request): - data = {} - data['version'] = 1 - data['locations'] = list(CheckLocation.objects.all().order_by('pk')) - to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) - response = HttpResponse(to_json, content_type='application/json') - return response - # vim: set ts=4 sw=4 et: diff --git a/mirrors/views/api.py b/mirrors/views/api.py new file mode 100644 index 00000000..06574082 --- /dev/null +++ b/mirrors/views/api.py @@ -0,0 +1,106 @@ +from datetime import timedelta +import json + +from django.core.serializers.json import DjangoJSONEncoder +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404 +from django.utils.timezone import now + +from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) +from ..utils import get_mirror_statuses, DEFAULT_CUTOFF + + +class MirrorStatusJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl + serialization. The base class takes care of datetime.datetime types.''' + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') + + def default(self, obj): + if isinstance(obj, timedelta): + # always returned as integer seconds + return obj.days * 24 * 3600 + obj.seconds + if isinstance(obj, MirrorUrl): + data = {attr: getattr(obj, attr) for attr in self.url_attributes} + country = obj.country + data['country'] = unicode(country.name) + data['country_code'] = country.code + return data + if isinstance(obj, MirrorProtocol): + return unicode(obj) + return super(MirrorStatusJSONEncoder, self).default(obj) + + +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - DEFAULT_CUTOFF + data['logs'] = list(obj.logs.filter( + check_time__gte=cutoff).order_by('check_time')) + return data + if isinstance(obj, MirrorLog): + return {attr: getattr(obj, attr) for attr in self.log_attributes} + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if isinstance(obj, CheckLocation): + return { + 'id': obj.pk, + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + status_info = get_mirror_statuses() + data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def mirror_details_json(request, name): + authorized = request.user.is_authenticated() + mirror = get_object_or_404(Mirror, name=name) + if not authorized and (not mirror.public or not mirror.active): + raise Http404 + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) + data = status_info.copy() + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = list(CheckLocation.objects.all().order_by('pk')) + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3 From 65daa766cad72f9d6271439789fc399999b3a973 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 26 Feb 2015 20:01:14 -0600 Subject: Include error message in JSON detail response Signed-off-by: Dan McGee --- mirrors/views/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mirrors/views/api.py b/mirrors/views/api.py index 06574082..b72585e6 100644 --- a/mirrors/views/api.py +++ b/mirrors/views/api.py @@ -45,7 +45,9 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): check_time__gte=cutoff).order_by('check_time')) return data if isinstance(obj, MirrorLog): - return {attr: getattr(obj, attr) for attr in self.log_attributes} + data = {attr: getattr(obj, attr) for attr in self.log_attributes} + data['error'] = obj.error or None + return data return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) -- cgit v1.2.3 From 03f6a63c6c6dff900b559b35ab571133990613a2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 26 Feb 2015 20:44:25 -0600 Subject: Fix URL precedence due to /json/ and mirror name overlap Move these back into one block and just use function references rather than the string-based way. Signed-off-by: Dan McGee --- mirrors/urls.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mirrors/urls.py b/mirrors/urls.py index a41bfdc3..fc510fbb 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -1,18 +1,18 @@ from django.conf.urls import patterns -urlpatterns = patterns('mirrors.views', - (r'^$', 'mirrors', {}, 'mirror-list'), - (r'^status/$', 'status', {}, 'mirror-status'), - (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), - (r'^(?P[\.\-\w]+)/$', 'mirror_details'), - (r'^(?P[\.\-\w]+)/(?P\d+)/$', 'url_details'), -) +from .views import mirrors, status, mirror_details, url_details +from .views.api import status_json, mirror_details_json, locations_json -urlpatterns += patterns('mirrors.views.api', - (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), - (r'^status/tier/(?P\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), - (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), - (r'^(?P[\.\-\w]+)/json/$', 'mirror_details_json'), +urlpatterns = patterns('', + (r'^$', mirrors, {}, 'mirror-list'), + (r'^status/$', status, {}, 'mirror-status'), + (r'^status/json/$', status_json, {}, 'mirror-status-json'), + (r'^status/tier/(?P\d+)/$', status, {}, 'mirror-status-tier'), + (r'^status/tier/(?P\d+)/json/$', status_json, {}, 'mirror-status-tier-json'), + (r'^locations/json/$', locations_json, {}, 'mirror-locations-json'), + (r'^(?P[\.\-\w]+)/$', mirror_details), + (r'^(?P[\.\-\w]+)/json/$', mirror_details_json), + (r'^(?P[\.\-\w]+)/(?P\d+)/$', url_details), ) # vim: set ts=4 sw=4 et: -- cgit v1.2.3 From 6d176f519c080406bdd624cb246227fd8ed26e4b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Mar 2015 12:17:33 -0500 Subject: Django version bump to 1.7.5 Signed-off-by: Dan McGee --- requirements.txt | 2 +- requirements_prod.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8e8dd0fb..b6c0acc1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.4 +Django==1.7.5 IPy==0.81 Jinja2==2.7.3 Markdown==2.6 diff --git a/requirements_prod.txt b/requirements_prod.txt index ce655fb5..52484e6a 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,5 +1,5 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.4 +Django==1.7.5 IPy==0.81 Jinja2==2.7.3 Markdown==2.6 -- cgit v1.2.3 From b5c0092e9a7581501b1087844d9cc460cbe8d925 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 15 Mar 2015 11:07:24 -0500 Subject: FS#44192 fix dead link to ftp.archlinux.org Use the new sources.archlinux.org domain instead. Signed-off-by: Dan McGee --- templates/public/art.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/public/art.html b/templates/public/art.html index 96312971..afab5498 100644 --- a/templates/public/art.html +++ b/templates/public/art.html @@ -68,14 +68,14 @@
  • archlinux-themes-slim - SLiM login themes
    • -

    Alternatively, you can download the source files via FTP.

    +

    Alternatively, you can download the source files.

    Former Logos

    Arch has gone through a few generations of branding and what follows are some of our past logos. Although these images are no longer used frequently, - they remain subject to license restrictions. Email + they remain subject to license restrictions. Email trademarks@archlinux.org with any questions.

    Original Ribbon Series

    -- cgit v1.2.3 From 29298e4953feb9f702ef7daedd61a5059a2fb447 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 15 Mar 2015 11:12:36 -0500 Subject: More dependency version bumps Signed-off-by: Dan McGee --- requirements.txt | 6 +++--- requirements_prod.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index b6c0acc1..c58422f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.5 +Django==1.7.6 IPy==0.81 Jinja2==2.7.3 -Markdown==2.6 +Markdown==2.6.1 MarkupSafe==0.23 bencode==1.0 django-countries==3.2 -django-jinja==1.1.1 +django-jinja==1.3.0 jsmin==2.1.1 pgpdump==1.5 pytz>=2014.10 diff --git a/requirements_prod.txt b/requirements_prod.txt index 52484e6a..087586fa 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,12 +1,12 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.5 +Django==1.7.6 IPy==0.81 Jinja2==2.7.3 -Markdown==2.6 +Markdown==2.6.1 MarkupSafe==0.23 bencode==1.0 django-countries==3.2 -django-jinja==1.1.1 +django-jinja==1.3.0 jsmin==2.1.1 pgpdump==1.5 psycopg2==2.6 -- cgit v1.2.3 From 91954bb99cdb58c5f43b86fbb0a2a70aca42529e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 20 Mar 2015 08:15:51 -0500 Subject: Update newrelic configuration to more recent template Signed-off-by: Dan McGee --- newrelic.ini | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/newrelic.ini b/newrelic.ini index 98cca5b6..5ced1ed2 100644 --- a/newrelic.ini +++ b/newrelic.ini @@ -54,7 +54,7 @@ monitor_mode = true # write out a log file, it is also possible to say "stderr" and # output to standard error output. This would normally result in # output appearing in your web server log. -log_file = /tmp/newrelic-python-agent.log +#log_file = /tmp/newrelic-python-agent.log # Sets the level of detail of messages sent to the log file, if # a log file location has been provided. Possible values, in @@ -68,14 +68,27 @@ log_file = /tmp/newrelic-python-agent.log log_level = info # The Python Agent communicates with the New Relic service using -# HTTP by default. If you want to communicate via HTTPS to -# increase security, then turn on SSL by setting this value to -# true. Note, this will result in increased CPU overhead to -# perform the encryption involved in SSL communication, but this -# work is done asynchronously to the threads that process your -# application code, so it should not impact response times. +# SSL by default. Note that this does result in an increase in +# CPU overhead, over and above what would occur for a non SSL +# connection, to perform the encryption involved in the SSL +# communication. This work is though done in a distinct thread +# to those handling your web requests, so it should not impact +# response times. You can if you wish revert to using a non SSL +# connection, but this will result in information being sent +# over a plain socket connection and will not be as secure. ssl = true +# High Security Mode enforces certain security settings, and +# prevents them from being overridden, so that no sensitive data +# is sent to New Relic. Enabling High Security Mode means that +# SSL is turned on, request parameters are not collected, and SQL +# can not be sent to New Relic in its raw form. To activate High +# Security Mode, it must be set to 'true' in this local .ini +# configuration file AND be set to 'true' in the server-side +# configuration in the New Relic user interface. For details, see +# https://docs.newrelic.com/docs/subscriptions/high-security +high_security = false + # The Python Agent will attempt to connect directly to the New # Relic service. If there is an intermediate firewall between # your host and the New Relic service that requires you to use a @@ -83,7 +96,12 @@ ssl = true # "proxy_port" settings to the required values for the HTTP # proxy. The "proxy_user" and "proxy_pass" settings should # additionally be set if proxy authentication is implemented by -# the HTTP proxy. +# the HTTP proxy. The "proxy_scheme" setting dictates what +# protocol scheme is used in talking to the HTTP proxy. This +# would normally always be set as "http" which will result in the +# agent then using a SSL tunnel through the HTTP proxy for end to +# end encryption. +# proxy_scheme = http # proxy_host = hostname # proxy_port = 8080 # proxy_user = @@ -153,7 +171,7 @@ error_collector.enabled = true # To stop specific errors from reporting to the UI, set this to # a space separated list of the Python exception type names to # ignore. The exception name should be of the form 'module:class'. -error_collector.ignore_errors = django.http.response:Http404 +error_collector.ignore_errors = # Browser monitoring is the Real User Monitoring feature of the UI. # For those Python web frameworks that are supported, this -- cgit v1.2.3 From 30bb320ed7be084e8b86e050d426f20d4b5a9045 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 20 Mar 2015 08:16:35 -0500 Subject: Turn off real user/JavaScript NewRelic monitoring Signed-off-by: Dan McGee --- newrelic.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic.ini b/newrelic.ini index 5ced1ed2..72158dc4 100644 --- a/newrelic.ini +++ b/newrelic.ini @@ -177,7 +177,7 @@ error_collector.ignore_errors = # For those Python web frameworks that are supported, this # setting enables the auto-insertion of the browser monitoring # JavaScript fragments. -browser_monitoring.auto_instrument = true +browser_monitoring.auto_instrument = false # A thread profiling session can be scheduled via the UI when # this option is enabled. The thread profiler will periodically -- cgit v1.2.3 From ffa9a3ca80dad541344571aa08b96372fd11ecaa Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 2 Apr 2015 08:11:14 -0500 Subject: Update to latest django-countries Signed-off-by: Dan McGee --- requirements.txt | 2 +- requirements_prod.txt | 2 +- settings.py | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index c58422f7..42e0a6a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ Jinja2==2.7.3 Markdown==2.6.1 MarkupSafe==0.23 bencode==1.0 -django-countries==3.2 +django-countries==3.3 django-jinja==1.3.0 jsmin==2.1.1 pgpdump==1.5 diff --git a/requirements_prod.txt b/requirements_prod.txt index 087586fa..b0d7cfec 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -5,7 +5,7 @@ Jinja2==2.7.3 Markdown==2.6.1 MarkupSafe==0.23 bencode==1.0 -django-countries==3.2 +django-countries==3.3 django-jinja==1.3.0 jsmin==2.1.1 pgpdump==1.5 diff --git a/settings.py b/settings.py index 3f0f8e69..0423541c 100644 --- a/settings.py +++ b/settings.py @@ -151,7 +151,7 @@ LOGGING = { 'handlers': ['mail_admins'], 'level': 'ERROR', 'propagate': True, - }, + } }, } @@ -174,6 +174,12 @@ TORRENT_TRACKERS = ( 'http://tracker.archlinux.org:6969/announce', ) +# Shorten some names just a bit +COUNTRIES_OVERRIDE = { + 'GB': 'United Kingdom', + 'US': 'United States', +} + ## Import local settings from local_settings import * -- cgit v1.2.3 From e84c4d01b135a7b96ab2cae0c58588c7ed597d1f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 2 Apr 2015 08:13:22 -0500 Subject: The usual requirements bumping Signed-off-by: Dan McGee --- requirements.txt | 6 +++--- requirements_prod.txt | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 42e0a6a8..7b8c4822 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.6 +Django==1.7.7 IPy==0.81 Jinja2==2.7.3 Markdown==2.6.1 MarkupSafe==0.23 bencode==1.0 django-countries==3.3 -django-jinja==1.3.0 +django-jinja==1.3.2 jsmin==2.1.1 pgpdump==1.5 -pytz>=2014.10 +pytz>=2015.2 diff --git a/requirements_prod.txt b/requirements_prod.txt index b0d7cfec..3396ebdb 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,15 +1,15 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.7.6 +Django==1.7.7 IPy==0.81 Jinja2==2.7.3 Markdown==2.6.1 MarkupSafe==0.23 bencode==1.0 django-countries==3.3 -django-jinja==1.3.0 +django-jinja==1.3.2 jsmin==2.1.1 pgpdump==1.5 psycopg2==2.6 pyinotify==0.9.5 -python-memcached==1.53 -pytz>=2014.10 +python-memcached==1.54 +pytz>=2015.2 -- cgit v1.2.3 From 57198b5cb45fb390b2fa75faff371d323de27dab Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Apr 2015 18:51:44 -0500 Subject: Fix URL details page when checks don't have locations Signed-off-by: Dan McGee --- templates/mirrors/url_details_logs.html.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/mirrors/url_details_logs.html.jinja b/templates/mirrors/url_details_logs.html.jinja index 58f179d8..51f54931 100644 --- a/templates/mirrors/url_details_logs.html.jinja +++ b/templates/mirrors/url_details_logs.html.jinja @@ -14,8 +14,8 @@ {% for log in logs %} {{ log.check_time|date('Y-m-d H:i') }} - {{ country_flag(log.location.country) }}{{ log.location.country.name }} - {{ log.location.source_ip }} + {% if log.location %}{{ country_flag(log.location.country) }}{{ log.location.country.name }}{% else %}Unknown{% endif %} + {% if log.location %}{{ log.location.source_ip }}{% else %}Unknown{% endif %} {{ log.last_sync|date('Y-m-d H:i') }} {{ log.delay|duration }} {{ log.duration|floatvalue }} -- cgit v1.2.3 From a2b020ed3554b47c0b27982cf47431126c103ef4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 12 Apr 2015 09:50:13 -0500 Subject: Add true parameter to Jinja2 default filter We need this to make it act more like the Django filter, which prints the default value not only when the variable is undefined, but also when it evaluates to being false (such as a None value). Noticed by felixonmars on packages that provided other packages; None was always showing as the comparison operator. More embarrassing is that our file lists were always showing 'None' concatenated onto all directory entries. Signed-off-by: Dan McGee --- templates/packages/details.html.jinja | 4 ++-- templates/packages/details_depend.html.jinja | 6 +++--- templates/packages/details_relatedto.html.jinja | 2 +- templates/packages/files_list.html.jinja | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/templates/packages/details.html.jinja b/templates/packages/details.html.jinja index 2215f8dc..9a72622d 100644 --- a/templates/packages/details.html.jinja +++ b/templates/packages/details.html.jinja @@ -103,7 +103,7 @@ {% endif %} Description: - {{ pkg.pkgdesc|default("") }} + {{ pkg.pkgdesc|default("", true) }} Upstream URL: {% if pkg.url %}