From a97ba8bbca30d656598f71f249e6fbe7020773bf Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 25 Nov 2013 21:31:00 -0600 Subject: Bump some requirements Signed-off-by: Dan McGee --- requirements.txt | 4 ++-- requirements_prod.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 08d89107..7ff363e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,9 @@ Django==1.6 IPy==0.81 Markdown==2.3.1 -South==0.8.2 +South==0.8.4 bencode==1.0 django-countries==1.5 -jsmin==2.0.6 +jsmin==2.0.8 pgpdump==1.4 pytz>=2013.8 diff --git a/requirements_prod.txt b/requirements_prod.txt index 565c3c6c..3f2958c6 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -2,10 +2,10 @@ Django==1.6 IPy==0.81 Markdown==2.3.1 -South==0.8.2 +South==0.8.4 bencode==1.0 django-countries==1.5 -jsmin==2.0.6 +jsmin==2.0.8 pgpdump==1.4 psycopg2==2.5.1 pyinotify==0.9.4 -- cgit v1.2.3-54-g00ecf From efd1bd15ef34b9ebafd703375108fb41b1a6ecf0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 25 Nov 2013 21:31:21 -0600 Subject: Capitalize URL properly in Django admin Signed-off-by: Dan McGee --- main/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/models.py b/main/models.py index 8d9d4c89..bf7a9409 100644 --- a/main/models.py +++ b/main/models.py @@ -97,7 +97,7 @@ class Package(models.Model): pkgrel = models.CharField(max_length=255) epoch = models.PositiveIntegerField(default=0) pkgdesc = models.TextField('description', null=True) - url = models.CharField(max_length=255, null=True) + url = models.CharField('URL', max_length=255, null=True) filename = models.CharField(max_length=255) compressed_size = PositiveBigIntegerField() installed_size = PositiveBigIntegerField() -- cgit v1.2.3-54-g00ecf From ae61d98ed63ce0f5932548b31ce516fc52f05c8d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 25 Nov 2013 21:54:12 -0600 Subject: Move markdown formatted readme to README.md Signed-off-by: Dan McGee --- README | 81 +-------------------------------------------------------------- README.md | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 80 deletions(-) mode change 100644 => 120000 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index beeb7f99..00000000 --- a/README +++ /dev/null @@ -1,80 +0,0 @@ -# Archweb README - -To get a pretty version of this document, run - - $ markdown README > README.html - -# License - -See LICENSE file. - -# Authors - -See AUTHORS file. - -# Dependencies - -- python2 -- python2-virtualenv - -# Python dependencies - -More detail in `requirements.txt` and `requirements_prod.txt`; it is best to -use virtualenv and pip to handle these. But if you insist on (Arch Linux) -packages, you will probably want the following: - -- django -- python2-psycopg2 -- python2-markdown -- python2-south -- python2-memcached - -# Testing Installation - -1. Run `virtualenv2`. - - $ cd /path/to/archweb && virtualenv2 ./env/ - -2. Activate the virtualenv. - - $ source ./env/bin/activate - -2. Install dependencies through `pip`. - - (archweb-env) $ pip install -r requirements.txt - -3. Copy `local_settings.py.example` to `local_settings.py` and modify. - Make sure to uncomment the appropriate database section (either sqlite or - PostgreSQL). - -4. Sync the database to create it. - - (archweb-env) $ ./manage.py syncdb - -5. Migrate changes. - - (archweb-env) $ ./manage.py migrate - -6. Load the fixtures to prepopulate some data. If you don't want some of the - provided data, adjust the file glob accordingly. - - (archweb-env) $ ./manage.py loaddata */fixtures/*.json - -7. Use the following commands to start a service instance - - (archweb-env) $ ./manage.py runserver - -8. To optionally populate the database with real data: - - (archweb-env) $ wget ftp://ftp.archlinux.org/core/os/i686/core.db.tar.gz - (archweb-env) $ ./manage.py reporead i686 core.db.tar.gz - (archweb-env) $ ./manage.py syncisos - -Alter architecture and repo to get x86\_64 and packages from other repos if -needed. - -# Production Installation - -Ask someone who knows, or you are going to be in trouble. - -vim: set syntax=markdown et: diff --git a/README b/README new file mode 120000 index 00000000..42061c01 --- /dev/null +++ b/README @@ -0,0 +1 @@ +README.md \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..beeb7f99 --- /dev/null +++ b/README.md @@ -0,0 +1,80 @@ +# Archweb README + +To get a pretty version of this document, run + + $ markdown README > README.html + +# License + +See LICENSE file. + +# Authors + +See AUTHORS file. + +# Dependencies + +- python2 +- python2-virtualenv + +# Python dependencies + +More detail in `requirements.txt` and `requirements_prod.txt`; it is best to +use virtualenv and pip to handle these. But if you insist on (Arch Linux) +packages, you will probably want the following: + +- django +- python2-psycopg2 +- python2-markdown +- python2-south +- python2-memcached + +# Testing Installation + +1. Run `virtualenv2`. + + $ cd /path/to/archweb && virtualenv2 ./env/ + +2. Activate the virtualenv. + + $ source ./env/bin/activate + +2. Install dependencies through `pip`. + + (archweb-env) $ pip install -r requirements.txt + +3. Copy `local_settings.py.example` to `local_settings.py` and modify. + Make sure to uncomment the appropriate database section (either sqlite or + PostgreSQL). + +4. Sync the database to create it. + + (archweb-env) $ ./manage.py syncdb + +5. Migrate changes. + + (archweb-env) $ ./manage.py migrate + +6. Load the fixtures to prepopulate some data. If you don't want some of the + provided data, adjust the file glob accordingly. + + (archweb-env) $ ./manage.py loaddata */fixtures/*.json + +7. Use the following commands to start a service instance + + (archweb-env) $ ./manage.py runserver + +8. To optionally populate the database with real data: + + (archweb-env) $ wget ftp://ftp.archlinux.org/core/os/i686/core.db.tar.gz + (archweb-env) $ ./manage.py reporead i686 core.db.tar.gz + (archweb-env) $ ./manage.py syncisos + +Alter architecture and repo to get x86\_64 and packages from other repos if +needed. + +# Production Installation + +Ask someone who knows, or you are going to be in trouble. + +vim: set syntax=markdown et: -- cgit v1.2.3-54-g00ecf From e557545ca803fa215b3a7f43f4c421dff1259fe3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 27 Nov 2013 16:00:49 -0500 Subject: Use pkg_details_link tag on differences report page Signed-off-by: Dan McGee --- templates/packages/differences.html | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/templates/packages/differences.html b/templates/packages/differences.html index f137126d..f4d93151 100644 --- a/templates/packages/differences.html +++ b/templates/packages/differences.html @@ -1,6 +1,7 @@ {% extends "base.html" %} {% load cycle from future %} {% load static from staticfiles %} +{% load package_extras %} {% block title %}Arch Linux - Package Differences Reports{% endblock %} {% block navbarclass %}anb-packages{% endblock %} @@ -78,12 +79,10 @@

Multilib Differences to Main Packages

{% for pkg1, pkg2 in multilib_differences %} - {{ pkg1.pkgname }} + {% pkg_details_link pkg1 %} {{ pkg1.full_version }} {{ pkg2.full_version }} - {{ pkg2.pkgname }} + {% pkg_details_link pkg2 %} {{ pkg2.repo }} {{ pkg1.last_update|date }} {{ pkg2.last_update|date }} -- cgit v1.2.3-54-g00ecf From 58356ec5b38c71204895eb77856a45ffd7ac1725 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 27 Nov 2013 16:07:49 -0500 Subject: Allow pkg_details_link to honor package flag state This allows the tag to be used in a few more places we weren't already able to use it, and hopefully speeds up rendering a tad on the package differences page. Signed-off-by: Dan McGee --- packages/templatetags/package_extras.py | 7 +++++-- templates/packages/differences.html | 8 ++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index f7392a96..3ce64748 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -67,13 +67,16 @@ def do_buildsortqs(parser, token): @register.simple_tag -def pkg_details_link(pkg, link_title=None): +def pkg_details_link(pkg, link_title=None, honor_flagged=False): if not pkg: return link_title or '' if link_title is None: link_title = pkg.pkgname + link_content = link_title + if honor_flagged and pkg.flag_date: + link_content = '%s' % link_title link = '%s' - return link % (pkg.get_absolute_url(), pkg.pkgname, link_title) + return link % (pkg.get_absolute_url(), pkg.pkgname, link_content) @register.simple_tag diff --git a/templates/packages/differences.html b/templates/packages/differences.html index f4d93151..5ebe18e2 100644 --- a/templates/packages/differences.html +++ b/templates/packages/differences.html @@ -46,14 +46,10 @@

Filter Differences View

{{ diff.pkgname }} {{ diff.repo.name }} {% if diff.pkg_a %} - - {{ diff.pkg_a.full_version }} + {% pkg_details_link diff.pkg_a diff.pkg_a.full_version True %} {% else %}-{% endif %} {% if diff.pkg_b %} - - {{ diff.pkg_b.full_version }} + {% pkg_details_link diff.pkg_b diff.pkg_b.full_version True %} {% else %}-{% endif %} {% endfor %} -- cgit v1.2.3-54-g00ecf From 429736f7e9fcdc92ea43f98e3a246b9e41e04707 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 27 Nov 2013 17:05:02 -0500 Subject: Remove date_hierarchy from several package admin views I was never using this and it adds some pretty substantial overhead to each render of the admin page, so toss it. Add the simple date filter to the sidebar instead. Signed-off-by: Dan McGee --- packages/admin.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/admin.py b/packages/admin.py index 4680c755..5df0043a 100644 --- a/packages/admin.py +++ b/packages/admin.py @@ -15,10 +15,9 @@ class PackageRelationAdmin(admin.ModelAdmin): class FlagRequestAdmin(admin.ModelAdmin): list_display = ('pkgbase', 'full_version', 'repo', 'created', 'who', 'is_spam', 'is_legitimate', 'message') - list_filter = ('is_spam', 'is_legitimate', 'repo') + list_filter = ('is_spam', 'is_legitimate', 'repo', 'created') search_fields = ('pkgbase', 'user_email', 'message') ordering = ('-created',) - date_hierarchy = 'created' def get_queryset(self, request): qs = super(FlagRequestAdmin, self).queryset(request) @@ -28,19 +27,17 @@ def get_queryset(self, request): class SignoffAdmin(admin.ModelAdmin): list_display = ('pkgbase', 'full_version', 'arch', 'repo', 'user', 'created', 'revoked') - list_filter = ('arch', 'repo', 'user') + list_filter = ('arch', 'repo', 'user', 'created') search_fields = ('pkgbase', 'user__username') ordering = ('-created',) - date_hierarchy = 'created' class SignoffSpecificationAdmin(admin.ModelAdmin): list_display = ('pkgbase', 'full_version', 'arch', 'repo', 'user', 'created', 'comments') - list_filter = ('arch', 'repo', 'user') + list_filter = ('arch', 'repo', 'user', 'created') search_fields = ('pkgbase', 'user__username') ordering = ('-created',) - date_hierarchy = 'created' def get_queryset(self, request): qs = super(SignoffSpecificationAdmin, self).queryset(request) @@ -50,10 +47,9 @@ def get_queryset(self, request): class UpdateAdmin(admin.ModelAdmin): list_display = ('pkgname', 'repo', 'arch', 'action_flag', 'old_version', 'new_version', 'created') - list_filter = ('action_flag', 'repo', 'arch') + list_filter = ('action_flag', 'repo', 'arch', 'created') search_fields = ('pkgname',) ordering = ('-created',) - date_hierarchy = 'created' raw_id_fields = ('package',) -- cgit v1.2.3-54-g00ecf From 954b61ee3a0760990f8e17d7d692f3cabd949f9f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 27 Nov 2013 17:08:42 -0500 Subject: Syntax cleanups to main admin Signed-off-by: Dan McGee --- main/admin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/main/admin.py b/main/admin.py index 6aff12e5..ec2b5bc8 100644 --- a/main/admin.py +++ b/main/admin.py @@ -1,23 +1,27 @@ from django.contrib import admin from main.models import Arch, Donor, Package, Repo + class DonorAdmin(admin.ModelAdmin): list_display = ('name', 'visible', 'created') list_filter = ('visible', 'created') search_fields = ('name',) exclude = ('created',) + class ArchAdmin(admin.ModelAdmin): list_display = ('name', 'agnostic', 'required_signoffs') list_filter = ('agnostic',) search_fields = ('name',) + class RepoAdmin(admin.ModelAdmin): list_display = ('name', 'testing', 'staging', 'bugs_project', 'bugs_category', 'svn_root') list_filter = ('testing', 'staging') search_fields = ('name',) + class PackageAdmin(admin.ModelAdmin): list_display = ('pkgname', 'full_version', 'repo', 'arch', 'packager', 'last_update', 'build_date') -- cgit v1.2.3-54-g00ecf From 9adc2e53124daa6d13090166830396ffff9013d3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 27 Nov 2013 17:49:58 -0500 Subject: Stop using Django-provided floatformat template tag It turns out this is a HUGE part of our slow mirror status template rendering, due to the internal workings. Everything is converted to a Python decimal object which is way slower than just staying in native floating point. Given we are always dealing with floats when we need to do our formatting, a home-rolled template tag can accomplish this much faster. Signed-off-by: Dan McGee --- mirrors/templatetags/mirror_status.py | 11 ++++++++--- templates/mirrors/mirror_details.html | 6 +++--- templates/mirrors/mirrorlist_status.txt | 4 ++-- templates/mirrors/status_table.html | 6 +++--- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/mirrors/templatetags/mirror_status.py b/mirrors/templatetags/mirror_status.py index 9a363fbe..b3810d9a 100644 --- a/mirrors/templatetags/mirror_status.py +++ b/mirrors/templatetags/mirror_status.py @@ -1,6 +1,5 @@ from datetime import timedelta from django import template -from django.template.defaultfilters import floatformat register = template.Library() @@ -27,10 +26,16 @@ def hours(value): return '%d hours' % hrs @register.filter -def percentage(value, arg=-1): +def floatvalue(value, arg=2): + if value is None: + return u'' + return '%.*f' % (arg, value) + +@register.filter +def percentage(value, arg=1): if not value and type(value) != float: return u'' new_val = value * 100.0 - return floatformat(new_val, arg) + '%' + return '%.*f%%' % (arg, new_val) # vim: set ts=4 sw=4 et: diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index f2ffce20..e1962b85 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -110,9 +110,9 @@

Available URLs

{{ m_url.last_sync|date:'Y-m-d H:i'|default:'unknown' }} {{ m_url.completion_pct|percentage:1 }} {{ m_url.delay|duration|default:'unknown' }} - {{ m_url.duration_avg|floatformat:2 }} - {{ m_url.duration_stddev|floatformat:2 }} - {{ m_url.score|floatformat:1|default:'∞' }} + {{ m_url.duration_avg|floatvalue:2 }} + {{ m_url.duration_stddev|floatvalue:2 }} + {{ m_url.score|floatvalue:1|default:'∞' }} {% endfor %} diff --git a/templates/mirrors/mirrorlist_status.txt b/templates/mirrors/mirrorlist_status.txt index 575d19f7..c69075a3 100644 --- a/templates/mirrors/mirrorlist_status.txt +++ b/templates/mirrors/mirrorlist_status.txt @@ -1,4 +1,4 @@ -{% comment %} +{% load mirror_status %}{% comment %} Yes, ugly templates are ugly, but in order to keep line breaks where we want them, sacrifices have to be made. If editing this template, it is easiest to forget about where line breaks are happening until you are done getting the @@ -9,6 +9,6 @@ content right, and then go back later to fix it all up. ## Generated on {% now "Y-m-d" %} ## {% for mirror_url in mirror_urls %} -## Score: {{ mirror_url.score|floatformat:1|default:'unknown' }}, {{ mirror_url.country.name|default:'Worldwide' }} +## Score: {{ mirror_url.score|floatvalue:1|default:'unknown' }}, {{ mirror_url.country.name|default:'Worldwide' }} #Server = {{ mirror_url.url}}$repo/os/$arch{% endfor %} {% endautoescape %} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html index 6fc07a31..00b9c1df 100644 --- a/templates/mirrors/status_table.html +++ b/templates/mirrors/status_table.html @@ -20,9 +20,9 @@ {% country_flag m_url.country %}{{ m_url.country.name }} {{ m_url.completion_pct|percentage:1 }} {{ m_url.delay|duration|default:'unknown' }} - {{ m_url.duration_avg|floatformat:2 }} - {{ m_url.duration_stddev|floatformat:2 }} - {{ m_url.score|floatformat:1|default:'∞' }} + {{ m_url.duration_avg|floatvalue:2 }} + {{ m_url.duration_stddev|floatvalue:2 }} + {{ m_url.score|floatvalue:1|default:'∞' }} {% endfor %} -- cgit v1.2.3-54-g00ecf From deb030c1fd4aa11f079df92064c7b243c3ebda66 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 25 Nov 2013 21:58:41 -0600 Subject: Old TODO file is very out of date Signed-off-by: Dan McGee --- TODO | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 TODO diff --git a/TODO b/TODO deleted file mode 100644 index 608d8470..00000000 --- a/TODO +++ /dev/null @@ -1,4 +0,0 @@ -TODO: - - refactor stats by templates in dashboard, maybe a templatetag - - -- cgit v1.2.3-54-g00ecf From 2751e3870ea7875a67b42d025ca0369cf0a1bebf Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 14 Dec 2013 09:10:38 -0600 Subject: Bump Django requirement for minor version release Signed-off-by: Dan McGee --- requirements.txt | 2 +- requirements_prod.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7ff363e6..de8a04e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.6 +Django==1.6.1 IPy==0.81 Markdown==2.3.1 South==0.8.4 diff --git a/requirements_prod.txt b/requirements_prod.txt index 3f2958c6..e609c5d6 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,5 +1,5 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.6 +Django==1.6.1 IPy==0.81 Markdown==2.3.1 South==0.8.4 -- cgit v1.2.3-54-g00ecf From cd427ead7e71bb5d7716f015b1f9753ee56a9b4e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Dec 2013 23:17:07 -0600 Subject: Fix double space in template Signed-off-by: Dan McGee --- templates/mirrors/status.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 331c18ee..e97ad4ba 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -59,7 +59,7 @@

Mirror Status{% if tier != None %} - Tier {{ tier }}{% endif %}

Out of Sync Mirrors

- {% with urls=bad_urls table_id='outofsync_mirrors' %} + {% with urls=bad_urls table_id='outofsync_mirrors' %} {% include "mirrors/status_table.html" %} {% endwith %} -- cgit v1.2.3-54-g00ecf From 3b1b677b49af194313da766579e9fa1a021afd84 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Dec 2013 23:15:30 -0600 Subject: Speed up master key listing page We were spending a lot of time getting the developer name for a given key on this page, which involved sending one query per key ID. Use the cache to our advantage here and save ourselves the "expensive" lookups. This eliminates ~100 queries per page load. Signed-off-by: Dan McGee --- main/templatetags/pgp.py | 18 +++++++++++++----- templates/public/keys.html | 8 ++++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/main/templatetags/pgp.py b/main/templatetags/pgp.py index e93e5bca..cc080439 100644 --- a/main/templatetags/pgp.py +++ b/main/templatetags/pgp.py @@ -3,6 +3,7 @@ from django.utils.html import conditional_escape from django.utils.safestring import mark_safe +from ..utils import cache_function from devel.models import DeveloperKey register = template.Library() @@ -42,15 +43,22 @@ def pgp_key_link(key_id, link_text=None): values = (url, format_key(key_id), link_text) return '%s' % values -@register.simple_tag -def user_pgp_key_link(key_id): - normalized = key_id[-16:] + +@cache_function(1800) +def name_for_key(normalized): try: matching_key = DeveloperKey.objects.select_related( 'owner').get(key=normalized, owner_id__isnull=False) + return matching_key.owner.get_full_name() except DeveloperKey.DoesNotExist: - return pgp_key_link(key_id) - return pgp_key_link(key_id, matching_key.owner.get_full_name()) + return None + + +@register.simple_tag +def user_pgp_key_link(key_id): + normalized = key_id[-16:] + name = name_for_key(normalized) + return pgp_key_link(key_id, name) @register.filter(needs_autoescape=True) diff --git a/templates/public/keys.html b/templates/public/keys.html index c7272db3..54d52ab8 100644 --- a/templates/public/keys.html +++ b/templates/public/keys.html @@ -86,16 +86,16 @@

Master Key Signatures

- {% for user in active_users %} + {% for user in active_users %}{% with user_key=user.userprofile.pgp_key %} {{ user.get_full_name }} - {% pgp_key_link user.userprofile.pgp_key %} + {% pgp_key_link user_key %} {% spaceless %}{% for key in keys %} - {% signature_exists signatures key.pgp_key user.userprofile.pgp_key as signed %} + {% signature_exists signatures key.pgp_key user_key as signed %} {{ signed|yesno|capfirst }} {% endfor %}{% endspaceless %} - {% endfor %} + {% endwith %}{% endfor %} -- cgit v1.2.3-54-g00ecf From ecece25814042d262bb7a102b9cbe48fc9c87db4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Dec 2013 11:42:25 -0600 Subject: Show all mirror status data to authorized users Regardless of whether the mirror URL is active or not, we often have data we can show the end user, especially if mirror admins care to see the data we've been gathering. Signed-off-by: Dan McGee --- mirrors/utils.py | 20 +++++++++++--------- mirrors/views.py | 18 +++++++++++++----- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/mirrors/utils.py b/mirrors/utils.py index 633731a7..9f40bca6 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -113,18 +113,17 @@ def annotate_url(url, url_data): url.score = (hours + url.duration_avg + stddev) / divisor -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): cutoff_time = now() - cutoff - # TODO: this prevents grabbing data points from any mirror that was active, - # receiving checks, and then marked private. we can probably be smarter and - # filter the data later? - valid_urls = MirrorUrl.objects.filter(active=True, - mirror__active=True, mirror__public=True, + valid_urls = MirrorUrl.objects.filter( logs__check_time__gte=cutoff_time).distinct() if mirror_id: valid_urls = valid_urls.filter(mirror_id=mirror_id) + if not show_all: + valid_urls = valid_urls.filter(active=True, mirror__active=True, + mirror__public=True) url_data = status_data(cutoff_time, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( @@ -159,11 +158,11 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): } -def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( - is_success=False, check_time__gte=cutoff_time, url__active=True, - url__mirror__active=True, url__mirror__public=True).values( + is_success=False, check_time__gte=cutoff_time, + url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') @@ -171,6 +170,9 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): if mirror_id: errors = errors.filter(url__mirror_id=mirror_id) + if not show_all: + errors = errors.filter(url__active=True, url__mirror__active=True, + url__mirror__public=True) errors = list(errors) for err in errors: diff --git a/mirrors/views.py b/mirrors/views.py index ec056696..9e05e5fc 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -153,15 +153,20 @@ def mirrors(request): def mirror_details(request, name): mirror = get_object_or_404(Mirror, name=name) - if not request.user.is_authenticated() and \ + authorized = request.user.is_authenticated() + if not authorized and \ (not mirror.public or not mirror.active): raise Http404 error_cutoff = timedelta(days=7) - status_info = get_mirror_statuses(mirror_id=mirror.id) + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} - all_urls = set(mirror.urls.filter(active=True).select_related('protocol')) + all_urls = mirror.urls.select_related('protocol') + if not authorized: + all_urls = all_urls.filter(active=True) + all_urls = set(all_urls) # Add dummy data for URLs that we haven't checked recently other_urls = all_urls.difference(checked_urls) for url in other_urls: @@ -170,7 +175,8 @@ def mirror_details(request, name): setattr(url, attr, None) all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) - error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff) + error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, + show_all=True) context = { 'mirror': mirror, @@ -181,8 +187,10 @@ def mirror_details(request, name): return render(request, 'mirrors/mirror_details.html', context) def mirror_details_json(request, name): + authorized = request.user.is_authenticated() mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses(mirror_id=mirror.id) + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, -- cgit v1.2.3-54-g00ecf From d83e0842053b3a4a4dfb31f373d34ecf1e0c5ce2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 14 Dec 2013 11:15:42 -0600 Subject: Add Flyspray Bug field to mirror model Signed-off-by: Dan McGee --- mirrors/admin.py | 3 +- .../migrations/0027_auto__add_field_mirror_bug.py | 83 ++++++++++++++++++++++ mirrors/models.py | 1 + 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 mirrors/migrations/0027_auto__add_field_mirror_bug.py diff --git a/mirrors/admin.py b/mirrors/admin.py index e35d9ce7..17365486 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -54,7 +54,7 @@ class Meta: model = Mirror fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email', 'public', 'active', 'isos', 'rsync_user', 'rsync_password', - 'notes') + 'bug', 'notes') upstream = forms.ModelChoiceField( queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), @@ -67,6 +67,7 @@ class MirrorAdmin(admin.ModelAdmin): 'isos', 'admin_email', 'alternate_email') list_filter = ('tier', 'active', 'public') search_fields = ('name', 'admin_email', 'alternate_email') + readonly_fields = ('created',) inlines = [ MirrorUrlInlineAdmin, MirrorRsyncInlineAdmin, diff --git a/mirrors/migrations/0027_auto__add_field_mirror_bug.py b/mirrors/migrations/0027_auto__add_field_mirror_bug.py new file mode 100644 index 00000000..57727333 --- /dev/null +++ b/mirrors/migrations/0027_auto__add_field_mirror_bug.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + def forwards(self, orm): + db.add_column(u'mirrors_mirror', 'bug', + self.gf('django.db.models.fields.PositiveIntegerField')(null=True), + keep_default=False) + + def backwards(self, orm): + db.delete_column(u'mirrors_mirror', 'bug') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'bug': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'ordering': "('ip',)", 'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index da3d8c0d..47e2051b 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -28,6 +28,7 @@ class Mirror(models.Model): isos = models.BooleanField("ISOs", default=True) rsync_user = models.CharField(max_length=50, blank=True, default='') rsync_password = models.CharField(max_length=50, blank=True, default='') + bug = models.PositiveIntegerField("Flyspray bug", null=True, blank=True) notes = models.TextField(blank=True) created = models.DateTimeField(editable=False) -- cgit v1.2.3-54-g00ecf From a116f0d94221f72fa14d90ec77b9777efbfada65 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 14 Dec 2013 11:24:57 -0600 Subject: Add update query for extracting Flyspray bug number Signed-off-by: Dan McGee --- mirrors/migrations/0027_auto__add_field_mirror_bug.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mirrors/migrations/0027_auto__add_field_mirror_bug.py b/mirrors/migrations/0027_auto__add_field_mirror_bug.py index 57727333..f7304ba8 100644 --- a/mirrors/migrations/0027_auto__add_field_mirror_bug.py +++ b/mirrors/migrations/0027_auto__add_field_mirror_bug.py @@ -9,6 +9,14 @@ def forwards(self, orm): db.add_column(u'mirrors_mirror', 'bug', self.gf('django.db.models.fields.PositiveIntegerField')(null=True), keep_default=False) + # UPDATE mirrors_mirror m + # SET bug = ( + # SELECT extracted::int FROM ( + # SELECT id, substring(notes from 'FS#([\d]+)') AS extracted FROM mirrors_mirror + # ) a + # WHERE extracted IS NOT NULL AND a.id = m.id + # ) + # WHERE notes LIKE '%FS#%'; def backwards(self, orm): db.delete_column(u'mirrors_mirror', 'bug') -- cgit v1.2.3-54-g00ecf From 97ca3f7286a533c7a5e7935c034578c06235571a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 14 Dec 2013 11:31:40 -0600 Subject: Add Flyspray Issue to mirror details template Signed-off-by: Dan McGee --- templates/mirrors/mirror_details.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index e1962b85..3770eae5 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -53,6 +53,10 @@

Mirror Details: {{ mirror.name }}

Alternate Email: {% if mirror.alternate_email %}{{ mirror.alternate_email }}{% else %}None{% endif %} + + Flyspray Issue: + {% if mirror.bug %}FS#{{ mirror.bug }}{% endif %} + Notes: {{ mirror.notes|linebreaks }} -- cgit v1.2.3-54-g00ecf From 5d74a99c6b8fbbd19ad441a74d835382025da522 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 14 Dec 2013 11:38:20 -0600 Subject: Don't list downstream mirrors one per line This looks really crazy on our current Tier 1 mirrors, as some of them have tons of downstream mirrors. Instead, do what we did on the package details page and allow wrapping of the comma-separated list. Signed-off-by: Dan McGee --- templates/mirrors/mirror_details.html | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index 3770eae5..aa0a9648 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -71,14 +71,13 @@

Mirror Details: {{ mirror.name }}

Downstream: {% with mirror.downstream as ds_mirrors %} - {% if ds_mirrors %} + {% if ds_mirrors %} {% for ds in ds_mirrors %} {{ ds.name }} - {% if not ds.active %}(inactive){% endif %} - {% if not ds.public %}(private){% endif %} -
- {% endfor %} + title="Mirror details for {{ ds.name }}">{{ ds.name }}{% comment %} + {% endcomment %}{% if not ds.active %} (inactive){% endif %}{% comment %} + {% endcomment %}{% if not ds.public %} (private){% endif %}{% comment %} + {% endcomment %}{% if not forloop.last %}, {% endif %}{% endfor %} {% else %}None{% endif %} {% endwith %} -- cgit v1.2.3-54-g00ecf From 0b0e2b9d2aabd1c34f742e6525ee075751600e37 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 14 Dec 2013 15:06:26 -0600 Subject: Fix some caching issues on the front page The return value from get_recent_updates() was too big for memcached due to all the attached objects, so the cache never actually worked. This sucks, because we ended up doing all the work in this function and most of the time we didn't use it because template fragment caching kicked in. Remove the cache_function decorator from this method, and instead implement delayed calling of the function so we don't compute values we aren't going to use. Template fragment caching will help us in most cases. Signed-off-by: Dan McGee --- public/utils.py | 3 +-- public/views.py | 8 +++++--- templates/public/index.html | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/public/utils.py b/public/utils.py index fcfd0f77..11091883 100644 --- a/public/utils.py +++ b/public/utils.py @@ -2,7 +2,7 @@ from operator import attrgetter from main.models import Arch, Repo, Package -from main.utils import cache_function, groupby_preserve_order, PackageStandin +from main.utils import groupby_preserve_order, PackageStandin class RecentUpdate(object): def __init__(self, packages): @@ -58,7 +58,6 @@ def __unicode__(self): return "RecentUpdate '%s %s' <%d packages>" % ( self.pkgbase, self.version, len(self.packages)) -@cache_function(62) def get_recent_updates(number=15, testing=True, staging=False): repos = Repo.objects.all() if not testing: diff --git a/public/views.py b/public/views.py index f79c8f32..3b23bd42 100644 --- a/public/views.py +++ b/public/views.py @@ -20,12 +20,14 @@ @cache_control(max_age=300) def index(request): if request.user.is_authenticated(): - pkgs = get_recent_updates(testing=True, staging=True) + def updates(): + return get_recent_updates(testing=True, staging=True) else: - pkgs = get_recent_updates() + def updates(): + return get_recent_updates() context = { 'news_updates': News.objects.order_by('-postdate', '-id')[:15], - 'pkg_updates': pkgs, + 'pkg_updates': updates, } return render(request, 'public/index.html', context) diff --git a/templates/public/index.html b/templates/public/index.html index c5924ab7..58dd3729 100644 --- a/templates/public/index.html +++ b/templates/public/index.html @@ -105,7 +105,7 @@

Recent Updates ( {% endcache %} -{% cache 59 main-page-right secure %} +{% cache 115 main-page-right secure %}