From e65d2bc6db279240a03fece91e5332f8602859dc Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Dec 2011 19:43:24 -0600 Subject: Switch back to using standard deviation in mirror check page This got checked in by default, whoops. Signed-off-by: Dan McGee --- mirrors/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 8518b3ba..686ec581 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -40,8 +40,7 @@ def get_mirror_statuses(cutoff=default_cutoff): last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), duration_avg=Avg('logs__duration'), - #duration_stddev=StdDev('logs__duration') - duration_stddev=Max('logs__duration') + duration_stddev=StdDev('logs__duration') ).order_by('-last_sync', '-duration_avg') # The Django ORM makes it really hard to get actual average delay in the -- cgit v1.2.3-54-g00ecf From a60801bb7dbc18080e7f6106bcf9c707d2801c9d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Dec 2011 19:52:27 -0600 Subject: PyLint suggested cleanups Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 2 +- devel/tests.py | 2 +- devel/views.py | 18 +++++++++--------- main/models.py | 6 ++---- mirrors/templatetags/mirror_status.py | 4 ++-- packages/management/commands/populate_signoffs.py | 1 - packages/models.py | 5 ++++- packages/templatetags/package_extras.py | 5 +++-- packages/utils.py | 2 +- public/views.py | 2 +- 10 files changed, 24 insertions(+), 23 deletions(-) (limited to 'mirrors') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index cc536b15..7e058afc 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -24,7 +24,6 @@ from optparse import make_option from django.core.management.base import BaseCommand, CommandError -from django.contrib.auth.models import User from django.db import connections, router, transaction from django.db.utils import IntegrityError @@ -91,6 +90,7 @@ def __init__(self, repo): setattr(self, k, None) for k in self.collections: setattr(self, k, ()) + self.builddate = None self.files = None self.has_files = False diff --git a/devel/tests.py b/devel/tests.py index 36691179..01eed0fc 100644 --- a/devel/tests.py +++ b/devel/tests.py @@ -99,7 +99,7 @@ def test_by_invalid(self): def test_cache(self): # simply look two of them up, but then do it repeatedly - for i in range(50): + for _ in range(5): self.assertEqual(self.user1, self.finder.find("XXX YYY ")) self.assertEqual(self.user3, diff --git a/devel/views.py b/devel/views.py index 79272c98..2e62200e 100644 --- a/devel/views.py +++ b/devel/views.py @@ -145,7 +145,7 @@ def change_profile(request): {'form': form, 'profile_form': profile_form}) @login_required -def report(request, report, username=None): +def report(request, report_name, username=None): title = 'Developer Report' packages = Package.objects.normal() names = attrs = user = None @@ -159,17 +159,17 @@ def report(request, report, username=None): maints = User.objects.filter(id__in=PackageRelation.objects.filter( type=PackageRelation.MAINTAINER).values('user')) - if report == 'old': + if report_name == 'old': title = 'Packages last built more than two years ago' cutoff = datetime.utcnow() - timedelta(days=365 * 2) packages = packages.filter( build_date__lt=cutoff).order_by('build_date') - elif report == 'long-out-of-date': + elif report_name == 'long-out-of-date': title = 'Packages marked out-of-date more than 90 days ago' cutoff = datetime.utcnow() - timedelta(days=90) packages = packages.filter( flag_date__lt=cutoff).order_by('flag_date') - elif report == 'big': + elif report_name == 'big': title = 'Packages with compressed size > 50 MiB' cutoff = 50 * 1024 * 1024 packages = packages.filter( @@ -182,7 +182,7 @@ def report(request, report, username=None): package.compressed_size) package.installed_size_pretty = filesizeformat( package.installed_size) - elif report == 'badcompression': + elif report_name == 'badcompression': title = 'Packages that have little need for compression' cutoff = 0.90 * F('installed_size') packages = packages.filter(compressed_size__gt=0, installed_size__gt=0, @@ -199,7 +199,7 @@ def report(request, report, username=None): ratio = package.compressed_size / float(package.installed_size) package.ratio = '%.2f' % ratio package.compress_type = package.filename.split('.')[-1] - elif report == 'uncompressed-man': + elif report_name == 'uncompressed-man': title = 'Packages with uncompressed manpages' # checking for all '.0'...'.9' + '.n' extensions bad_files = PackageFile.objects.filter(directory__contains='/man/', @@ -209,7 +209,7 @@ def report(request, report, username=None): bad_files = bad_files.filter(pkg__in=pkg_ids) bad_files = bad_files.values_list('pkg_id', flat=True).distinct() packages = packages.filter(id__in=set(bad_files)) - elif report == 'uncompressed-info': + elif report_name == 'uncompressed-info': title = 'Packages with uncompressed infopages' # we don't worry about looking for '*.info-1', etc., given that an # uncompressed root page probably exists in the package anyway @@ -220,7 +220,7 @@ def report(request, report, username=None): bad_files = bad_files.filter(pkg__in=pkg_ids) bad_files = bad_files.values_list('pkg_id', flat=True).distinct() packages = packages.filter(id__in=set(bad_files)) - elif report == 'unneeded-orphans': + elif report_name == 'unneeded-orphans': title = 'Orphan packages required by no other packages' owned = PackageRelation.objects.all().values('pkgbase') required = PackageDepend.objects.all().values('depname') @@ -273,7 +273,7 @@ def clean_username(self): def save(self, commit=True): profile = super(NewUserForm, self).save(False) pwletters = ascii_letters + digits - password = ''.join([random.choice(pwletters) for i in xrange(8)]) + password = ''.join([random.choice(pwletters) for _ in xrange(8)]) user = User.objects.create_user(username=self.cleaned_data['username'], email=self.cleaned_data['private_email'], password=password) user.first_name = self.cleaned_data['first_name'] diff --git a/main/models.py b/main/models.py index cefebf76..c54e2972 100644 --- a/main/models.py +++ b/main/models.py @@ -1,8 +1,6 @@ from django.db import models -from django.db.models.signals import pre_save from django.contrib.auth.models import User from django.contrib.sites.models import Site -from django.forms import ValidationError from .fields import PositiveBigIntegerField, PGPKeyField from .utils import cache_function, make_choice, set_created_field @@ -229,7 +227,7 @@ def get_requiredby(self): pkg__arch__in=self.applicable_arches()) # sort out duplicate packages; this happens if something has a double # versioned dep such as a kernel module - requiredby = [list(vals)[0] for k, vals in + requiredby = [list(vals)[0] for _, vals in groupby(requiredby, lambda x: x.pkg.id)] # find another package by this name in the opposite testing setup @@ -244,7 +242,7 @@ def get_requiredby(self): # for each unique package name, try to screen our package list down to # those packages in the same testing category (yes or no) iff there is # a package in the same testing category. - for name, dep_pkgs in groupby(requiredby, lambda x: x.pkg.pkgname): + for _, dep_pkgs in groupby(requiredby, lambda x: x.pkg.pkgname): dep_pkgs = list(dep_pkgs) dep = dep_pkgs[0] if len(dep_pkgs) > 1: diff --git a/mirrors/templatetags/mirror_status.py b/mirrors/templatetags/mirror_status.py index 0031d83b..9a363fbe 100644 --- a/mirrors/templatetags/mirror_status.py +++ b/mirrors/templatetags/mirror_status.py @@ -10,7 +10,7 @@ def duration(value): return u'' # does not take microseconds into account total_secs = value.seconds + value.days * 24 * 3600 - mins, secs = divmod(total_secs, 60) + mins = total_secs // 60 hrs, mins = divmod(mins, 60) return '%d:%02d' % (hrs, mins) @@ -20,7 +20,7 @@ def hours(value): return u'' # does not take microseconds into account total_secs = value.seconds + value.days * 24 * 3600 - mins, secs = divmod(total_secs, 60) + mins = total_secs // 60 hrs, mins = divmod(mins, 60) if hrs == 1: return '%d hour' % hrs diff --git a/packages/management/commands/populate_signoffs.py b/packages/management/commands/populate_signoffs.py index 42496e9d..97ba4146 100644 --- a/packages/management/commands/populate_signoffs.py +++ b/packages/management/commands/populate_signoffs.py @@ -15,7 +15,6 @@ from xml.etree.ElementTree import XML from django.conf import settings -from django.contrib.auth.models import User from django.core.management.base import NoArgsCommand from ...models import SignoffSpecification diff --git a/packages/models.py b/packages/models.py index 77cade68..17843891 100644 --- a/packages/models.py +++ b/packages/models.py @@ -1,7 +1,7 @@ from collections import namedtuple from django.db import models -from django.db.models.signals import pre_save, post_save +from django.db.models.signals import pre_save from django.contrib.auth.models import User from main.models import Arch, Repo @@ -167,6 +167,9 @@ def __unicode__(self): class FlagRequest(models.Model): + ''' + A notification the package is out-of-date submitted through the web site. + ''' user = models.ForeignKey(User, blank=True, null=True) user_email = models.EmailField('email address') created = models.DateTimeField(editable=False) diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index 3eb7578d..47b2fecf 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -27,6 +27,7 @@ def url_unquote(original_url): class BuildQueryStringNode(template.Node): def __init__(self, sortfield): self.sortfield = sortfield + super(BuildQueryStringNode, self).__init__() def render(self, context): qs = parse_qs(context['current_query']) @@ -53,8 +54,8 @@ def do_buildsortqs(parser, token): @register.simple_tag def pkg_details_link(pkg): - template = '%s' - return template % (pkg.get_absolute_url(), pkg.pkgname, pkg.pkgname) + link = '%s' + return link % (pkg.get_absolute_url(), pkg.pkgname, pkg.pkgname) @register.simple_tag def multi_pkg_details(pkgs): diff --git a/packages/utils.py b/packages/utils.py index 5703db3b..d4a9d8f6 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -377,7 +377,7 @@ def get_target_repo_map(repos): AND r.testing = %s AND p2.repo_id IN ( """ - sql += ','.join(['%s' for r in repos]) + sql += ','.join(['%s' for _ in repos]) sql += ")" params = [False, False] diff --git a/public/views.py b/public/views.py index 8fb60c78..bcae4bd7 100644 --- a/public/views.py +++ b/public/views.py @@ -8,7 +8,7 @@ from main.models import Arch, Repo, Donor from mirrors.models import MirrorUrl from news.models import News -from utils import get_recent_updates +from .utils import get_recent_updates def index(request): pkgs = get_recent_updates() -- cgit v1.2.3-54-g00ecf From 6b16b9487a95118a6109a2c5119d430dc1192e80 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 5 Jan 2012 13:03:00 -0600 Subject: Adjust page and content caching lengths and decorators Remove never_cache from many places now that we don't actually need it since we aren't caching by default. Adjust our cache_function decorator times be shorter values, and also randomize them a bit to make cache invalidations not all line up. Signed-off-by: Dan McGee --- devel/views.py | 14 ++++++++++---- main/models.py | 8 ++++---- mirrors/utils.py | 4 ++-- packages/utils.py | 4 ++-- packages/views/__init__.py | 2 -- packages/views/signoff.py | 2 -- public/utils.py | 2 +- todolists/views.py | 2 -- 8 files changed, 19 insertions(+), 19 deletions(-) (limited to 'mirrors') diff --git a/devel/views.py b/devel/views.py index 2e62200e..35918d66 100644 --- a/devel/views.py +++ b/devel/views.py @@ -13,6 +13,7 @@ from django.template.defaultfilters import filesizeformat from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template +from django.utils.http import http_date from main.models import Package, PackageDepend, PackageFile, TodolistPkg from main.models import Arch, Repo @@ -27,9 +28,9 @@ import pytz import random from string import ascii_letters, digits +import time @login_required -@never_cache def index(request): '''the developer dashboard''' if(request.user.is_authenticated()): @@ -80,7 +81,6 @@ def index(request): return direct_to_template(request, 'devel/index.html', page_dict) @login_required -@never_cache def clock(request): devs = User.objects.filter(is_active=True).order_by( 'first_name', 'last_name').select_related('userprofile') @@ -98,7 +98,14 @@ def clock(request): 'utc_now': utc_now, } - return direct_to_template(request, 'devel/clock.html', page_dict) + response = direct_to_template(request, 'devel/clock.html', page_dict) + if not response.has_header('Expires'): + # why this works only with the non-UTC date I have no idea... + expire_time = now.replace(minute=utc_now.minute + 1, + second=0, microsecond=0) + expire_time = time.mktime(expire_time.timetuple()) + response['Expires'] = http_date(expire_time) + return response class ProfileForm(forms.Form): email = forms.EmailField(label='Private email (not shown publicly):', @@ -342,7 +349,6 @@ def inner_save(): return direct_to_template(request, 'general_form.html', context) @user_passes_test(lambda u: u.is_superuser) -@never_cache def admin_log(request, username=None): user = None if username: diff --git a/main/models.py b/main/models.py index c54e2972..d72f2c05 100644 --- a/main/models.py +++ b/main/models.py @@ -201,14 +201,14 @@ def maintainers(self): def maintainers(self, maintainers): self._maintainers = maintainers - @cache_function(300) + @cache_function(1800) def applicable_arches(self): '''The list of (this arch) + (available agnostic arches).''' arches = set(Arch.objects.filter(agnostic=True)) arches.add(self.arch) return list(arches) - @cache_function(300) + @cache_function(119) def get_requiredby(self): """ Returns a list of package objects. An attempt will be made to keep this @@ -254,7 +254,7 @@ def get_requiredby(self): trimmed.append(dep) return trimmed - @cache_function(300) + @cache_function(121) def get_depends(self): """ Returns a list of dicts. Each dict contains ('dep', 'pkg', and @@ -279,7 +279,7 @@ def get_depends(self): deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) return deps - @cache_function(300) + @cache_function(125) def base_package(self): """ Locate the base package for this package. It may be this very package, diff --git a/mirrors/utils.py b/mirrors/utils.py index 686ec581..f05ffc77 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -26,7 +26,7 @@ def annotate_url(url, delays): url.delay = None url.score = None -@cache_function(300) +@cache_function(123) def get_mirror_statuses(cutoff=default_cutoff): cutoff_time = datetime.datetime.utcnow() - cutoff protocols = list(MirrorProtocol.objects.filter(is_download=True)) @@ -80,7 +80,7 @@ def get_mirror_statuses(cutoff=default_cutoff): 'urls': urls, } -@cache_function(300) +@cache_function(117) def get_mirror_errors(cutoff=default_cutoff): cutoff_time = datetime.datetime.utcnow() - cutoff errors = MirrorLog.objects.filter( diff --git a/packages/utils.py b/packages/utils.py index d4a9d8f6..2a43a08b 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -11,7 +11,7 @@ from .models import (PackageGroup, PackageRelation, SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) -@cache_function(300) +@cache_function(127) def get_group_info(include_arches=None): raw_groups = PackageGroup.objects.values_list( 'name', 'pkg__arch__name').order_by('name').annotate( @@ -92,7 +92,7 @@ def __cmp__(self, other): return False -@cache_function(300) +@cache_function(127) def get_differences_info(arch_a, arch_b): # This is a monster. Join packages against itself, looking for packages in # our non-'any' architectures only, and not having a corresponding package diff --git a/packages/views/__init__.py b/packages/views/__init__.py index 9f24056a..5be4833e 100644 --- a/packages/views/__init__.py +++ b/packages/views/__init__.py @@ -5,7 +5,6 @@ from django.http import HttpResponse, Http404 from django.shortcuts import get_object_or_404, redirect from django.utils import simplejson -from django.views.decorators.cache import never_cache from django.views.decorators.http import require_POST from django.views.decorators.vary import vary_on_headers from django.views.generic.simple import direct_to_template @@ -247,7 +246,6 @@ def arch_differences(request): return direct_to_template(request, 'packages/differences.html', context) @permission_required('main.change_package') -@never_cache def stale_relations(request): relations = PackageRelation.objects.select_related('user') pkgbases = Package.objects.all().values('pkgbase') diff --git a/packages/views/signoff.py b/packages/views/signoff.py index e57b4d9a..e3daf0dd 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -18,7 +18,6 @@ PackageSignoffGroup) @permission_required('main.change_package') -@never_cache def signoffs(request): signoff_groups = sorted(get_signoff_groups(), key=attrgetter('pkgbase')) for group in signoff_groups: @@ -178,7 +177,6 @@ def default(self, obj): return super(SignoffJSONEncoder, self).default(obj) @permission_required('main.change_package') -@never_cache def signoffs_json(request): signoff_groups = sorted(get_signoff_groups(), key=attrgetter('pkgbase')) data = { diff --git a/public/utils.py b/public/utils.py index 6566b8c4..a40c9b53 100644 --- a/public/utils.py +++ b/public/utils.py @@ -49,7 +49,7 @@ def package_links(self): if package.arch not in arches and not arches.add(package.arch): yield PackageStandin(package) -@cache_function(300) +@cache_function(62) def get_recent_updates(number=15): # This is a bit of magic. We are going to show 15 on the front page, but we # want to try and eliminate cross-architecture wasted space. Pull enough diff --git a/todolists/views.py b/todolists/views.py index d413ca47..e5cc0823 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -47,7 +47,6 @@ def flag(request, list_id, pkg_id): return redirect(todolist) @login_required -@never_cache def view(request, list_id): todolist = get_object_or_404(Todolist, id=list_id) svn_roots = Repo.objects.order_by().values_list( @@ -71,7 +70,6 @@ def list_pkgbases(request, list_id, svn_root): mimetype='text/plain') @login_required -@never_cache def todolist_list(request): lists = get_annotated_todolists() return direct_to_template(request, 'todolists/list.html', {'lists': lists}) -- cgit v1.2.3-54-g00ecf