From 31d39e75eea7fb6cdf3bb8bfd8b490d45de04ee9 Mon Sep 17 00:00:00 2001
From: Dan McGee
Date: Tue, 16 Apr 2013 21:59:32 -0500
Subject: Add shortcut for HEAD requests on slower views
We sometimes see some web bots and crawlers make HEAD requests to verify
existence of certain pages in the application. However, they are less
than kind as 20-50 requests might arrive at the same time, and package
search and details pages are some of the slowest rendering pages we have
due to the Django template engine.
Rather than waste time generating the content only to throw it away,
response as soon as we can with either a 404 or 200 response as
appropriate, omitting the 'Content-Length' header completely, which
seems to be acceptable by the HTTP spec.
Signed-off-by: Dan McGee
---
main/utils.py | 9 +++++++++
1 file changed, 9 insertions(+)
(limited to 'main/utils.py')
diff --git a/main/utils.py b/main/utils.py
index cdd4ff71..8394e5cd 100644
--- a/main/utils.py
+++ b/main/utils.py
@@ -8,6 +8,7 @@
from django.core.cache import cache
from django.db import connections, router
+from django.http import HttpResponse
from django.utils.timezone import now
from django.template.defaultfilters import slugify
@@ -55,6 +56,14 @@ def clear_cache_function(func, args, kwargs):
cache.delete(key)
+def empty_response():
+ empty = HttpResponse('')
+ # designating response as 'streaming' forces ConditionalGetMiddleware to
+ # not add a 'Content-Length: 0' header
+ empty.streaming = True
+ return empty
+
+
def format_http_headers(request):
headers = sorted((k, v) for k, v in request.META.items()
if k.startswith('HTTP_'))
--
cgit v1.2.3-54-g00ecf
From b7b24740640e24883cd17fd683e1d465fbb343f8 Mon Sep 17 00:00:00 2001
From: Dan McGee
Date: Tue, 16 Apr 2013 22:12:01 -0500
Subject: Various minor code cleanups and fixes
Most of these were suggested by PyCharm, and include everything from
little syntax issues and other bad smells to dead or bad code.
Signed-off-by: Dan McGee
---
devel/management/commands/pgp_import.py | 1 +
devel/models.py | 1 -
devel/utils.py | 2 +-
devel/views.py | 2 +-
main/log.py | 1 -
main/migrations/0029_fill_in_repo_data.py | 1 -
main/models.py | 12 +++++-------
main/utils.py | 1 -
mirrors/management/commands/mirrorcheck.py | 12 +++---------
mirrors/models.py | 2 +-
mirrors/utils.py | 6 +++---
packages/migrations/0002_populate_package_relation.py | 2 --
packages/templatetags/package_extras.py | 4 ++--
packages/utils.py | 2 +-
packages/views/display.py | 2 --
packages/views/flag.py | 3 +--
public/views.py | 1 -
releng/management/commands/syncisos.py | 2 +-
releng/models.py | 2 +-
releng/views.py | 2 +-
retro/templates/retro/index-20030330.html | 1 -
sitestatic/archweb.js | 1 -
todolists/utils.py | 1 -
todolists/views.py | 1 -
visualize/static/visualize.js | 2 +-
25 files changed, 23 insertions(+), 44 deletions(-)
(limited to 'main/utils.py')
diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py
index 10e6cfcb..b1f29d77 100644
--- a/devel/management/commands/pgp_import.py
+++ b/devel/management/commands/pgp_import.py
@@ -95,6 +95,7 @@ def parse_keydata(data):
# parse all of the output from our successful GPG command
logger.info("parsing command output")
+ node = None
for line in data.split('\n'):
parts = line.split(':')
if parts[0] == 'pub':
diff --git a/devel/models.py b/devel/models.py
index 67de40a6..4354e0f2 100644
--- a/devel/models.py
+++ b/devel/models.py
@@ -4,7 +4,6 @@
from django.db import models
from django.db.models.signals import pre_save
from django.contrib.auth.models import User
-from django.utils.timezone import now
from django_countries import CountryField
from .fields import PGPKeyField
diff --git a/devel/utils.py b/devel/utils.py
index e8e3a6c4..340841f5 100644
--- a/devel/utils.py
+++ b/devel/utils.py
@@ -131,7 +131,7 @@ def find(self, userstring):
self.username_email, self.user_name)
for matcher in find_methods:
user = matcher(name, email)
- if user != None:
+ if user is not None:
break
self.cache[userstring] = user
diff --git a/devel/views.py b/devel/views.py
index 61c1e568..4258ea7f 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -34,7 +34,7 @@
@login_required
def index(request):
'''the developer dashboard'''
- if(request.user.is_authenticated()):
+ if request.user.is_authenticated():
inner_q = PackageRelation.objects.filter(user=request.user)
else:
inner_q = PackageRelation.objects.none()
diff --git a/main/log.py b/main/log.py
index 63634874..5c745cc8 100644
--- a/main/log.py
+++ b/main/log.py
@@ -46,7 +46,6 @@ def filter(self, record):
trace = '\n'.join(traceback.format_exception(*record.exc_info))
key = md5(trace).hexdigest()
- duplicate = False
cache = self.cache_module.cache
# Test if the cache works
diff --git a/main/migrations/0029_fill_in_repo_data.py b/main/migrations/0029_fill_in_repo_data.py
index 0887b28c..7da6b1c4 100644
--- a/main/migrations/0029_fill_in_repo_data.py
+++ b/main/migrations/0029_fill_in_repo_data.py
@@ -7,7 +7,6 @@
class Migration(DataMigration):
def forwards(self, orm):
- "Write your forwards methods here."
orm.Repo.objects.filter(name__istartswith='community').update(bugs_project=5, svn_root='community')
orm.Repo.objects.filter(name__iexact='multilib').update(bugs_project=5, svn_root='community')
diff --git a/main/models.py b/main/models.py
index 89215f05..24aeed89 100644
--- a/main/models.py
+++ b/main/models.py
@@ -7,7 +7,6 @@
from django.db.models import Q
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
-from django.utils.timezone import now
from .fields import PositiveBigIntegerField
from .utils import set_created_field
@@ -140,7 +139,7 @@ def get_full_url(self, proto='https'):
@property
def signature(self):
try:
- data = b64decode(self.pgp_signature)
+ data = b64decode(self.pgp_signature.encode('utf-8'))
except TypeError:
return None
if not data:
@@ -274,7 +273,6 @@ def get_depends(self):
Packages will match the testing status of this package if possible.
"""
deps = []
- arches = None
# TODO: we can use list comprehension and an 'in' query to make this
# more effective
for dep in self.depends.all():
@@ -400,13 +398,13 @@ def elsewhere(self):
'''attempt to locate this package anywhere else, regardless of
architecture or repository. Excludes this package from the list.'''
names = [self.pkgname]
- if self.pkgname.startswith('lib32-'):
+ if self.pkgname.startswith(u'lib32-'):
names.append(self.pkgname[6:])
- elif self.pkgname.endswith('-multilib'):
+ elif self.pkgname.endswith(u'-multilib'):
names.append(self.pkgname[:-9])
else:
- names.append('lib32-' + self.pkgname)
- names.append(self.pkgname + '-multilib')
+ names.append(u'lib32-' + self.pkgname)
+ names.append(self.pkgname + u'-multilib')
return Package.objects.normal().filter(
pkgname__in=names).exclude(id=self.id).order_by(
'arch__name', 'repo__name')
diff --git a/main/utils.py b/main/utils.py
index 8394e5cd..9ee8db58 100644
--- a/main/utils.py
+++ b/main/utils.py
@@ -3,7 +3,6 @@
except ImportError:
import pickle
-from datetime import datetime
import hashlib
from django.core.cache import cache
diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py
index d6de8f22..e7dd7b49 100644
--- a/mirrors/management/commands/mirrorcheck.py
+++ b/mirrors/management/commands/mirrorcheck.py
@@ -106,19 +106,13 @@ def parse_lastsync(log, data):
def check_mirror_url(mirror_url, location, timeout):
- if location:
- if location.family == socket.AF_INET6:
- ipopt = '--ipv6'
- elif location.family == socket.AF_INET:
- ipopt = '--ipv4'
-
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s", url)
log = MirrorLog(url=mirror_url, check_time=now(), location=location)
headers = {'User-Agent': 'archweb/1.0'}
req = urllib2.Request(url, None, headers)
+ start = time.time()
try:
- start = time.time()
result = urllib2.urlopen(req, timeout=timeout)
data = result.read()
result.close()
@@ -147,12 +141,12 @@ def check_mirror_url(mirror_url, location, timeout):
elif isinstance(e.reason, socket.error):
log.error = e.reason.args[1]
logger.debug("failed: %s, %s", url, log.error)
- except HTTPException as e:
+ except HTTPException:
# e.g., BadStatusLine
log.is_success = False
log.error = "Exception in processing HTTP request."
logger.debug("failed: %s, %s", url, log.error)
- except socket.timeout as e:
+ except socket.timeout:
log.is_success = False
log.error = "Connection timed out."
logger.debug("failed: %s, %s", url, log.error)
diff --git a/mirrors/models.py b/mirrors/models.py
index 791b0078..d8ac7952 100644
--- a/mirrors/models.py
+++ b/mirrors/models.py
@@ -92,7 +92,7 @@ def clean(self):
families = self.address_families()
self.has_ipv4 = socket.AF_INET in families
self.has_ipv6 = socket.AF_INET6 in families
- except socket.error as e:
+ except socket.error:
# We don't fail in this case; we'll just set both to False
self.has_ipv4 = False
self.has_ipv6 = False
diff --git a/mirrors/utils.py b/mirrors/utils.py
index 5a8bbf5d..531cf005 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -1,13 +1,13 @@
from datetime import timedelta
from django.db import connection
-from django.db.models import Avg, Count, Max, Min, StdDev
+from django.db.models import Count, Max, Min
from django.utils.dateparse import parse_datetime
from django.utils.timezone import now
from django_countries.fields import Country
from main.utils import cache_function, database_vendor
-from .models import MirrorLog, MirrorProtocol, MirrorUrl
+from .models import MirrorLog, MirrorUrl
DEFAULT_CUTOFF = timedelta(hours=24)
@@ -165,7 +165,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None):
).order_by('-last_occurred', '-error_count')
if mirror_id:
- urls = urls.filter(mirror_id=mirror_id)
+ errors = errors.filter(url__mirror_id=mirror_id)
errors = list(errors)
for err in errors:
diff --git a/packages/migrations/0002_populate_package_relation.py b/packages/migrations/0002_populate_package_relation.py
index 738e068f..b0d32c7a 100644
--- a/packages/migrations/0002_populate_package_relation.py
+++ b/packages/migrations/0002_populate_package_relation.py
@@ -11,7 +11,6 @@ class Migration(DataMigration):
)
def forwards(self, orm):
- "Write your forwards methods here."
# search by pkgbase first and insert those records
qs = orm['main.Package'].objects.exclude(maintainer=None).exclude(
pkgbase=None).distinct().values('pkgbase', 'maintainer_id')
@@ -29,7 +28,6 @@ def forwards(self, orm):
defaults={'user_id': row['maintainer_id']})
def backwards(self, orm):
- "Write your backwards methods here."
if not db.dry_run:
orm.PackageRelation.objects.all().delete()
pass
diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py
index f14fab1e..ef0e1aea 100644
--- a/packages/templatetags/package_extras.py
+++ b/packages/templatetags/package_extras.py
@@ -53,10 +53,10 @@ def do_buildsortqs(parser, token):
tagname, sortfield = token.split_contents()
except ValueError:
raise template.TemplateSyntaxError(
- "%r tag requires a single argument" % tagname)
+ "%r tag requires a single argument" % token)
if not (sortfield[0] == sortfield[-1] and sortfield[0] in ('"', "'")):
raise template.TemplateSyntaxError(
- "%r tag's argument should be in quotes" % tagname)
+ "%r tag's argument should be in quotes" % token)
return BuildQueryStringNode(sortfield[1:-1])
diff --git a/packages/utils.py b/packages/utils.py
index a4217fbd..4f3b8665 100644
--- a/packages/utils.py
+++ b/packages/utils.py
@@ -391,7 +391,7 @@ def signoffs_id_query(model, repos):
"""
cursor = connection.cursor()
# query pre-process- fill in table name and placeholders for IN
- repo_sql = ','.join(['%s' for r in repos])
+ repo_sql = ','.join(['%s' for _ in repos])
sql = sql % (model._meta.db_table, repo_sql, repo_sql)
repo_ids = [r.pk for r in repos]
# repo_ids are needed twice, so double the array
diff --git a/packages/views/display.py b/packages/views/display.py
index 87424483..021c7ed8 100644
--- a/packages/views/display.py
+++ b/packages/views/display.py
@@ -228,8 +228,6 @@ def download(request, name, repo, arch):
if pkg.arch.agnostic:
# grab the first non-any arch to fake the download path
arch = Arch.objects.exclude(agnostic=True)[0].name
- values = {
- }
url = '{host}{repo}/os/{arch}/{filename}'.format(host=url.url,
repo=pkg.repo.name.lower(), arch=arch, filename=pkg.filename)
return redirect(url)
diff --git a/packages/views/flag.py b/packages/views/flag.py
index 5c76e1d5..39cdcef8 100644
--- a/packages/views/flag.py
+++ b/packages/views/flag.py
@@ -110,7 +110,7 @@ def perform_updates():
subject = '%s package [%s] marked out-of-date' % \
(pkg.repo.name, pkg.pkgname)
for maint in maints:
- if maint.userprofile.notify == True:
+ if maint.userprofile.notify is True:
toemail.append(maint.email)
if toemail:
@@ -133,7 +133,6 @@ def perform_updates():
return redirect('package-flag-confirmed', name=name, repo=repo,
arch=arch)
else:
- initial = {}
form = FlagForm(authenticated=authenticated)
context = {
diff --git a/public/views.py b/public/views.py
index 22cb8759..39273396 100644
--- a/public/views.py
+++ b/public/views.py
@@ -125,7 +125,6 @@ def keys(request):
master_keys = MasterKey.objects.select_related('owner', 'revoker',
'owner__userprofile', 'revoker__userprofile').filter(
revoked__isnull=True)
- master_key_ids = frozenset(key.pgp_key[-16:] for key in master_keys)
sig_counts = PGPSignature.objects.filter(not_expired, valid=True,
signee__in=user_key_ids).order_by().values_list('signer').annotate(
diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py
index c9f61964..f182cc33 100644
--- a/releng/management/commands/syncisos.py
+++ b/releng/management/commands/syncisos.py
@@ -20,7 +20,7 @@ def handle_starttag(self, tag, attrs):
if tag == 'a':
for name, value in attrs:
if name == "href":
- if value != '../' and self.url_re.search(value) != None:
+ if value != '../' and self.url_re.search(value) is not None:
self.hyperlinks.append(value[:-1])
def parse(self, url):
diff --git a/releng/models.py b/releng/models.py
index b95f7d52..5ee2f325 100644
--- a/releng/models.py
+++ b/releng/models.py
@@ -160,7 +160,7 @@ def info_html(self):
def torrent(self):
try:
- data = b64decode(self.torrent_data)
+ data = b64decode(self.torrent_data.encode('utf-8'))
except TypeError:
return None
if not data:
diff --git a/releng/views.py b/releng/views.py
index ad4b07d1..b1c76a4a 100644
--- a/releng/views.py
+++ b/releng/views.py
@@ -231,7 +231,7 @@ def release_torrent(request, version):
release = get_object_or_404(Release, version=version)
if not release.torrent_data:
raise Http404
- data = b64decode(release.torrent_data)
+ data = b64decode(release.torrent_data.encode('utf-8'))
response = HttpResponse(data, content_type='application/x-bittorrent')
# TODO: this is duplicated from Release.iso_url()
filename = 'archlinux-%s-dual.iso.torrent' % release.version
diff --git a/retro/templates/retro/index-20030330.html b/retro/templates/retro/index-20030330.html
index 449731af..51cc8ba3 100644
--- a/retro/templates/retro/index-20030330.html
+++ b/retro/templates/retro/index-20030330.html
@@ -232,7 +232,6 @@
-
diff --git a/sitestatic/archweb.js b/sitestatic/archweb.js
index dda22d9e..aa225f5f 100644
--- a/sitestatic/archweb.js
+++ b/sitestatic/archweb.js
@@ -146,7 +146,6 @@ if (typeof $ !== 'undefined' && typeof $.tablesorter !== 'undefined') {
(function($) {
$.fn.enableCheckboxRangeSelection = function() {
var lastCheckbox = null,
- lastElement = null,
spec = this;
spec.unbind("click.checkboxrange");
diff --git a/todolists/utils.py b/todolists/utils.py
index 51a75a3c..7b98c887 100644
--- a/todolists/utils.py
+++ b/todolists/utils.py
@@ -1,5 +1,4 @@
from django.db import connections, router
-from django.db.models import Count
from .models import Todolist, TodolistPackage
from packages.models import Package
diff --git a/todolists/views.py b/todolists/views.py
index 7636d38e..d5b39934 100644
--- a/todolists/views.py
+++ b/todolists/views.py
@@ -9,7 +9,6 @@
from django.views.decorators.cache import never_cache
from django.views.generic import DeleteView
from django.template import Context, loader
-from django.template.defaultfilters import slugify
from django.utils.timezone import now
from main.models import Package, Repo
diff --git a/visualize/static/visualize.js b/visualize/static/visualize.js
index 7e240d44..5004fe6c 100644
--- a/visualize/static/visualize.js
+++ b/visualize/static/visualize.js
@@ -55,7 +55,7 @@ function packages_treemap(chart_id, orderings, default_order) {
var nodes = d3_div.data([json]).selectAll("div")
.data(treemap.nodes, key_func);
/* start out new nodes in the center of the picture area */
- var w_center = jq_div.width() / 2;
+ var w_center = jq_div.width() / 2,
h_center = jq_div.height() / 2;
nodes.enter().append("div")
.attr("class", "treemap-cell")
--
cgit v1.2.3-54-g00ecf