From f2a6316be0b025a9ee22f22d34df1c00f60a8bdf Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 11:56:26 -0500 Subject: Add additional pg_trgm indexes for quicker searches This allows our normal keyword-based search to be index-optimized rather than always doing full table scans. It requires the pg_trgm extension which is shipped out of the box with any sane install of PostgreSQL. Signed-off-by: Dan McGee --- packages/sql/search_indexes.postgresql_psycopg2.sql | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 packages/sql/search_indexes.postgresql_psycopg2.sql (limited to 'packages') diff --git a/packages/sql/search_indexes.postgresql_psycopg2.sql b/packages/sql/search_indexes.postgresql_psycopg2.sql new file mode 100644 index 00000000..a7eaf998 --- /dev/null +++ b/packages/sql/search_indexes.postgresql_psycopg2.sql @@ -0,0 +1,3 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE INDEX packages_pkgname_trgm_gist ON packages USING gist (UPPER(pkgname) gist_trgm_ops); +CREATE INDEX packages_pkgdesc_trgm_gist ON packages USING gist (UPPER(pkgdesc) gist_trgm_ops); -- cgit v1.2.3 From 7fc8da7d959556b1204b7864959e73e7f5f5ec59 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 13:05:02 -0500 Subject: Show replacments for package if it has been removed This covers the case where we can't find the package in any other repositories, but it was removed recently enough that we have a found package update object. Signed-off-by: Dan McGee --- packages/models.py | 9 +++++++++ packages/views/display.py | 2 ++ 2 files changed, 11 insertions(+) (limited to 'packages') diff --git a/packages/models.py b/packages/models.py index 7bcdc000..92566a56 100644 --- a/packages/models.py +++ b/packages/models.py @@ -321,6 +321,15 @@ class Update(models.Model): return Package.objects.normal().filter( pkgname=self.pkgname, arch=self.arch) + def replacements(self): + pkgs = Package.objects.normal().filter( + replaces__name=self.pkgname) + if not self.arch.agnostic: + # make sure we match architectures if possible + arches = self.pkg.applicable_arches() + pkgs = pkgs.filter(arch__in=arches) + return pkgs + def __unicode__(self): return u'%s of %s on %s' % (self.get_action_flag_display(), self.pkgname, self.created) diff --git a/packages/views/display.py b/packages/views/display.py index fcf8fdea..50783835 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -55,6 +55,8 @@ def recently_removed_package(request, name, repo, arch, cutoff=CUTOFF): try: update = match.latest() elsewhere = update.elsewhere() + if len(elsewhere) == 0: + elsewhere = update.replacements() if len(elsewhere) == 1: return redirect(elsewhere[0]) context = { -- cgit v1.2.3 From 4d7d08f93de9e6af9e664a00e090158e738a890c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:45:00 -0500 Subject: Fix missing attribute error in replacment find code Whoops. Just introduced this when ensuring we look for both the packgae in other repositories as well as any replacments. Signed-off-by: Dan McGee --- packages/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'packages') diff --git a/packages/models.py b/packages/models.py index 92566a56..f830aade 100644 --- a/packages/models.py +++ b/packages/models.py @@ -326,7 +326,8 @@ class Update(models.Model): replaces__name=self.pkgname) if not self.arch.agnostic: # make sure we match architectures if possible - arches = self.pkg.applicable_arches() + arches = set(Arch.objects.filter(agnostic=True)) + arches.add(self.arch) pkgs = pkgs.filter(arch__in=arches) return pkgs -- cgit v1.2.3 From 283cd944beefce8e364f238f25133e2d65b7702b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 20:16:06 -0500 Subject: Use require_safe decorator rather than require_GET This was added in Django 1.4, and ensures both GET and HEAD requests, but not POST requests, are allowed through. Signed-off-by: Dan McGee --- packages/views/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'packages') diff --git a/packages/views/__init__.py b/packages/views/__init__.py index 4c195385..c1f0f492 100644 --- a/packages/views/__init__.py +++ b/packages/views/__init__.py @@ -9,7 +9,7 @@ from django.db.models import Q from django.http import HttpResponse from django.shortcuts import redirect, render from django.views.decorators.cache import cache_control -from django.views.decorators.http import require_GET, require_POST +from django.views.decorators.http import require_safe, require_POST from main.models import Package, Arch from ..models import PackageRelation @@ -24,7 +24,7 @@ from .search import search_json from .signoff import signoffs, signoff_package, signoff_options, signoffs_json -@require_GET +@require_safe @cache_control(public=True, max_age=86400) def opensearch(request): if request.is_secure(): @@ -37,7 +37,7 @@ def opensearch(request): content_type='application/opensearchdescription+xml') -@require_GET +@require_safe @cache_control(public=True, max_age=300) def opensearch_suggest(request): search_term = request.GET.get('q', '') -- cgit v1.2.3 From 31d39e75eea7fb6cdf3bb8bfd8b490d45de04ee9 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 21:59:32 -0500 Subject: Add shortcut for HEAD requests on slower views We sometimes see some web bots and crawlers make HEAD requests to verify existence of certain pages in the application. However, they are less than kind as 20-50 requests might arrive at the same time, and package search and details pages are some of the slowest rendering pages we have due to the Django template engine. Rather than waste time generating the content only to throw it away, response as soon as we can with either a 404 or 200 response as appropriate, omitting the 'Content-Length' header completely, which seems to be acceptable by the HTTP spec. Signed-off-by: Dan McGee --- packages/views/display.py | 3 +++ packages/views/search.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'packages') diff --git a/packages/views/display.py b/packages/views/display.py index 50783835..87424483 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -7,6 +7,7 @@ from django.shortcuts import get_object_or_404, redirect, render from django.utils.timezone import now from main.models import Package, PackageFile, Arch, Repo +from main.utils import empty_response from mirrors.utils import get_mirror_url_for_download from ..models import Update from ..utils import get_group_info, PackageJSONEncoder @@ -126,6 +127,8 @@ def details(request, name='', repo='', arch=''): pkg = Package.objects.select_related( 'arch', 'repo', 'packager').get(pkgname=name, repo=repo_obj, arch=arch_obj) + if request.method == 'HEAD': + return empty_response() return render(request, 'packages/details.html', {'pkg': pkg}) except Package.DoesNotExist: # attempt a variety of fallback options before 404ing diff --git a/packages/views/search.py b/packages/views/search.py index 0362602e..b3778172 100644 --- a/packages/views/search.py +++ b/packages/views/search.py @@ -7,7 +7,7 @@ from django.http import HttpResponse from django.views.generic import ListView from main.models import Package, Arch, Repo -from main.utils import make_choice +from main.utils import empty_response, make_choice from ..models import PackageRelation from ..utils import attach_maintainers, PackageJSONEncoder @@ -99,6 +99,8 @@ class SearchListView(ListView): allowed_sort = list(sort_fields) + ["-" + s for s in sort_fields] def get(self, request, *args, **kwargs): + if request.method == 'HEAD': + return empty_response() self.form = PackageSearchForm(data=request.GET, show_staging=self.request.user.is_authenticated()) return super(SearchListView, self).get(request, *args, **kwargs) -- cgit v1.2.3 From b7b24740640e24883cd17fd683e1d465fbb343f8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 22:12:01 -0500 Subject: Various minor code cleanups and fixes Most of these were suggested by PyCharm, and include everything from little syntax issues and other bad smells to dead or bad code. Signed-off-by: Dan McGee --- packages/migrations/0002_populate_package_relation.py | 2 -- packages/templatetags/package_extras.py | 4 ++-- packages/utils.py | 2 +- packages/views/display.py | 2 -- packages/views/flag.py | 3 +-- 5 files changed, 4 insertions(+), 9 deletions(-) (limited to 'packages') diff --git a/packages/migrations/0002_populate_package_relation.py b/packages/migrations/0002_populate_package_relation.py index 738e068f..b0d32c7a 100644 --- a/packages/migrations/0002_populate_package_relation.py +++ b/packages/migrations/0002_populate_package_relation.py @@ -11,7 +11,6 @@ class Migration(DataMigration): ) def forwards(self, orm): - "Write your forwards methods here." # search by pkgbase first and insert those records qs = orm['main.Package'].objects.exclude(maintainer=None).exclude( pkgbase=None).distinct().values('pkgbase', 'maintainer_id') @@ -29,7 +28,6 @@ class Migration(DataMigration): defaults={'user_id': row['maintainer_id']}) def backwards(self, orm): - "Write your backwards methods here." if not db.dry_run: orm.PackageRelation.objects.all().delete() pass diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index f14fab1e..ef0e1aea 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -53,10 +53,10 @@ def do_buildsortqs(parser, token): tagname, sortfield = token.split_contents() except ValueError: raise template.TemplateSyntaxError( - "%r tag requires a single argument" % tagname) + "%r tag requires a single argument" % token) if not (sortfield[0] == sortfield[-1] and sortfield[0] in ('"', "'")): raise template.TemplateSyntaxError( - "%r tag's argument should be in quotes" % tagname) + "%r tag's argument should be in quotes" % token) return BuildQueryStringNode(sortfield[1:-1]) diff --git a/packages/utils.py b/packages/utils.py index a4217fbd..4f3b8665 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -391,7 +391,7 @@ SELECT DISTINCT s.id """ cursor = connection.cursor() # query pre-process- fill in table name and placeholders for IN - repo_sql = ','.join(['%s' for r in repos]) + repo_sql = ','.join(['%s' for _ in repos]) sql = sql % (model._meta.db_table, repo_sql, repo_sql) repo_ids = [r.pk for r in repos] # repo_ids are needed twice, so double the array diff --git a/packages/views/display.py b/packages/views/display.py index 87424483..021c7ed8 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -228,8 +228,6 @@ def download(request, name, repo, arch): if pkg.arch.agnostic: # grab the first non-any arch to fake the download path arch = Arch.objects.exclude(agnostic=True)[0].name - values = { - } url = '{host}{repo}/os/{arch}/{filename}'.format(host=url.url, repo=pkg.repo.name.lower(), arch=arch, filename=pkg.filename) return redirect(url) diff --git a/packages/views/flag.py b/packages/views/flag.py index 5c76e1d5..39cdcef8 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -110,7 +110,7 @@ def flag(request, name, repo, arch): subject = '%s package [%s] marked out-of-date' % \ (pkg.repo.name, pkg.pkgname) for maint in maints: - if maint.userprofile.notify == True: + if maint.userprofile.notify is True: toemail.append(maint.email) if toemail: @@ -133,7 +133,6 @@ def flag(request, name, repo, arch): return redirect('package-flag-confirmed', name=name, repo=repo, arch=arch) else: - initial = {} form = FlagForm(authenticated=authenticated) context = { -- cgit v1.2.3 From bb18fa3323a0494a2774ea61911572b089d04b6d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 13:32:09 -0500 Subject: Fix parsing issues when query string keys contain unicode This is dirty, but it works. There is probably a better and cleaner way to do all of this, but for now just fix it quickly. Signed-off-by: Dan McGee --- packages/templatetags/package_extras.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'packages') diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index ef0e1aea..f7392a96 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -37,6 +37,12 @@ class BuildQueryStringNode(template.Node): def render(self, context): qs = parse_qs(context['current_query']) + # This is really dirty. The crazy round trips we do on our query string + # mean we get things like u'\xe2\x98\x83' in our views, when we should + # have simply u'\u2603' or a byte string of the UTF-8 value. Force the + # keys and list of values to be byte strings only. + qs = {k.encode('latin-1'): [v.encode('latin-1') for v in vals] + for k, vals in qs.items()} if 'sort' in qs and self.sortfield in qs['sort']: if self.sortfield.startswith('-'): qs['sort'] = [self.sortfield[1:]] -- cgit v1.2.3