From 2f19529c922172007ff3894ad0e0fca0b3e33f5b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 18 Oct 2014 13:48:07 -0500 Subject: Convert details sub-tempates to Jinja2 This is a start at improving performance of rendering the package details page, our most-visited and slowest page on the production website. The Django template system is not very efficient due to our heavy use of broken out templates and pulling of various attributes and such on related packages. Signed-off-by: Dan McGee --- packages/templatetags/package_extras.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'packages') diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index 3ce64748..2784546d 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -79,11 +79,6 @@ def pkg_details_link(pkg, link_title=None, honor_flagged=False): return link % (pkg.get_absolute_url(), pkg.pkgname, link_content) -@register.simple_tag -def multi_pkg_details(pkgs): - return ', '.join([pkg_details_link(pkg) for pkg in pkgs]) - - @register.simple_tag def maintainer_link(user): if user: -- cgit v1.2.3 From fff9cf9c5537d9f758a9e30fcb18df800c70f051 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 18 Oct 2014 15:17:56 -0500 Subject: Convert bulk of package details page to Jinja2 We wrap the whole thing in a Django template so we don't have to convert our base template to Jinja2 (or have one for each templating engine). This also simplifies the static stuff so we can just keep using the Django tags. Signed-off-by: Dan McGee --- packages/templatetags/package_extras.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) (limited to 'packages') diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index 2784546d..f6d6ee73 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -6,6 +6,9 @@ except ImportError: from django import template from django.utils.html import escape +from django_jinja import library + +from main.templatetags import pgp register = template.Library() @@ -14,11 +17,11 @@ def link_encode(url, query): # massage the data into all utf-8 encoded strings first, so urlencode # doesn't barf at the data we pass it query = {k: unicode(v).encode('utf-8') for k, v in query.items()} - data = urlencode(query).replace('&', '&') + data = urlencode(query) return "%s?%s" % (url, data) -@register.filter +@library.filter def url_unquote(original_url): try: url = original_url @@ -79,7 +82,7 @@ def pkg_details_link(pkg, link_title=None, honor_flagged=False): return link % (pkg.get_absolute_url(), pkg.pkgname, link_content) -@register.simple_tag +@library.global_function def maintainer_link(user): if user: # TODO don't hardcode @@ -92,7 +95,7 @@ def maintainer_link(user): return '' -@register.simple_tag +@library.global_function def packager_link(user): if user: # TODO don't hardcode @@ -105,7 +108,12 @@ def packager_link(user): return '' -@register.simple_tag +@library.global_function +def pgp_key_link(key_id, link_text=None): + return pgp.pgp_key_link(key_id, link_text) + + +@library.global_function def scm_link(package, operation): parts = (package.repo.svn_root, operation, package.pkgbase) linkbase = ( @@ -114,8 +122,8 @@ def scm_link(package, operation): return linkbase % tuple(urlquote(part.encode('utf-8')) for part in parts) -@register.simple_tag -def get_wiki_link(package): +@library.global_function +def wiki_link(package): url = "https://wiki.archlinux.org/index.php/Special:Search" data = { 'search': package.pkgname, @@ -123,7 +131,7 @@ def get_wiki_link(package): return link_encode(url, data) -@register.simple_tag +@library.global_function def bugs_list(package): url = "https://bugs.archlinux.org/" data = { @@ -134,7 +142,7 @@ def bugs_list(package): return link_encode(url, data) -@register.simple_tag +@library.global_function def bug_report(package): url = "https://bugs.archlinux.org/newtask" data = { -- cgit v1.2.3 From e5e5632fb3432e153d0677b4bf145bc2eb72ef65 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 18 Oct 2014 15:25:30 -0500 Subject: Break out Jinja2-specific template helpers Signed-off-by: Dan McGee --- packages/templatetags/jinja2.py | 98 +++++++++++++++++++++++++++++++++ packages/templatetags/package_extras.py | 95 +------------------------------- 2 files changed, 99 insertions(+), 94 deletions(-) create mode 100644 packages/templatetags/jinja2.py (limited to 'packages') diff --git a/packages/templatetags/jinja2.py b/packages/templatetags/jinja2.py new file mode 100644 index 00000000..263fc156 --- /dev/null +++ b/packages/templatetags/jinja2.py @@ -0,0 +1,98 @@ +from urllib import urlencode, quote as urlquote, unquote +from django.utils.html import escape +from django_jinja import library +from main.templatetags import pgp + + +@library.filter +def url_unquote(original_url): + try: + url = original_url + if isinstance(url, unicode): + url = url.encode('ascii') + url = unquote(url).decode('utf-8') + return url + except UnicodeError: + return original_url + + +def link_encode(url, query): + # massage the data into all utf-8 encoded strings first, so urlencode + # doesn't barf at the data we pass it + query = {k: unicode(v).encode('utf-8') for k, v in query.items()} + data = urlencode(query) + return "%s?%s" % (url, data) + + +@library.global_function +def maintainer_link(user): + if user: + # TODO don't hardcode + title = escape('View packages maintained by ' + user.get_full_name()) + return '%s' % ( + user.username, + title, + user.get_full_name(), + ) + return '' + + +@library.global_function +def packager_link(user): + if user: + # TODO don't hardcode + title = escape('View packages packaged by ' + user.get_full_name()) + return '%s' % ( + user.username, + title, + user.get_full_name(), + ) + return '' + + +@library.global_function +def pgp_key_link(key_id, link_text=None): + return pgp.pgp_key_link(key_id, link_text) + + +@library.global_function +def scm_link(package, operation): + parts = (package.repo.svn_root, operation, package.pkgbase) + linkbase = ( + "https://projects.archlinux.org/svntogit/%s.git/%s/trunk?" + "h=packages/%s") + return linkbase % tuple(urlquote(part.encode('utf-8')) for part in parts) + + +@library.global_function +def wiki_link(package): + url = "https://wiki.archlinux.org/index.php/Special:Search" + data = { + 'search': package.pkgname, + } + return link_encode(url, data) + + +@library.global_function +def bugs_list(package): + url = "https://bugs.archlinux.org/" + data = { + 'project': package.repo.bugs_project, + 'cat[]': package.repo.bugs_category, + 'string': package.pkgname, + } + return link_encode(url, data) + + +@library.global_function +def bug_report(package): + url = "https://bugs.archlinux.org/newtask" + data = { + 'project': package.repo.bugs_project, + 'product_category': package.repo.bugs_category, + 'item_summary': '[%s] PLEASE ENTER SUMMARY' % package.pkgname, + } + return link_encode(url, data) + + +# vim: set ts=4 sw=4 et: diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index f6d6ee73..73a39092 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -1,38 +1,15 @@ -from urllib import urlencode, quote as urlquote, unquote +from urllib import urlencode try: from urlparse import parse_qs except ImportError: from cgi import parse_qs from django import template -from django.utils.html import escape -from django_jinja import library -from main.templatetags import pgp register = template.Library() -def link_encode(url, query): - # massage the data into all utf-8 encoded strings first, so urlencode - # doesn't barf at the data we pass it - query = {k: unicode(v).encode('utf-8') for k, v in query.items()} - data = urlencode(query) - return "%s?%s" % (url, data) - - -@library.filter -def url_unquote(original_url): - try: - url = original_url - if isinstance(url, unicode): - url = url.encode('ascii') - url = unquote(url).decode('utf-8') - return url - except UnicodeError: - return original_url - - class BuildQueryStringNode(template.Node): def __init__(self, sortfield): self.sortfield = sortfield @@ -82,74 +59,4 @@ def pkg_details_link(pkg, link_title=None, honor_flagged=False): return link % (pkg.get_absolute_url(), pkg.pkgname, link_content) -@library.global_function -def maintainer_link(user): - if user: - # TODO don't hardcode - title = escape('View packages maintained by ' + user.get_full_name()) - return '%s' % ( - user.username, - title, - user.get_full_name(), - ) - return '' - - -@library.global_function -def packager_link(user): - if user: - # TODO don't hardcode - title = escape('View packages packaged by ' + user.get_full_name()) - return '%s' % ( - user.username, - title, - user.get_full_name(), - ) - return '' - - -@library.global_function -def pgp_key_link(key_id, link_text=None): - return pgp.pgp_key_link(key_id, link_text) - - -@library.global_function -def scm_link(package, operation): - parts = (package.repo.svn_root, operation, package.pkgbase) - linkbase = ( - "https://projects.archlinux.org/svntogit/%s.git/%s/trunk?" - "h=packages/%s") - return linkbase % tuple(urlquote(part.encode('utf-8')) for part in parts) - - -@library.global_function -def wiki_link(package): - url = "https://wiki.archlinux.org/index.php/Special:Search" - data = { - 'search': package.pkgname, - } - return link_encode(url, data) - - -@library.global_function -def bugs_list(package): - url = "https://bugs.archlinux.org/" - data = { - 'project': package.repo.bugs_project, - 'cat[]': package.repo.bugs_category, - 'string': package.pkgname, - } - return link_encode(url, data) - - -@library.global_function -def bug_report(package): - url = "https://bugs.archlinux.org/newtask" - data = { - 'project': package.repo.bugs_project, - 'product_category': package.repo.bugs_category, - 'item_summary': '[%s] PLEASE ENTER SUMMARY' % package.pkgname, - } - return link_encode(url, data) - # vim: set ts=4 sw=4 et: -- cgit v1.2.3 From 122302e9e5fb9752a942abe5691ad3dd6d186196 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 18 Oct 2014 15:36:56 -0500 Subject: Move maintainer/packager link code back into template Signed-off-by: Dan McGee --- packages/templatetags/jinja2.py | 26 -------------------------- 1 file changed, 26 deletions(-) (limited to 'packages') diff --git a/packages/templatetags/jinja2.py b/packages/templatetags/jinja2.py index 263fc156..22f9914b 100644 --- a/packages/templatetags/jinja2.py +++ b/packages/templatetags/jinja2.py @@ -24,32 +24,6 @@ def link_encode(url, query): return "%s?%s" % (url, data) -@library.global_function -def maintainer_link(user): - if user: - # TODO don't hardcode - title = escape('View packages maintained by ' + user.get_full_name()) - return '%s' % ( - user.username, - title, - user.get_full_name(), - ) - return '' - - -@library.global_function -def packager_link(user): - if user: - # TODO don't hardcode - title = escape('View packages packaged by ' + user.get_full_name()) - return '%s' % ( - user.username, - title, - user.get_full_name(), - ) - return '' - - @library.global_function def pgp_key_link(key_id, link_text=None): return pgp.pgp_key_link(key_id, link_text) -- cgit v1.2.3 From 0e90423fc09f53317992bbb024e82ef65728aa11 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 18 Oct 2014 17:25:21 -0500 Subject: Add possible optimization as a TODO for later --- packages/models.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'packages') diff --git a/packages/models.py b/packages/models.py index dd69e7d3..03f03422 100644 --- a/packages/models.py +++ b/packages/models.py @@ -379,6 +379,13 @@ class RelatedToBase(models.Model): given criteria. It will not search provisions, but will find packages named and matching repo characteristics if possible.''' pkgs = Package.objects.normal().filter(pkgname=self.name) + # TODO: this may in fact be faster- select only the fields we know will + # actually get used, saving us some bandwidth and hopefully query + # construction time. However, reality hasn't quite proved it out yet. + #pkgs = Package.objects.select_related('repo', 'arch').only( + # 'id', 'pkgname', 'epoch', 'pkgver', 'pkgrel', + # 'repo__id', 'repo__name', 'repo__testing', 'repo__staging', + # 'arch__id', 'arch__name').filter(pkgname=self.name) if not self.pkg.arch.agnostic: # make sure we match architectures if possible arches = self.pkg.applicable_arches() -- cgit v1.2.3 From 061805aa73d3d5b418a61fa6d6591d1a7d03d9ed Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 18 Oct 2014 18:28:26 -0500 Subject: FS#30773: put exact matches at top There have been a few proposed solutions to this, but there really isn't anything without a drawback. Things break pagination, require loading the entire result set from the database, etc. Just plop a new table on the page if someone did a so-called "simple" search and we have a match. Only show on the first page of the search results. This results in a relatively fast experience for someone doing something like searching for the "perl" package. Signed-off-by: Dan McGee --- packages/views/search.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'packages') diff --git a/packages/views/search.py b/packages/views/search.py index b3778172..0b776d79 100644 --- a/packages/views/search.py +++ b/packages/views/search.py @@ -45,6 +45,12 @@ class PackageSearchForm(forms.Form): [('', 'All'), ('unknown', 'Unknown')] + \ [(m.username, m.get_full_name()) for m in maints] + def exact_matches(self): + # only do exact match search if 'q' is sole parameter + if self.changed_data != ['q']: + return [] + return Package.objects.normal().filter(pkgname=self.cleaned_data['q']) + def parse_form(form, packages): if form.cleaned_data['repo']: -- cgit v1.2.3 From 7c46d07f6417382e560d32c361c94dd8f2b9ddb8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 19 Oct 2014 18:12:58 -0500 Subject: Use varied prime numbers for caching lengths Signed-off-by: Dan McGee --- packages/views/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages') diff --git a/packages/views/__init__.py b/packages/views/__init__.py index 46c99985..6b44206a 100644 --- a/packages/views/__init__.py +++ b/packages/views/__init__.py @@ -35,7 +35,7 @@ def opensearch(request): @require_safe -@cache_control(public=True, max_age=300) +@cache_control(public=True, max_age=613) def opensearch_suggest(request): search_term = request.GET.get('q', '') if search_term == '': @@ -55,7 +55,7 @@ def opensearch_suggest(request): 'pkgname', flat=True).order_by('pkgname').distinct()[:10] results = [search_term, list(names)] to_json = json.dumps(results, ensure_ascii=False) - cache.set(cache_key, to_json, 300) + cache.set(cache_key, to_json, 613) return HttpResponse(to_json, content_type='application/x-suggestions+json') -- cgit v1.2.3 From 65e3ef63f7475a217da5205a788f52f3888ebea7 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 14 Apr 2015 02:17:54 -0400 Subject: Replace usage of parabolagnulinux.org with parabola.nu, also replace repo.parabola.nu/isos/ with /iso/, in settings.py --- packages/templatetags/jinja2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages') diff --git a/packages/templatetags/jinja2.py b/packages/templatetags/jinja2.py index c9d602a6..86bbd03e 100644 --- a/packages/templatetags/jinja2.py +++ b/packages/templatetags/jinja2.py @@ -33,13 +33,13 @@ def pgp_key_link(key_id, link_text=None): def scm_link(package, operation): parts = ("abslibre", operation, package.repo.name.lower(), package.pkgbase) linkbase = ( - "https://projects.parabolagnulinux.org/%s.git/%s/%s/%s") + "https://projects.parabola.nu/%s.git/%s/%s/%s") return linkbase % tuple(urlquote(part.encode('utf-8')) for part in parts) @library.global_function def wiki_link(package): - url = "https://wiki.parabolagnulinux.org/index.php" + url = "https://wiki.parabola.nu/index.php" data = { 'title': "Special:Search", 'search': package.pkgname, -- cgit v1.2.3