From 7c26f6b7a4d29faede58d2feb13ef961e4725637 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 19 Oct 2014 14:08:15 -0500 Subject: Use raw DB query to fetch last modified date Signed-off-by: Dan McGee --- mirrors/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mirrors/views.py b/mirrors/views.py index 26b5b802..55c40c4d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -6,6 +6,7 @@ from operator import attrgetter, itemgetter from django import forms from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder +from django.db import connection from django.db.models import Q from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, redirect, render @@ -222,7 +223,9 @@ def url_details(request, name, url_id): def status_last_modified(request, *args, **kwargs): - return MirrorLog.objects.values_list('check_time', flat=True).latest() + cursor = connection.cursor() + cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog") + return cursor.fetchone()[0] @condition(last_modified_func=status_last_modified) -- cgit v1.2.3 From 1ff2e37e049004852681794537417a1947bf6f18 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 19 Oct 2014 14:19:05 -0500 Subject: Simplify last modified and etags processing for feeds We had this elaborate system set up with caching and invalidation, which is overkill since we cache the result of the view anyway. Just hit the database when needed to find the last change to the respective model class and be done with it. Signed-off-by: Dan McGee --- feeds.py | 14 +++++++++----- main/models.py | 6 +----- main/utils.py | 42 ------------------------------------------ news/models.py | 7 ++----- 4 files changed, 12 insertions(+), 57 deletions(-) diff --git a/feeds.py b/feeds.py index feb8a84a..d1836178 100644 --- a/feeds.py +++ b/feeds.py @@ -4,11 +4,11 @@ from pytz import utc from django.contrib.sites.models import Site from django.contrib.syndication.views import Feed +from django.db import connection from django.db.models import Q from django.utils.feedgenerator import Rss201rev2Feed from django.views.decorators.http import condition -from main.utils import retrieve_latest from main.models import Arch, Repo, Package from news.models import News from releng.models import Release @@ -64,13 +64,15 @@ class GuidNotPermalinkFeed(Rss201rev2Feed): def package_etag(request, *args, **kwargs): - latest = retrieve_latest(Package) + latest = package_last_modified(request) if latest: return hashlib.md5(str(kwargs) + str(latest)).hexdigest() return None def package_last_modified(request, *args, **kwargs): - return retrieve_latest(Package) + cursor = connection.cursor() + cursor.execute("SELECT MAX(last_update) FROM packages") + return cursor.fetchone()[0] class PackageFeed(Feed): @@ -148,13 +150,15 @@ class PackageFeed(Feed): def news_etag(request, *args, **kwargs): - latest = retrieve_latest(News, 'last_modified') + latest = news_last_modified(request) if latest: return hashlib.md5(str(latest)).hexdigest() return None def news_last_modified(request, *args, **kwargs): - return retrieve_latest(News, 'last_modified') + cursor = connection.cursor() + cursor.execute("SELECT MAX(last_modified) FROM news") + return cursor.fetchone()[0] class NewsFeed(Feed): diff --git a/main/models.py b/main/models.py index 09b1adc0..1b95f3fa 100644 --- a/main/models.py +++ b/main/models.py @@ -443,12 +443,8 @@ class PackageFile(models.Model): db_table = 'package_files' -# connect signals needed to keep cache in line with reality -from main.utils import refresh_latest -from django.db.models.signals import pre_save, post_save +from django.db.models.signals import pre_save -post_save.connect(refresh_latest, sender=Package, - dispatch_uid="main.models") # note: reporead sets the 'created' field on Package objects, so no signal # listener is set up here to do so pre_save.connect(set_created_field, sender=Donor, diff --git a/main/utils.py b/main/utils.py index 97cc540a..cf156566 100644 --- a/main/utils.py +++ b/main/utils.py @@ -12,11 +12,6 @@ from django.utils.timezone import now from django.template.defaultfilters import slugify -CACHE_TIMEOUT = 1800 -INVALIDATE_TIMEOUT = 10 -CACHE_LATEST_PREFIX = 'cache_latest_' - - def cache_function_key(func, args, kwargs): raw = [func.__name__, func.__module__, args, kwargs] pickled = pickle.dumps(raw, protocol=pickle.HIGHEST_PROTOCOL) @@ -76,43 +71,6 @@ def format_http_headers(request): make_choice = lambda l: [(str(m), str(m)) for m in l] -# These are in here because we would be jumping around in some import circles -# and hoops otherwise. The only thing currently using these keys is the feed -# caching stuff. - -def refresh_latest(**kwargs): - '''A post_save signal handler to clear out the cached latest value for a - given model.''' - cache_key = CACHE_LATEST_PREFIX + kwargs['sender'].__name__ - # We could delete the value, but that could open a race condition - # where the new data wouldn't have been committed yet by the calling - # thread. Instead, explicitly set it to None for a short amount of time. - # Hopefully by the time it expires we will have committed, and the cache - # will be valid again. See "Scaling Django" by Mike Malone, slide 30. - cache.set(cache_key, None, INVALIDATE_TIMEOUT) - - -def retrieve_latest(sender, latest_by=None): - # we could break this down based on the request url, but it would probably - # cost us more in query time to do so. - cache_key = CACHE_LATEST_PREFIX + sender.__name__ - latest = cache.get(cache_key) - if latest: - return latest - try: - if latest_by is None: - latest_by = sender._meta.get_latest_by - latest = sender.objects.values(latest_by).latest(latest_by)[latest_by] - # Using add means "don't overwrite anything in there". What could be in - # there is an explicit None value that our refresh signal set, which - # means we want to avoid race condition possibilities for a bit. - cache.add(cache_key, latest, CACHE_TIMEOUT) - return latest - except sender.DoesNotExist: - pass - return None - - def set_created_field(sender, **kwargs): '''This will set the 'created' field on any object to the current UTC time if it is unset. diff --git a/news/models.py b/news/models.py index d51db7c7..985c1088 100644 --- a/news/models.py +++ b/news/models.py @@ -45,12 +45,9 @@ def set_news_fields(sender, **kwargs): news.guid = 'tag:%s,%s:%s' % (Site.objects.get_current(), current_time.strftime('%Y-%m-%d'), news.get_absolute_url()) -# connect signals needed to keep cache in line with reality -from main.utils import refresh_latest -from django.db.models.signals import pre_save, post_save -post_save.connect(refresh_latest, sender=News, - dispatch_uid="news.models") +from django.db.models.signals import pre_save + pre_save.connect(set_news_fields, sender=News, dispatch_uid="news.models") -- cgit v1.2.3 From 7c46d07f6417382e560d32c361c94dd8f2b9ddb8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 19 Oct 2014 18:12:58 -0500 Subject: Use varied prime numbers for caching lengths Signed-off-by: Dan McGee --- devel/utils.py | 2 +- main/templatetags/pgp.py | 2 +- packages/views/__init__.py | 4 ++-- public/views.py | 14 +++++++------- urls.py | 17 ++++++++--------- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/devel/utils.py b/devel/utils.py index 340841f5..0cafbb59 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -9,7 +9,7 @@ from main.utils import cache_function from main.models import Package from packages.models import PackageRelation -@cache_function(300) +@cache_function(283) def get_annotated_maintainers(): maintainers = User.objects.filter(is_active=True).order_by( 'first_name', 'last_name') diff --git a/main/templatetags/pgp.py b/main/templatetags/pgp.py index cc080439..2417f688 100644 --- a/main/templatetags/pgp.py +++ b/main/templatetags/pgp.py @@ -44,7 +44,7 @@ def pgp_key_link(key_id, link_text=None): return '%s' % values -@cache_function(1800) +@cache_function(1741) def name_for_key(normalized): try: matching_key = DeveloperKey.objects.select_related( diff --git a/packages/views/__init__.py b/packages/views/__init__.py index 46c99985..6b44206a 100644 --- a/packages/views/__init__.py +++ b/packages/views/__init__.py @@ -35,7 +35,7 @@ def opensearch(request): @require_safe -@cache_control(public=True, max_age=300) +@cache_control(public=True, max_age=613) def opensearch_suggest(request): search_term = request.GET.get('q', '') if search_term == '': @@ -55,7 +55,7 @@ def opensearch_suggest(request): 'pkgname', flat=True).order_by('pkgname').distinct()[:10] results = [search_term, list(names)] to_json = json.dumps(results, ensure_ascii=False) - cache.set(cache_key, to_json, 300) + cache.set(cache_key, to_json, 613) return HttpResponse(to_json, content_type='application/x-suggestions+json') diff --git a/public/views.py b/public/views.py index 3b23bd42..62d1137a 100644 --- a/public/views.py +++ b/public/views.py @@ -17,7 +17,7 @@ from releng.models import Release from .utils import get_recent_updates -@cache_control(max_age=300) +@cache_control(max_age=307) def index(request): if request.user.is_authenticated(): def updates(): @@ -50,7 +50,7 @@ USER_LISTS = { } -@cache_control(max_age=300) +@cache_control(max_age=307) def userlist(request, user_type='devs'): users = User.objects.order_by( 'first_name', 'last_name').select_related('userprofile') @@ -70,7 +70,7 @@ def userlist(request, user_type='devs'): return render(request, 'public/userlist.html', context) -@cache_control(max_age=300) +@cache_control(max_age=307) def donate(request): context = { 'donors': Donor.objects.filter(visible=True).order_by('name'), @@ -88,7 +88,7 @@ def _mirror_urls(): return sorted(urls, key=sort_by) -@cache_control(max_age=300) +@cache_control(max_age=307) def download(request): try: release = Release.objects.filter(available=True).latest() @@ -104,7 +104,7 @@ def download(request): return render(request, 'public/download.html', context) -@cache_control(max_age=300) +@cache_control(max_age=307) def feeds(request): repos = Repo.objects.all() if not request.user.is_authenticated(): @@ -116,7 +116,7 @@ def feeds(request): return render(request, 'public/feeds.html', context) -@cache_control(max_age=300) +@cache_control(max_age=307) def keys(request): users = User.objects.filter(is_active=True).select_related( 'userprofile__pgp_key').order_by('first_name', 'last_name') @@ -153,7 +153,7 @@ def keys(request): return render(request, 'public/keys.html', context) -@cache_page(1800) +@cache_page(1789) def keys_json(request): node_list = [] diff --git a/urls.py b/urls.py index e7f2b24c..8f0bbe21 100644 --- a/urls.py +++ b/urls.py @@ -41,20 +41,19 @@ urlpatterns += patterns('public.views', # Feeds patterns, used below feeds_patterns = patterns('', (r'^$', 'public.views.feeds', {}, 'feeds-list'), - (r'^news/$', cache_page(300)(NewsFeed())), - (r'^packages/$', cache_page(300)(PackageFeed())), + (r'^news/$', cache_page(311)(NewsFeed())), + (r'^packages/$', cache_page(313)(PackageFeed())), (r'^packages/(?P[A-z0-9]+)/$', - cache_page(300)(PackageFeed())), + cache_page(313)(PackageFeed())), (r'^packages/all/(?P[A-z0-9\-]+)/$', - cache_page(300)(PackageFeed())), + cache_page(313)(PackageFeed())), (r'^packages/(?P[A-z0-9]+)/(?P[A-z0-9\-]+)/$', - cache_page(300)(PackageFeed())), - (r'^releases/$', cache_page(300)(ReleaseFeed())), + cache_page(313)(PackageFeed())), + (r'^releases/$', cache_page(317)(ReleaseFeed())), ) # Includes and other remaining stuff urlpatterns += patterns('', - # cache this static JS resource for 1 week rather than default 5 minutes (r'^admin/', include(admin.site.urls)), (r'^devel/', include('devel.urls')), (r'^feeds/', include(feeds_patterns)), @@ -80,10 +79,10 @@ urlpatterns += patterns('retro.views', # Sitemaps urlpatterns += patterns('', (r'^sitemap.xml$', - cache_page(1800)(sitemap_views.index), + cache_page(1831)(sitemap_views.index), {'sitemaps': our_sitemaps, 'sitemap_url_name': 'sitemaps'}), (r'^sitemap-(?P
.+)\.xml$', - cache_page(1800)(sitemap_views.sitemap), + cache_page(1831)(sitemap_views.sitemap), {'sitemaps': our_sitemaps}, 'sitemaps'), ) -- cgit v1.2.3 From f7d1940a731370ceee6e1c6eaae2cc2f5bab0432 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 19 Oct 2014 18:30:32 -0500 Subject: Remove usage of templates for RSS feeds Signed-off-by: Dan McGee --- feeds.py | 8 ++++++-- templates/feeds/news_description.html | 2 -- templates/feeds/packages_title.html | 1 - 3 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 templates/feeds/news_description.html delete mode 100644 templates/feeds/packages_title.html diff --git a/feeds.py b/feeds.py index d1836178..0bbac270 100644 --- a/feeds.py +++ b/feeds.py @@ -79,7 +79,6 @@ class PackageFeed(Feed): feed_type = GuidNotPermalinkFeed link = '/packages/' - title_template = 'feeds/packages_title.html' def __call__(self, request, *args, **kwargs): wrapper = condition(etag_func=package_etag, last_modified_func=package_last_modified) @@ -142,6 +141,9 @@ class PackageFeed(Feed): def item_pubdate(self, item): return item.last_update + def item_title(self, item): + return '%s %s %s' % (item.pkgname, item.full_version, item.arch.name) + def item_description(self, item): return item.pkgdesc @@ -168,7 +170,6 @@ class NewsFeed(Feed): link = '/news/' description = 'The latest and greatest news from the Arch Linux distribution.' subtitle = description - description_template = 'feeds/news_description.html' def __call__(self, request, *args, **kwargs): wrapper = condition(etag_func=news_etag, last_modified_func=news_last_modified) @@ -192,6 +193,9 @@ class NewsFeed(Feed): def item_title(self, item): return item.title + def item_description(self, item): + return item.html() + class ReleaseFeed(Feed): feed_type = GuidNotPermalinkFeed diff --git a/templates/feeds/news_description.html b/templates/feeds/news_description.html deleted file mode 100644 index 61ceedf3..00000000 --- a/templates/feeds/news_description.html +++ /dev/null @@ -1,2 +0,0 @@ -

{{obj.author.get_full_name}} wrote:

-{{ obj.html }} diff --git a/templates/feeds/packages_title.html b/templates/feeds/packages_title.html deleted file mode 100644 index f92ac684..00000000 --- a/templates/feeds/packages_title.html +++ /dev/null @@ -1 +0,0 @@ -{{ obj.pkgname }} {{ obj.full_version }} {{ obj.arch.name }} \ No newline at end of file -- cgit v1.2.3 From f9f8683799ef96904a7165adcfdeb0d13cb7ff61 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 20 Oct 2014 07:55:24 -0500 Subject: Fix display of flag requests for non-logged-in users More Jinja2 conversion fallout, whoops. Signed-off-by: Dan McGee --- templates/packages/details.html.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/packages/details.html.jinja b/templates/packages/details.html.jinja index 21be80f5..2215f8dc 100644 --- a/templates/packages/details.html.jinja +++ b/templates/packages/details.html.jinja @@ -183,7 +183,7 @@ Last Updated: {{ pkg.last_update|date("DATETIME_FORMAT") }} UTC - {% if user.is_authenticated %}{% with flag_request = pkg.flag_request() %}{% if flag_request %} + {% if user.is_authenticated() %}{% with flag_request = pkg.flag_request() %}{% if flag_request %} Last Flag Request: From {{ flag_request.who() }} on {{ flag_request.created|date }}:
{{ flag_request.message|linebreaksbr|default("{no message}") }}
-- cgit v1.2.3