From dc94eade03022ce3a5286f5e781576321a5f1653 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 27 Apr 2012 08:59:00 -0500 Subject: Incomplete-only todolists optimization We can push this down to the database if we know in advance we only need the incomplete lists. This helps our call on the developer dashboard quite a bit; the time of the single query in question drops from >1300ms to around 40ms. Signed-off-by: Dan McGee --- todolists/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'todolists/utils.py') diff --git a/todolists/utils.py b/todolists/utils.py index 24101e86..94f39f71 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -3,7 +3,7 @@ from main.models import Todolist -def get_annotated_todolists(): +def get_annotated_todolists(incomplete_only=False): qs = Todolist.objects.all() lists = qs.select_related('creator').defer( 'creator__email', 'creator__password', 'creator__is_staff', @@ -13,8 +13,12 @@ def get_annotated_todolists(): incomplete = qs.filter(todolistpkg__complete=False).annotate( Count('todolistpkg')).values_list('id', 'todolistpkg__count') - # tag each list with an incomplete package count lookup = dict(incomplete) + + if incomplete_only: + lists = lists.filter(id__in=lookup.keys()) + + # tag each list with an incomplete package count for todolist in lists: todolist.incomplete_count = lookup.get(todolist.id, 0) -- cgit v1.2.3-54-g00ecf From 160a08bba5324b25abd9e866b884c91d75e597b0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 20 Nov 2012 15:57:05 -0600 Subject: Improve performance of todolists query Use some standard SQL and split the query into two different parts to save a lot of unnecessary sorting and field retrieval at the database level. The `CASE WHEN complete THEN 1 ELSE 0 END` syntax should be accepted by any database that implements proper SQL; it was tested in PostgreSQL and sqlite3 without issues. Signed-off-by: Dan McGee --- todolists/utils.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) (limited to 'todolists/utils.py') diff --git a/todolists/utils.py b/todolists/utils.py index 94f39f71..03c47931 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -1,26 +1,37 @@ +from django.db import connections, router from django.db.models import Count -from main.models import Todolist +from main.models import Todolist, TodolistPkg -def get_annotated_todolists(incomplete_only=False): - qs = Todolist.objects.all() - lists = qs.select_related('creator').defer( - 'creator__email', 'creator__password', 'creator__is_staff', - 'creator__is_active', 'creator__is_superuser', - 'creator__last_login', 'creator__date_joined').annotate( - pkg_count=Count('todolistpkg')).order_by('-date_added') - incomplete = qs.filter(todolistpkg__complete=False).annotate( - Count('todolistpkg')).values_list('id', 'todolistpkg__count') +def todo_counts(): + sql = """ +SELECT list_id, count(*), sum(CASE WHEN complete THEN 1 ELSE 0 END) + FROM todolist_pkgs + GROUP BY list_id + """ + database = router.db_for_write(TodolistPkg) + connection = connections[database] + cursor = connection.cursor() + cursor.execute(sql) + results = cursor.fetchall() + return {row[0]: (row[1], row[2]) for row in results} - lookup = dict(incomplete) - if incomplete_only: - lists = lists.filter(id__in=lookup.keys()) +def get_annotated_todolists(incomplete_only=False): + lists = Todolist.objects.all().select_related( + 'creator').order_by('-date_added') + lookup = todo_counts() - # tag each list with an incomplete package count + # tag each list with package counts for todolist in lists: - todolist.incomplete_count = lookup.get(todolist.id, 0) + counts = lookup.get(todolist.id, (0, 0)) + todolist.pkg_count = counts[0] + todolist.complete_count = counts[1] + todolist.incomplete_count = counts[0] - counts[1] + + if incomplete_only: + lists = [l for l in lists if l.incomplete_count > 0] return lists -- cgit v1.2.3-54-g00ecf From c8ece67cec9c421ac0c711554edd34f022623b45 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 28 Dec 2012 00:27:20 -0600 Subject: Convert to using new todolist models everywhere This is a rather widespread set of changes converting usage to the new todo list and todo list package model recently introduced. The data migration is not included in this commit. After this commit, the old model should no longer be referenced anywhere. Signed-off-by: Dan McGee --- devel/views.py | 12 +++-- sitestatic/archweb.css | 5 ++ sitestatic/archweb.js | 17 +++---- templates/devel/index.html | 10 ++-- templates/todolists/email_notification.txt | 2 +- templates/todolists/list.html | 4 +- templates/todolists/public_list.html | 18 +++---- templates/todolists/view.html | 27 +++++------ todolists/urls.py | 2 +- todolists/utils.py | 14 +++--- todolists/views.py | 76 +++++++++++++++++------------- 11 files changed, 97 insertions(+), 90 deletions(-) (limited to 'todolists/utils.py') diff --git a/devel/views.py b/devel/views.py index 7d5947d1..e01590a0 100644 --- a/devel/views.py +++ b/devel/views.py @@ -20,11 +20,12 @@ from django.utils.timezone import now from .forms import ProfileForm, UserProfileForm, NewUserForm -from main.models import Package, PackageFile, TodolistPkg +from main.models import Package, PackageFile from main.models import Arch, Repo from news.models import News from packages.models import PackageRelation, Signoff, FlagRequest, Depend from packages.utils import get_signoff_groups +from todolists.models import TodolistPackage from todolists.utils import get_annotated_todolists from .utils import get_annotated_maintainers, UserFinder @@ -41,10 +42,11 @@ def index(request): flagged = Package.objects.normal().filter( flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname') - todopkgs = TodolistPkg.objects.select_related( - 'pkg', 'pkg__arch', 'pkg__repo').filter(complete=False) - todopkgs = todopkgs.filter(pkg__pkgbase__in=inner_q).order_by( - 'list__name', 'pkg__pkgname') + todopkgs = TodolistPackage.objects.select_related( + 'todolist', 'pkg', 'arch', 'repo').exclude( + status=TodolistPackage.COMPLETE) + todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by( + 'todolist__name', 'pkgname') todolists = get_annotated_todolists(incomplete_only=True) diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index 6d6e1569..cfa30f5e 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -999,6 +999,11 @@ ul.admin-actions { .todo-table .incomplete { color: red; } + +.todo-table .inprogress { + color: darkorange; +} + .todo-info { margin: 0; color: #999; } diff --git a/sitestatic/archweb.js b/sitestatic/archweb.js index 42efb3fa..4a02fb63 100644 --- a/sitestatic/archweb.js +++ b/sitestatic/archweb.js @@ -15,7 +15,12 @@ if (typeof $ !== 'undefined' && typeof $.tablesorter !== 'undefined') { id: 'todostatus', is: function(s) { return false; }, format: function(s) { - return s.match(/incomplete/i) ? 1 : 0; + if (s.match(/incomplete/i)) { + return 1; + } else if (s.match(/in-progress/i)) { + return 0.5; + } + return 0; }, type: 'numeric' }); @@ -304,13 +309,9 @@ function todolist_flag() { // TODO: fix usage of this var link = this; $.getJSON(link.href, function(data) { - if (data.complete) { - $(link).text('Complete').addClass( - 'complete').removeClass('incomplete'); - } else { - $(link).text('Incomplete').addClass( - 'incomplete').removeClass('complete'); - } + $(link).text(data.status).removeClass( + 'complete inprogress incomplete').addClass( + data.css_class.toLowerCase()); /* let tablesorter know the cell value has changed */ $('.results').trigger('updateCell', [$(link).closest('td')[0], false, null]); }); diff --git a/templates/devel/index.html b/templates/devel/index.html index a07a4190..57151041 100644 --- a/templates/devel/index.html +++ b/templates/devel/index.html @@ -59,11 +59,11 @@

My Incomplete Todo List Packages

{% for todopkg in todopkgs %} - {{ todopkg.list.name }} + {{ todopkg.todolist.name }} {% pkg_details_link todopkg.pkg %} - {{ todopkg.pkg.repo.name }} - {{ todopkg.pkg.arch.name }} + {{ todopkg.repo.name }} + {{ todopkg.arch.name }} {{ todopkg.pkg.maintainers|join:', ' }} {% empty %} @@ -90,7 +90,7 @@

Package Todo Lists

{{ todo.name }} - {{ todo.date_added|date }} + {{ todo.created|date }} {{ todo.creator.get_full_name }} {{ todo.description|urlize }} {{ todo.pkg_count }} diff --git a/templates/todolists/email_notification.txt b/templates/todolists/email_notification.txt index 8b22b465..e454ec79 100644 --- a/templates/todolists/email_notification.txt +++ b/templates/todolists/email_notification.txt @@ -1,7 +1,7 @@ {% autoescape off %}The todo list "{{ todolist.name }}" has had the following packages added to it for which you are a maintainer: {% for tpkg in todo_packages %} -* {{ tpkg.pkg.repo.name|lower }}/{{ tpkg.pkg.pkgname }} ({{ tpkg.pkg.arch.name }}) - {{ tpkg.pkg.get_full_url }}{% endfor %} +* {{ tpkg.repo.name|lower }}/{{ tpkg.pkgname }} ({{ tpkg.arch.name }}) - {{ tpkg.pkg.get_full_url }}{% endfor %} Todo list information: Name: {{ todolist.name }} diff --git a/templates/todolists/list.html b/templates/todolists/list.html index 51f9b570..4e456d28 100644 --- a/templates/todolists/list.html +++ b/templates/todolists/list.html @@ -8,7 +8,7 @@

Package Todo Lists

- {% if perms.main.add_todolist %} + {% if perms.todolists.add_todolist %} @@ -31,7 +31,7 @@

Package Todo Lists

{{ list.name }} - {{ list.date_added|date }} + {{ list.created|date }} {{ list.creator.get_full_name }} {{ list.description|urlize }} {{ list.pkg_count }} diff --git a/templates/todolists/public_list.html b/templates/todolists/public_list.html index 9e0e5234..41caf5b0 100644 --- a/templates/todolists/public_list.html +++ b/templates/todolists/public_list.html @@ -30,7 +30,7 @@

Open Developer Todo Lists

{{ list.name }}

-

{{ list.date_added|date }} - {{ list.creator.get_full_name }}

+

{{ list.created|date }} - {{ list.creator.get_full_name }}

{{ list.description|urlize|linebreaks }}
@@ -45,17 +45,11 @@

{{ list.name }}

{% for pkg in list.packages %} - - - - - + + + + + {% endfor %} diff --git a/templates/todolists/view.html b/templates/todolists/view.html index b6f59704..0f3475a2 100644 --- a/templates/todolists/view.html +++ b/templates/todolists/view.html @@ -10,17 +10,17 @@

Todo List: {{ list.name }}

    - {% if perms.main.delete_todolist %} + {% if perms.todolists.delete_todolist %}
  • Delete Todo List
    • {% endif %} - {% if perms.main.change_todolist %} + {% if perms.todolists.change_todolist %}
  • Edit Todo List
    • {% endif %}
-

{{ list.date_added|date }} - {{ list.creator.get_full_name }}

+

{{ list.created|date }} - {{ list.creator.get_full_name }}

{{list.description|urlize|linebreaks}}
@@ -68,27 +68,22 @@

Filter Todo List Packages

{% for pkg in list.packages %} - - - - + + + + {% if pkg.pkg.flag_date %} {% else %} {% endif %} - + diff --git a/todolists/urls.py b/todolists/urls.py index a379468f..81ac11f5 100644 --- a/todolists/urls.py +++ b/todolists/urls.py @@ -11,7 +11,7 @@ (r'^edit/(?P\d+)/$', 'edit'), (r'^flag/(\d+)/(\d+)/$', 'flag'), (r'^delete/(?P\d+)/$', - permission_required('main.delete_todolist')(DeleteTodolist.as_view())), + permission_required('todolists.delete_todolist')(DeleteTodolist.as_view())), ) # vim: set ts=4 sw=4 et: diff --git a/todolists/utils.py b/todolists/utils.py index 03c47931..d084c645 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -1,26 +1,26 @@ from django.db import connections, router from django.db.models import Count -from main.models import Todolist, TodolistPkg +from .models import Todolist, TodolistPackage def todo_counts(): sql = """ -SELECT list_id, count(*), sum(CASE WHEN complete THEN 1 ELSE 0 END) - FROM todolist_pkgs - GROUP BY list_id +SELECT todolist_id, count(*), sum(CASE WHEN status = %s THEN 1 ELSE 0 END) + FROM todolists_todolistpackage + GROUP BY todolist_id """ - database = router.db_for_write(TodolistPkg) + database = router.db_for_write(TodolistPackage) connection = connections[database] cursor = connection.cursor() - cursor.execute(sql) + cursor.execute(sql, [TodolistPackage.COMPLETE]) results = cursor.fetchall() return {row[0]: (row[1], row[2]) for row in results} def get_annotated_todolists(incomplete_only=False): lists = Todolist.objects.all().select_related( - 'creator').order_by('-date_added') + 'creator').order_by('-created') lookup = todo_counts() # tag each list with package counts diff --git a/todolists/views.py b/todolists/views.py index 9984ef9a..94164391 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -10,39 +10,45 @@ from django.views.generic import DeleteView from django.template import Context, loader -from main.models import Todolist, TodolistPkg, Package, Repo +from main.models import Package, Repo from packages.utils import attach_maintainers +from .models import Todolist, TodolistPackage from .utils import get_annotated_todolists + class TodoListForm(forms.ModelForm): - packages = forms.CharField(required=False, + raw = forms.CharField(label='Packages', required=False, help_text='(one per line)', widget=forms.Textarea(attrs={'rows': '20', 'cols': '60'})) - def clean_packages(self): - package_names = [s.strip() for s in - self.cleaned_data['packages'].split("\n")] - package_names = set(package_names) - packages = Package.objects.filter(pkgname__in=package_names).filter( - repo__testing=False, repo__staging=False).select_related( - 'arch', 'repo').order_by('arch') - return packages + def packages(self): + package_names = {s.strip() for s in + self.cleaned_data['raw'].split("\n")} + return Package.objects.normal().filter(pkgname__in=package_names, + repo__testing=False, repo__staging=False).order_by('arch') class Meta: model = Todolist - fields = ('name', 'description') + fields = ('name', 'description', 'raw') + -@permission_required('main.change_todolistpkg') +@permission_required('todolists.change_todolistpackage') @never_cache def flag(request, list_id, pkg_id): todolist = get_object_or_404(Todolist, id=list_id) - pkg = get_object_or_404(TodolistPkg, id=pkg_id) - pkg.complete = not pkg.complete - pkg.save() + tlpkg = get_object_or_404(TodolistPackage, id=pkg_id) + # TODO: none of this; require absolute value on submit + if tlpkg.status == TodolistPackage.INCOMPLETE: + tlpkg.status = TodolistPackage.COMPLETE + else: + tlpkg.status = TodolistPackage.INCOMPLETE + tlpkg.save() if request.is_ajax(): - return HttpResponse( - json.dumps({'complete': pkg.complete}), - mimetype='application/json') + data = { + 'status': tlpkg.get_status_display(), + 'css_class': tlpkg.status_css_class(), + } + return HttpResponse(json.dumps(data), mimetype='application/json') return redirect(todolist) @login_required @@ -52,9 +58,9 @@ def view(request, list_id): 'svn_root', flat=True).order_by().distinct() # we don't hold onto the result, but the objects are the same here, # so accessing maintainers in the template is now cheap - attach_maintainers(tp.pkg for tp in todolist.packages) - arches = {tp.pkg.arch for tp in todolist.packages} - repos = {tp.pkg.repo for tp in todolist.packages} + attach_maintainers(todolist.packages()) + arches = {tp.arch for tp in todolist.packages()} + repos = {tp.repo for tp in todolist.packages()} return render(request, 'todolists/view.html', { 'list': todolist, 'svn_roots': svn_roots, @@ -67,9 +73,9 @@ def list_pkgbases(request, list_id, svn_root): '''Used to make bulk moves of packages a lot easier.''' todolist = get_object_or_404(Todolist, id=list_id) repos = get_list_or_404(Repo, svn_root=svn_root) - pkgbases = {tp.pkg.pkgbase for tp in todolist.packages - if tp.pkg.repo in repos} - return HttpResponse('\n'.join(sorted(pkgbases)), + pkgbases = TodolistPackage.objects.values_list('pkgbase', flat=True).filter( + todolist=todolist, repo__in=repos).distinct().order_by('pkgbase') + return HttpResponse('\n'.join(pkgbases), mimetype='text/plain') @login_required @@ -77,7 +83,7 @@ def todolist_list(request): lists = get_annotated_todolists() return render(request, 'todolists/list.html', {'lists': lists}) -@permission_required('main.add_todolist') +@permission_required('todolists.add_todolist') @never_cache def add(request): if request.POST: @@ -98,7 +104,7 @@ def add(request): return render(request, 'general_form.html', page_dict) # TODO: this calls for transaction management and async emailing -@permission_required('main.change_todolist') +@permission_required('todolists.change_todolist') @never_cache def edit(request, list_id): todo_list = get_object_or_404(Todolist, id=list_id) @@ -110,7 +116,7 @@ def edit(request, list_id): return redirect(todo_list) else: form = TodoListForm(instance=todo_list, - initial={ 'packages': '\n'.join(todo_list.package_names) }) + initial={ 'packages': todo_list.raw }) page_dict = { 'title': 'Edit Todo List: %s' % todo_list.name, @@ -128,7 +134,7 @@ class DeleteTodolist(DeleteView): @transaction.commit_on_success def create_todolist_packages(form, creator=None): - packages = form.cleaned_data['packages'] + packages = form.packages() if creator: # todo list is new todolist = form.save(commit=False) @@ -140,18 +146,22 @@ def create_todolist_packages(form, creator=None): # todo list already existed form.save() todolist = form.instance + # first delete any packages not in the new list - for todo_pkg in todolist.packages: + for todo_pkg in todolist.packages(): if todo_pkg.pkg not in packages: todo_pkg.delete() # save the old package list so we know what to add - old_packages = [p.pkg for p in todolist.packages] + old_packages = [todo_pkg.pkg for todo_pkg in todolist.packages()] todo_pkgs = [] for package in packages: if package not in old_packages: - todo_pkg = TodolistPkg.objects.create(list=todolist, pkg=package) + todo_pkg = TodolistPackage.objects.create(todolist=todolist, + pkg=package, pkgname=package.pkgname, + pkgbase=package.pkgbase, + arch=package.arch, repo=package.repo) todo_pkgs.append(todo_pkg) return todo_pkgs @@ -186,8 +196,8 @@ def send_todolist_emails(todo_list, new_packages): def public_list(request): todo_lists = Todolist.objects.incomplete() # total hackjob, but it makes this a lot less query-intensive. - all_pkgs = [tp for tl in todo_lists for tp in tl.packages] - attach_maintainers([tp.pkg for tp in all_pkgs]) + all_pkgs = [tp for tl in todo_lists for tp in tl.packages()] + attach_maintainers(all_pkgs) return render(request, "todolists/public_list.html", {"todo_lists": todo_lists}) -- cgit v1.2.3-54-g00ecf From 39a603bf65c4aec780e4711074e9ed27fb7c301e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 29 Dec 2012 12:26:50 -0600 Subject: Defer the 'raw' field when listing todolists A lot like skipping fetching of the news content; we definitely don't need this just to list the todolists on index pages. Signed-off-by: Dan McGee --- todolists/utils.py | 2 +- todolists/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'todolists/utils.py') diff --git a/todolists/utils.py b/todolists/utils.py index d084c645..0daca3b6 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -19,7 +19,7 @@ def todo_counts(): def get_annotated_todolists(incomplete_only=False): - lists = Todolist.objects.all().select_related( + lists = Todolist.objects.all().defer('raw').select_related( 'creator').order_by('-created') lookup = todo_counts() diff --git a/todolists/views.py b/todolists/views.py index 413d8675..788d74f2 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -206,7 +206,7 @@ def send_todolist_emails(todo_list, new_packages): def public_list(request): - todo_lists = Todolist.objects.incomplete() + todo_lists = Todolist.objects.incomplete().defer('raw') # total hackjob, but it makes this a lot less query-intensive. all_pkgs = [tp for tl in todo_lists for tp in tl.packages()] attach_maintainers(all_pkgs) -- cgit v1.2.3-54-g00ecf From 7952fe0ede3a5a68a64f05eccb180194394652f3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 31 Dec 2012 11:31:35 -0600 Subject: Mark todolist packages as removed rather than deleting them This makes it easier to see the progression of a todolist and its contents easier since we are no longer losing the data. Signed-off-by: Dan McGee --- devel/views.py | 2 +- todolists/models.py | 8 +++++--- todolists/utils.py | 1 + todolists/views.py | 50 +++++++++++++++++++++++++++++++------------------- 4 files changed, 38 insertions(+), 23 deletions(-) (limited to 'todolists/utils.py') diff --git a/devel/views.py b/devel/views.py index e01590a0..90839847 100644 --- a/devel/views.py +++ b/devel/views.py @@ -44,7 +44,7 @@ def index(request): todopkgs = TodolistPackage.objects.select_related( 'todolist', 'pkg', 'arch', 'repo').exclude( - status=TodolistPackage.COMPLETE) + status=TodolistPackage.COMPLETE).filter(removed__isnull=True) todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by( 'todolist__name', 'pkgname') diff --git a/todolists/models.py b/todolists/models.py index 156b041d..040f8a29 100644 --- a/todolists/models.py +++ b/todolists/models.py @@ -10,8 +10,9 @@ class TodolistManager(models.Manager): def incomplete(self): - not_done = (Q(todolistpackage__status=TodolistPackage.INCOMPLETE) | - Q(todolistpackage__status=TodolistPackage.IN_PROGRESS)) + not_done = ((Q(todolistpackage__status=TodolistPackage.INCOMPLETE) | + Q(todolistpackage__status=TodolistPackage.IN_PROGRESS)) & + Q(todolistpackage__removed__isnull=True)) return self.order_by().filter(not_done).distinct() @@ -44,7 +45,8 @@ def get_full_url(self, proto='https'): def packages(self): if not hasattr(self, '_packages'): - self._packages = self.todolistpackage_set.select_related( + self._packages = self.todolistpackage_set.filter( + removed__isnull=True).select_related( 'pkg', 'repo', 'arch').order_by('pkgname', 'arch') return self._packages diff --git a/todolists/utils.py b/todolists/utils.py index 0daca3b6..e86d9054 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -8,6 +8,7 @@ def todo_counts(): sql = """ SELECT todolist_id, count(*), sum(CASE WHEN status = %s THEN 1 ELSE 0 END) FROM todolists_todolistpackage + WHERE removed IS NULL GROUP BY todolist_id """ database = router.db_for_write(TodolistPackage) diff --git a/todolists/views.py b/todolists/views.py index 113b27e7..32ee7cc5 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -10,6 +10,7 @@ from django.views.generic import DeleteView from django.template import Context, loader from django.template.defaultfilters import slugify +from django.utils.timezone import now from main.models import Package, Repo from main.utils import find_unique_slug @@ -23,10 +24,12 @@ class TodoListForm(forms.ModelForm): help_text='(one per line)', widget=forms.Textarea(attrs={'rows': '20', 'cols': '60'})) + def package_names(self): + return {s.strip() for s in self.cleaned_data['raw'].split("\n")} + def packages(self): - package_names = {s.strip() for s in - self.cleaned_data['raw'].split("\n")} - return Package.objects.normal().filter(pkgname__in=package_names, + return Package.objects.normal().filter( + pkgname__in=self.package_names(), repo__testing=False, repo__staging=False).order_by('arch') class Meta: @@ -37,7 +40,7 @@ class Meta: @never_cache def flag(request, slug, pkg_id): todolist = get_object_or_404(Todolist, slug=slug) - tlpkg = get_object_or_404(TodolistPackage, id=pkg_id) + tlpkg = get_object_or_404(TodolistPackage, id=pkg_id, removed__isnull=True) # TODO: none of this; require absolute value on submit if tlpkg.status == TodolistPackage.INCOMPLETE: tlpkg.status = TodolistPackage.COMPLETE @@ -75,13 +78,14 @@ def view(request, slug): }) -# really no need for login_required on this one... def list_pkgbases(request, slug, svn_root): '''Used to make bulk moves of packages a lot easier.''' todolist = get_object_or_404(Todolist, slug=slug) repos = get_list_or_404(Repo, svn_root=svn_root) - pkgbases = TodolistPackage.objects.values_list('pkgbase', flat=True).filter( - todolist=todolist, repo__in=repos).distinct().order_by('pkgbase') + pkgbases = TodolistPackage.objects.values_list( + 'pkgbase', flat=True).filter( + todolist=todolist, repo__in=repos, removed__isnull=True).order_by( + 'pkgbase').distinct() return HttpResponse('\n'.join(pkgbases), mimetype='text/plain') @@ -143,36 +147,44 @@ class DeleteTodolist(DeleteView): @transaction.commit_on_success def create_todolist_packages(form, creator=None): + package_names = form.package_names() packages = form.packages() + timestamp = now() if creator: # todo list is new, populate creator and slug fields todolist = form.save(commit=False) todolist.creator = creator todolist.slug = find_unique_slug(Todolist, todolist.name) todolist.save() - - old_packages = [] else: # todo list already existed form.save() todolist = form.instance - # first delete any packages not in the new list + # first mark removed any packages not in the new list + to_remove = set() for todo_pkg in todolist.packages(): - if todo_pkg.pkg not in packages: - todo_pkg.delete() + if todo_pkg.pkg and todo_pkg.pkg not in packages: + to_remove.add(todo_pkg.pk) + elif todo_pkg.pkgname not in package_names: + to_remove.add(todo_pkg.pk) - # save the old package list so we know what to add - old_packages = [todo_pkg.pkg for todo_pkg in todolist.packages()] + TodolistPackage.objects.filter( + pk__in=to_remove).update(removed=timestamp) + # Add (or mark unremoved) any packages in the new packages list todo_pkgs = [] for package in packages: - if package not in old_packages: - todo_pkg = TodolistPackage.objects.create(todolist=todolist, - pkg=package, pkgname=package.pkgname, - pkgbase=package.pkgbase, - arch=package.arch, repo=package.repo) + todo_pkg, created = TodolistPackage.objects.get_or_create( + todolist=todolist, + pkg=package, pkgname=package.pkgname, + pkgbase=package.pkgbase, + arch=package.arch, repo=package.repo) + if created: todo_pkgs.append(todo_pkg) + elif todo_pkg.removed is not None: + todo_pkg.removed = None + todo_pkg.save() return todo_pkgs -- cgit v1.2.3-54-g00ecf From e9e1c071654edd7b95e20c8105abbc23f426cecc Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 22 Jan 2013 16:47:43 -0600 Subject: Show staging version on todolist view page If one exists, it is easy enough to show it here so in-progress todolists can easily be cross-checked with the current state of the repository. Signed-off-by: Dan McGee --- templates/todolists/view.html | 5 +++++ todolists/utils.py | 19 +++++++++++++++++++ todolists/views.py | 3 ++- 3 files changed, 26 insertions(+), 1 deletion(-) (limited to 'todolists/utils.py') diff --git a/templates/todolists/view.html b/templates/todolists/view.html index 86221127..e544fa12 100644 --- a/templates/todolists/view.html +++ b/templates/todolists/view.html @@ -1,5 +1,6 @@ {% extends "base.html" %} {% load static from staticfiles %} +{% load package_extras %} {% load todolists %} {% block title %}Arch Linux - Todo: {{ list.name }}{% endblock %} @@ -62,6 +63,7 @@

Filter Todo List Packages

 + @@ -77,6 +79,9 @@

Filter Todo List Packages

{% else %} {% endif %} + {% with staging=pkg.staging %} + + {% endwith %}
{% pkg_details_link pkg.pkg %}{{ pkg.pkg.arch.name }}{{ pkg.pkg.repo.name|capfirst }}{{ pkg.pkg.maintainers|join:', ' }} - {% if pkg.complete %} - Complete - {% else %} - Incomplete - {% endif %} - {% pkg_details_link pkg.pkg pkg.pkgname %}{{ pkg.arch.name }}{{ pkg.repo.name|capfirst }}{{ pkg.maintainers|join:', ' }}{{ pkg.get_status_display }}
{{ pkg.pkg.arch.name }}{{ pkg.pkg.repo.name|capfirst }}{% pkg_details_link pkg.pkg %}
{{ pkg.arch.name }}{{ pkg.repo.name|capfirst }}{% pkg_details_link pkg.pkg pkg.pkgname %}{{ pkg.pkg.full_version }}{{ pkg.pkg.full_version }}{{ pkg.pkg.maintainers|join:', ' }}{{ pkg.maintainers|join:', ' }} - {% if perms.main.change_todolistpkg %} - {% if pkg.complete %} + {% if perms.todolist.change_todolistpackage %} Complete + class="status-link {{ pkg.status_css_class }}" title="Toggle completion status">{{ pkg.get_status_display }} {% else %} - Incomplete - {% endif %} - {% else %} - {% if pkg.complete %}Complete{% else %}Incomplete{% endif %} + {{ pkg.get_status_display }} {% endif %}
Repository Name Current VersionStaging Version Maintainers Status
{{ pkg.pkg.full_version }}{% pkg_details_link staging staging.full_version %}{{ pkg.maintainers|join:', ' }} {% if perms.todolists.change_todolistpackage %} diff --git a/todolists/utils.py b/todolists/utils.py index e86d9054..51a75a3c 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -2,6 +2,7 @@ from django.db.models import Count from .models import Todolist, TodolistPackage +from packages.models import Package def todo_counts(): @@ -36,4 +37,22 @@ def get_annotated_todolists(incomplete_only=False): return lists + +def attach_staging(packages, list_id): + '''Look for any staging version of the packages provided and attach them + to the 'staging' attribute on each package if found.''' + pkgnames = TodolistPackage.objects.filter( + todolist_id=list_id).values('pkgname') + staging_pkgs = Package.objects.normal().filter(repo__staging=True, + pkgname__in=pkgnames) + # now build a lookup dict to attach to the correct package + lookup = {(p.pkgname, p.arch): p for p in staging_pkgs} + + annotated = [] + for package in packages: + in_staging = lookup.get((package.pkgname, package.arch), None) + package.staging = in_staging + + return annotated + # vim: set ts=4 sw=4 et: diff --git a/todolists/views.py b/todolists/views.py index fcf62e23..f333728a 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -16,7 +16,7 @@ from main.utils import find_unique_slug from packages.utils import attach_maintainers from .models import Todolist, TodolistPackage -from .utils import get_annotated_todolists +from .utils import get_annotated_todolists, attach_staging class TodoListForm(forms.ModelForm): @@ -69,6 +69,7 @@ def view(request, slug): # we don't hold onto the result, but the objects are the same here, # so accessing maintainers in the template is now cheap attach_maintainers(todolist.packages()) + attach_staging(todolist.packages(), todolist.pk) arches = {tp.arch for tp in todolist.packages()} repos = {tp.repo for tp in todolist.packages()} return render(request, 'todolists/view.html', { -- cgit v1.2.3-54-g00ecf From b7b24740640e24883cd17fd683e1d465fbb343f8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 22:12:01 -0500 Subject: Various minor code cleanups and fixes Most of these were suggested by PyCharm, and include everything from little syntax issues and other bad smells to dead or bad code. Signed-off-by: Dan McGee --- devel/management/commands/pgp_import.py | 1 + devel/models.py | 1 - devel/utils.py | 2 +- devel/views.py | 2 +- main/log.py | 1 - main/migrations/0029_fill_in_repo_data.py | 1 - main/models.py | 12 +++++------- main/utils.py | 1 - mirrors/management/commands/mirrorcheck.py | 12 +++--------- mirrors/models.py | 2 +- mirrors/utils.py | 6 +++--- packages/migrations/0002_populate_package_relation.py | 2 -- packages/templatetags/package_extras.py | 4 ++-- packages/utils.py | 2 +- packages/views/display.py | 2 -- packages/views/flag.py | 3 +-- public/views.py | 1 - releng/management/commands/syncisos.py | 2 +- releng/models.py | 2 +- releng/views.py | 2 +- retro/templates/retro/index-20030330.html | 1 - sitestatic/archweb.js | 1 - todolists/utils.py | 1 - todolists/views.py | 1 - visualize/static/visualize.js | 2 +- 25 files changed, 23 insertions(+), 44 deletions(-) (limited to 'todolists/utils.py') diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py index 10e6cfcb..b1f29d77 100644 --- a/devel/management/commands/pgp_import.py +++ b/devel/management/commands/pgp_import.py @@ -95,6 +95,7 @@ def parse_keydata(data): # parse all of the output from our successful GPG command logger.info("parsing command output") + node = None for line in data.split('\n'): parts = line.split(':') if parts[0] == 'pub': diff --git a/devel/models.py b/devel/models.py index 67de40a6..4354e0f2 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,7 +4,6 @@ from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User -from django.utils.timezone import now from django_countries import CountryField from .fields import PGPKeyField diff --git a/devel/utils.py b/devel/utils.py index e8e3a6c4..340841f5 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -131,7 +131,7 @@ def find(self, userstring): self.username_email, self.user_name) for matcher in find_methods: user = matcher(name, email) - if user != None: + if user is not None: break self.cache[userstring] = user diff --git a/devel/views.py b/devel/views.py index 61c1e568..4258ea7f 100644 --- a/devel/views.py +++ b/devel/views.py @@ -34,7 +34,7 @@ @login_required def index(request): '''the developer dashboard''' - if(request.user.is_authenticated()): + if request.user.is_authenticated(): inner_q = PackageRelation.objects.filter(user=request.user) else: inner_q = PackageRelation.objects.none() diff --git a/main/log.py b/main/log.py index 63634874..5c745cc8 100644 --- a/main/log.py +++ b/main/log.py @@ -46,7 +46,6 @@ def filter(self, record): trace = '\n'.join(traceback.format_exception(*record.exc_info)) key = md5(trace).hexdigest() - duplicate = False cache = self.cache_module.cache # Test if the cache works diff --git a/main/migrations/0029_fill_in_repo_data.py b/main/migrations/0029_fill_in_repo_data.py index 0887b28c..7da6b1c4 100644 --- a/main/migrations/0029_fill_in_repo_data.py +++ b/main/migrations/0029_fill_in_repo_data.py @@ -7,7 +7,6 @@ class Migration(DataMigration): def forwards(self, orm): - "Write your forwards methods here." orm.Repo.objects.filter(name__istartswith='community').update(bugs_project=5, svn_root='community') orm.Repo.objects.filter(name__iexact='multilib').update(bugs_project=5, svn_root='community') diff --git a/main/models.py b/main/models.py index 89215f05..24aeed89 100644 --- a/main/models.py +++ b/main/models.py @@ -7,7 +7,6 @@ from django.db.models import Q from django.contrib.auth.models import User from django.contrib.sites.models import Site -from django.utils.timezone import now from .fields import PositiveBigIntegerField from .utils import set_created_field @@ -140,7 +139,7 @@ def get_full_url(self, proto='https'): @property def signature(self): try: - data = b64decode(self.pgp_signature) + data = b64decode(self.pgp_signature.encode('utf-8')) except TypeError: return None if not data: @@ -274,7 +273,6 @@ def get_depends(self): Packages will match the testing status of this package if possible. """ deps = [] - arches = None # TODO: we can use list comprehension and an 'in' query to make this # more effective for dep in self.depends.all(): @@ -400,13 +398,13 @@ def elsewhere(self): '''attempt to locate this package anywhere else, regardless of architecture or repository. Excludes this package from the list.''' names = [self.pkgname] - if self.pkgname.startswith('lib32-'): + if self.pkgname.startswith(u'lib32-'): names.append(self.pkgname[6:]) - elif self.pkgname.endswith('-multilib'): + elif self.pkgname.endswith(u'-multilib'): names.append(self.pkgname[:-9]) else: - names.append('lib32-' + self.pkgname) - names.append(self.pkgname + '-multilib') + names.append(u'lib32-' + self.pkgname) + names.append(self.pkgname + u'-multilib') return Package.objects.normal().filter( pkgname__in=names).exclude(id=self.id).order_by( 'arch__name', 'repo__name') diff --git a/main/utils.py b/main/utils.py index 8394e5cd..9ee8db58 100644 --- a/main/utils.py +++ b/main/utils.py @@ -3,7 +3,6 @@ except ImportError: import pickle -from datetime import datetime import hashlib from django.core.cache import cache diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index d6de8f22..e7dd7b49 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -106,19 +106,13 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, location, timeout): - if location: - if location.family == socket.AF_INET6: - ipopt = '--ipv6' - elif location.family == socket.AF_INET: - ipopt = '--ipv4' - url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=now(), location=location) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) + start = time.time() try: - start = time.time() result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() @@ -147,12 +141,12 @@ def check_mirror_url(mirror_url, location, timeout): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except HTTPException as e: + except HTTPException: # e.g., BadStatusLine log.is_success = False log.error = "Exception in processing HTTP request." logger.debug("failed: %s, %s", url, log.error) - except socket.timeout as e: + except socket.timeout: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) diff --git a/mirrors/models.py b/mirrors/models.py index 791b0078..d8ac7952 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -92,7 +92,7 @@ def clean(self): families = self.address_families() self.has_ipv4 = socket.AF_INET in families self.has_ipv6 = socket.AF_INET6 in families - except socket.error as e: + except socket.error: # We don't fail in this case; we'll just set both to False self.has_ipv4 = False self.has_ipv6 = False diff --git a/mirrors/utils.py b/mirrors/utils.py index 5a8bbf5d..531cf005 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,13 @@ from datetime import timedelta from django.db import connection -from django.db.models import Avg, Count, Max, Min, StdDev +from django.db.models import Count, Max, Min from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country from main.utils import cache_function, database_vendor -from .models import MirrorLog, MirrorProtocol, MirrorUrl +from .models import MirrorLog, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) @@ -165,7 +165,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): ).order_by('-last_occurred', '-error_count') if mirror_id: - urls = urls.filter(mirror_id=mirror_id) + errors = errors.filter(url__mirror_id=mirror_id) errors = list(errors) for err in errors: diff --git a/packages/migrations/0002_populate_package_relation.py b/packages/migrations/0002_populate_package_relation.py index 738e068f..b0d32c7a 100644 --- a/packages/migrations/0002_populate_package_relation.py +++ b/packages/migrations/0002_populate_package_relation.py @@ -11,7 +11,6 @@ class Migration(DataMigration): ) def forwards(self, orm): - "Write your forwards methods here." # search by pkgbase first and insert those records qs = orm['main.Package'].objects.exclude(maintainer=None).exclude( pkgbase=None).distinct().values('pkgbase', 'maintainer_id') @@ -29,7 +28,6 @@ def forwards(self, orm): defaults={'user_id': row['maintainer_id']}) def backwards(self, orm): - "Write your backwards methods here." if not db.dry_run: orm.PackageRelation.objects.all().delete() pass diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index f14fab1e..ef0e1aea 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -53,10 +53,10 @@ def do_buildsortqs(parser, token): tagname, sortfield = token.split_contents() except ValueError: raise template.TemplateSyntaxError( - "%r tag requires a single argument" % tagname) + "%r tag requires a single argument" % token) if not (sortfield[0] == sortfield[-1] and sortfield[0] in ('"', "'")): raise template.TemplateSyntaxError( - "%r tag's argument should be in quotes" % tagname) + "%r tag's argument should be in quotes" % token) return BuildQueryStringNode(sortfield[1:-1]) diff --git a/packages/utils.py b/packages/utils.py index a4217fbd..4f3b8665 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -391,7 +391,7 @@ def signoffs_id_query(model, repos): """ cursor = connection.cursor() # query pre-process- fill in table name and placeholders for IN - repo_sql = ','.join(['%s' for r in repos]) + repo_sql = ','.join(['%s' for _ in repos]) sql = sql % (model._meta.db_table, repo_sql, repo_sql) repo_ids = [r.pk for r in repos] # repo_ids are needed twice, so double the array diff --git a/packages/views/display.py b/packages/views/display.py index 87424483..021c7ed8 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -228,8 +228,6 @@ def download(request, name, repo, arch): if pkg.arch.agnostic: # grab the first non-any arch to fake the download path arch = Arch.objects.exclude(agnostic=True)[0].name - values = { - } url = '{host}{repo}/os/{arch}/{filename}'.format(host=url.url, repo=pkg.repo.name.lower(), arch=arch, filename=pkg.filename) return redirect(url) diff --git a/packages/views/flag.py b/packages/views/flag.py index 5c76e1d5..39cdcef8 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -110,7 +110,7 @@ def perform_updates(): subject = '%s package [%s] marked out-of-date' % \ (pkg.repo.name, pkg.pkgname) for maint in maints: - if maint.userprofile.notify == True: + if maint.userprofile.notify is True: toemail.append(maint.email) if toemail: @@ -133,7 +133,6 @@ def perform_updates(): return redirect('package-flag-confirmed', name=name, repo=repo, arch=arch) else: - initial = {} form = FlagForm(authenticated=authenticated) context = { diff --git a/public/views.py b/public/views.py index 22cb8759..39273396 100644 --- a/public/views.py +++ b/public/views.py @@ -125,7 +125,6 @@ def keys(request): master_keys = MasterKey.objects.select_related('owner', 'revoker', 'owner__userprofile', 'revoker__userprofile').filter( revoked__isnull=True) - master_key_ids = frozenset(key.pgp_key[-16:] for key in master_keys) sig_counts = PGPSignature.objects.filter(not_expired, valid=True, signee__in=user_key_ids).order_by().values_list('signer').annotate( diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index c9f61964..f182cc33 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -20,7 +20,7 @@ def handle_starttag(self, tag, attrs): if tag == 'a': for name, value in attrs: if name == "href": - if value != '../' and self.url_re.search(value) != None: + if value != '../' and self.url_re.search(value) is not None: self.hyperlinks.append(value[:-1]) def parse(self, url): diff --git a/releng/models.py b/releng/models.py index b95f7d52..5ee2f325 100644 --- a/releng/models.py +++ b/releng/models.py @@ -160,7 +160,7 @@ def info_html(self): def torrent(self): try: - data = b64decode(self.torrent_data) + data = b64decode(self.torrent_data.encode('utf-8')) except TypeError: return None if not data: diff --git a/releng/views.py b/releng/views.py index ad4b07d1..b1c76a4a 100644 --- a/releng/views.py +++ b/releng/views.py @@ -231,7 +231,7 @@ def release_torrent(request, version): release = get_object_or_404(Release, version=version) if not release.torrent_data: raise Http404 - data = b64decode(release.torrent_data) + data = b64decode(release.torrent_data.encode('utf-8')) response = HttpResponse(data, content_type='application/x-bittorrent') # TODO: this is duplicated from Release.iso_url() filename = 'archlinux-%s-dual.iso.torrent' % release.version diff --git a/retro/templates/retro/index-20030330.html b/retro/templates/retro/index-20030330.html index 449731af..51cc8ba3 100644 --- a/retro/templates/retro/index-20030330.html +++ b/retro/templates/retro/index-20030330.html @@ -232,7 +232,6 @@
[ Older News ]

-


diff --git a/sitestatic/archweb.js b/sitestatic/archweb.js index dda22d9e..aa225f5f 100644 --- a/sitestatic/archweb.js +++ b/sitestatic/archweb.js @@ -146,7 +146,6 @@ if (typeof $ !== 'undefined' && typeof $.tablesorter !== 'undefined') { (function($) { $.fn.enableCheckboxRangeSelection = function() { var lastCheckbox = null, - lastElement = null, spec = this; spec.unbind("click.checkboxrange"); diff --git a/todolists/utils.py b/todolists/utils.py index 51a75a3c..7b98c887 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -1,5 +1,4 @@ from django.db import connections, router -from django.db.models import Count from .models import Todolist, TodolistPackage from packages.models import Package diff --git a/todolists/views.py b/todolists/views.py index 7636d38e..d5b39934 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -9,7 +9,6 @@ from django.views.decorators.cache import never_cache from django.views.generic import DeleteView from django.template import Context, loader -from django.template.defaultfilters import slugify from django.utils.timezone import now from main.models import Package, Repo diff --git a/visualize/static/visualize.js b/visualize/static/visualize.js index 7e240d44..5004fe6c 100644 --- a/visualize/static/visualize.js +++ b/visualize/static/visualize.js @@ -55,7 +55,7 @@ function packages_treemap(chart_id, orderings, default_order) { var nodes = d3_div.data([json]).selectAll("div") .data(treemap.nodes, key_func); /* start out new nodes in the center of the picture area */ - var w_center = jq_div.width() / 2; + var w_center = jq_div.width() / 2, h_center = jq_div.height() / 2; nodes.enter().append("div") .attr("class", "treemap-cell") -- cgit v1.2.3-54-g00ecf