From 4e872235010b1b7ad8cb6912b9e3ccf4c39a352d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 10 May 2013 17:52:36 -0500 Subject: Simplify with statements by using new syntax Signed-off-by: Dan McGee --- templates/mirrors/status.html | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 283ff681..a3da1ad6 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -59,19 +59,15 @@

Mirror Status{% if tier != None %} - Tier {{ tier }}{% endif %}

Out of Sync Mirrors

- {% with bad_urls as urls %} - {% with 'outofsync_mirrors' as table_id %} + {% with urls=bad_urls table_id='outofsync_mirrors' %} {% include "mirrors/status_table.html" %} {% endwith %} - {% endwith %}

Successfully Syncing Mirrors

- {% with good_urls as urls %} - {% with 'successful_mirrors' as table_id %} + {% with urls=good_urls table_id='successful_mirrors' %} {% include "mirrors/status_table.html" %} {% endwith %} - {% endwith %}

Mirror Syncing Error Log

-- cgit v1.2.3-54-g00ecf From 76be96ac2c11604a2bfceed4212b686719620b13 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 10 May 2013 17:59:53 -0500 Subject: Pull mirror error table into inclusion template Signed-off-by: Dan McGee --- templates/mirrors/error_table.html | 23 +++++++++++++++++++++++ templates/mirrors/status.html | 24 +----------------------- templates/mirrors/status_table.html | 3 +-- 3 files changed, 25 insertions(+), 25 deletions(-) create mode 100644 templates/mirrors/error_table.html diff --git a/templates/mirrors/error_table.html b/templates/mirrors/error_table.html new file mode 100644 index 00000000..6054814f --- /dev/null +++ b/templates/mirrors/error_table.html @@ -0,0 +1,23 @@ +{% load flags mirror_status %} + + + + + + + + + + + + + {% for log in error_logs %} + + + + + + + {% endfor %} + +
Mirror URLProtocolCountryError MessageLast OccurredOccurrences (last {{ cutoff|hours }})
{{ log.url__url }}{{ log.url__protocol__protocol }}{% country_flag log.country %}{{ log.country.name }}{{ log.error|linebreaksbr }}{{ log.last_occurred|date:'Y-m-d H:i' }}{{ log.error_count }}
diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index a3da1ad6..331c18ee 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -71,29 +71,7 @@

Successfully Syncing Mirrors

Mirror Syncing Error Log

- - - - - - - - - - - - - {% for log in error_logs %} - - - - - - - {% endfor %} - -
Mirror URLProtocolCountryError MessageLast OccurredOccurrences (last {{ cutoff|hours }})
{{ log.url__url }}{{ log.url__protocol__protocol }}{% country_flag log.country %}{{ log.country.name }}{{ log.error|linebreaksbr }}{{ log.last_occurred|date:'Y-m-d H:i' }}{{ log.error_count }}
- + {% include "mirrors/error_table.html" %} {% endblock %} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html index e848a9c9..28175420 100644 --- a/templates/mirrors/status_table.html +++ b/templates/mirrors/status_table.html @@ -1,5 +1,4 @@ -{% load mirror_status %} -{% load flags %} +{% load flags mirror_status %} -- cgit v1.2.3-54-g00ecf From 8097a1fdeff36fb3db521a3b354ea8bf576869cc Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 10 May 2013 18:10:04 -0500 Subject: Add mirror error logs to mirror details page Give a window of 7 days for logs here rather than the default 24 hours we do on the main status page since we are only retrieving details for a single mirror with a handful of URLs. This should make it easier to have all information regarding one mirror in a single location. Signed-off-by: Dan McGee --- mirrors/views.py | 11 +++++++++-- templates/mirrors/mirror_details.html | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mirrors/views.py b/mirrors/views.py index 73d40297..d1dd9da3 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -175,6 +175,7 @@ def mirror_details(request, name): if not request.user.is_authenticated() and \ (not mirror.public or not mirror.active): raise Http404 + error_cutoff = timedelta(days=7) status_info = get_mirror_statuses(mirror_id=mirror.id) checked_urls = {url for url in status_info['urls'] \ @@ -188,9 +189,15 @@ def mirror_details(request, name): setattr(url, attr, None) all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) - return render(request, 'mirrors/mirror_details.html', - {'mirror': mirror, 'urls': all_urls}) + error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff) + context = { + 'mirror': mirror, + 'urls': all_urls, + 'cutoff': error_cutoff, + 'error_logs': error_logs, + } + return render(request, 'mirrors/mirror_details.html', context) def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index d3e85b00..6f0ac6c1 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -106,6 +106,9 @@

Available URLs

{% endfor %}
+ +

Error Log

+ {% include "mirrors/error_table.html" %}
-- cgit v1.2.3-54-g00ecf From dad70d0d7c70e7973cd9e3960b24e7f9ef883c61 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 10 May 2013 18:42:00 -0500 Subject: Add a 'last touched by' column to todolist detail view This allows you to see very easily who last manipulated a package todolist item in case you have a need to know. Signed-off-by: Dan McGee --- templates/todolists/view.html | 2 ++ todolists/models.py | 3 ++- todolists/views.py | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/templates/todolists/view.html b/templates/todolists/view.html index fc4b29ea..b26cd91c 100644 --- a/templates/todolists/view.html +++ b/templates/todolists/view.html @@ -68,6 +68,7 @@

Filter Todo List Packages

Staging Version Maintainers Status + Last Touched By @@ -95,6 +96,7 @@

Filter Todo List Packages

{{ pkg.get_status_display }} {% endif %} + {{ pkg.user|default:"" }} {% endfor %} diff --git a/todolists/models.py b/todolists/models.py index 3ea80f37..59b14616 100644 --- a/todolists/models.py +++ b/todolists/models.py @@ -47,7 +47,8 @@ def packages(self): if not hasattr(self, '_packages'): self._packages = self.todolistpackage_set.filter( removed__isnull=True).select_related( - 'pkg', 'repo', 'arch').order_by('pkgname', 'arch') + 'pkg', 'repo', 'arch', 'user__username').order_by( + 'pkgname', 'arch') return self._packages diff --git a/todolists/views.py b/todolists/views.py index d5b39934..ff75686d 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -71,12 +71,13 @@ def view(request, slug): attach_staging(todolist.packages(), todolist.pk) arches = {tp.arch for tp in todolist.packages()} repos = {tp.repo for tp in todolist.packages()} - return render(request, 'todolists/view.html', { + context = { 'list': todolist, 'svn_roots': svn_roots, 'arches': sorted(arches), 'repos': sorted(repos), - }) + } + return render(request, 'todolists/view.html', context) def list_pkgbases(request, slug, svn_root): -- cgit v1.2.3-54-g00ecf From 13bc975d3022ea8e72769ad38de224b9869f5aab Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 12 May 2013 14:57:04 -0500 Subject: Fix reporead issue with temporary database files We shouldn't be tripping ourselves up on "hidden" files. Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 8c1e47bf..6aa4e0e0 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -192,10 +192,11 @@ def my_init(self, **kwargs): def process_default(self, event): '''Primary event processing function which kicks off reporead timer threads if a files database was updated.''' - if not event.name: + name = event.name + if not name: return - # screen to only the files we care about - if event.name.endswith('.files.tar.gz'): + # screen to only the files we care about, skipping temp files + if name.endswith('.files.tar.gz') and not name.startswith('.'): path = event.pathname stat = os.stat(path) database = self.databases.get(path, None) -- cgit v1.2.3-54-g00ecf From 64a92174c056a4cca1e6b7e287ca9993bcd2be1a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 May 2013 20:22:11 -0500 Subject: Begin removal of FTP mirror support FTP is a terrible protocol these days compared to HTTP. IPv6 support is spotty at best, it is much slower for the connect/begin transfer cycle, and overall just doesn't provide anything HTTP does better. Start killing bits that we've added to treat FTP as a first-class protocol and regulate it to the back seat. The expectation here is once this commit goes live to the production site, the FTP mirror URLs themselves will get removed completely from the database, and the FTP protocol object itself will get deleted. Signed-off-by: Dan McGee --- mirrors/admin.py | 1 + mirrors/fixtures/mirrorprotocols.json | 9 --------- mirrors/models.py | 2 +- mirrors/urls_mirrorlist.py | 4 +++- mirrors/views.py | 29 +++++------------------------ templates/mirrors/mirrorlist_generate.html | 2 -- 6 files changed, 10 insertions(+), 37 deletions(-) diff --git a/mirrors/admin.py b/mirrors/admin.py index 9c88207d..d0f2f475 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -10,6 +10,7 @@ class MirrorUrlForm(forms.ModelForm): class Meta: model = MirrorUrl + def clean_url(self): # is this a valid-looking URL? url_parts = urlparse(self.cleaned_data["url"]) diff --git a/mirrors/fixtures/mirrorprotocols.json b/mirrors/fixtures/mirrorprotocols.json index 8822ef8e..1a07510b 100644 --- a/mirrors/fixtures/mirrorprotocols.json +++ b/mirrors/fixtures/mirrorprotocols.json @@ -8,15 +8,6 @@ "protocol": "http" } }, - { - "pk": 2, - "model": "mirrors.mirrorprotocol", - "fields": { - "is_download": true, - "default": false, - "protocol": "ftp" - } - }, { "pk": 3, "model": "mirrors.mirrorprotocol", diff --git a/mirrors/models.py b/mirrors/models.py index d8ac7952..91793e0a 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -47,7 +47,7 @@ def get_absolute_url(self): class MirrorProtocol(models.Model): protocol = models.CharField(max_length=10, unique=True) is_download = models.BooleanField(default=True, - help_text="Is protocol useful for end-users, e.g. FTP/HTTP") + help_text="Is protocol useful for end-users, e.g. HTTP") default = models.BooleanField(default=True, help_text="Included by default when building mirror list?") created = models.DateTimeField(editable=False) diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index 1444eca9..bba54ec9 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -1,9 +1,11 @@ from django.conf.urls import patterns + urlpatterns = patterns('mirrors.views', (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), (r'^all/$', 'find_mirrors', {'countries': ['all']}), - (r'^all/(?P[A-z]+)/$', 'find_mirrors_simple') + (r'^all/(?P[A-z]+)/$', 'find_mirrors_simple', + {}, 'mirrorlist_simple') ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index d1dd9da3..22eb57b8 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -8,7 +8,7 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404, render +from django.shortcuts import get_object_or_404, redirect, render from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES @@ -78,18 +78,6 @@ def generate_mirrorlist(request): {'mirrorlist_form': form}) -def default_protocol_filter(original_urls): - key_func = attrgetter('country') - sorted_urls = sorted(original_urls, key=key_func) - urls = [] - for _, group in groupby(sorted_urls, key=key_func): - cntry_urls = list(group) - if any(url.protocol.default for url in cntry_urls): - cntry_urls = [url for url in cntry_urls if url.protocol.default] - urls.extend(cntry_urls) - return urls - - def status_filter(original_urls): status_info = get_mirror_statuses() scores = {u.id: u.score for u in status_info['urls']} @@ -105,7 +93,7 @@ def status_filter(original_urls): def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True, smart_protocol=False): + ipv4_supported=True, ipv6_supported=True): if not protocols: protocols = MirrorProtocol.objects.filter(is_download=True) elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: @@ -126,17 +114,12 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, ip_version |= Q(has_ipv6=True) qset = qset.filter(ip_version) - if smart_protocol: - urls = default_protocol_filter(qset) - else: - urls = qset - if not use_status: sort_key = attrgetter('country.name', 'mirror.name', 'url') - urls = sorted(urls, key=sort_key) + urls = sorted(qset, key=sort_key) template = 'mirrors/mirrorlist.txt' else: - urls = status_filter(urls) + urls = status_filter(qset) template = 'mirrors/mirrorlist_status.txt' context = { @@ -147,9 +130,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, def find_mirrors_simple(request, protocol): if protocol == 'smart': - # generate a 'smart' mirrorlist, one that only includes FTP mirrors if - # no HTTP mirror is available in that country. - return find_mirrors(request, smart_protocol=True) + return redirect('mirrorlist_simple', 'http', permanent=True) proto = get_object_or_404(MirrorProtocol, protocol=protocol) return find_mirrors(request, protocols=[proto]) diff --git a/templates/mirrors/mirrorlist_generate.html b/templates/mirrors/mirrorlist_generate.html index f072730e..59b0d358 100644 --- a/templates/mirrors/mirrorlist_generate.html +++ b/templates/mirrors/mirrorlist_generate.html @@ -21,10 +21,8 @@

Mirrorlist with all available mirrors

Customized by country mirrorlist

-- cgit v1.2.3-54-g00ecf From 8c614b4abb2ae0b0474744420aab6ba86383a3e9 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 May 2013 20:35:50 -0500 Subject: Add 'Edit Mirror' link to mirror details page For those that have permissions to do so. Makes it a heck of a lot easier to navigate around and enable or disable mirrors as necessary. Signed-off-by: Dan McGee --- templates/mirrors/mirror_details.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index 6f0ac6c1..ccd6eb57 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -2,6 +2,7 @@ {% load static from staticfiles %} {% load mirror_status %} {% load flags %} +{% load admin_urls %} {% block title %}Arch Linux - {{ mirror.name }} - Mirror Details{% endblock %} @@ -9,9 +10,14 @@ {% block content %}
-

Mirror Details: {{ mirror.name }}

+ {% if perms.mirrors.change_mirror %} + + {% endif %} + -- cgit v1.2.3-54-g00ecf From 80ae71464cadcf91f04cc194bb3930462643015f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 26 May 2013 18:09:09 -0500 Subject: Bump several requirements versions Signed-off-by: Dan McGee --- requirements.txt | 4 ++-- requirements_prod.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 84450c76..1909da1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,9 @@ Django==1.5.1 IPy==0.81 Markdown==2.3.1 -South==0.7.6 +South==0.8.1 bencode==1.0 django-countries==1.5 -jsmin==2.0.2-1 +jsmin==2.0.3 pgpdump==1.4 pytz>=2013b diff --git a/requirements_prod.txt b/requirements_prod.txt index cde5f513..1c331cf9 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -2,12 +2,12 @@ Django==1.5.1 IPy==0.81 Markdown==2.3.1 -South==0.7.6 +South==0.8.1 bencode==1.0 django-countries==1.5 -jsmin==2.0.2-1 +jsmin==2.0.3 pgpdump==1.4 psycopg2==2.5 pyinotify==0.9.4 -python-memcached==1.48 +python-memcached==1.51 pytz>=2013b -- cgit v1.2.3-54-g00ecf From bbeb9276051c2dc34f6d63fac4963343aacdf9b2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 31 May 2013 19:12:29 -0500 Subject: Add active column to mirror URLs --- .../0026_auto__add_field_mirrorurl_active.py | 83 ++++++++++++++++++++++ mirrors/models.py | 1 + 2 files changed, 84 insertions(+) create mode 100644 mirrors/migrations/0026_auto__add_field_mirrorurl_active.py diff --git a/mirrors/migrations/0026_auto__add_field_mirrorurl_active.py b/mirrors/migrations/0026_auto__add_field_mirrorurl_active.py new file mode 100644 index 00000000..f989435f --- /dev/null +++ b/mirrors/migrations/0026_auto__add_field_mirrorurl_active.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.add_column(u'mirrors_mirrorurl', 'active', + self.gf('django.db.models.fields.BooleanField')(default=True), + keep_default=True) + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorurl', 'active') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 91793e0a..975ead39 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -70,6 +70,7 @@ class MirrorUrl(models.Model): has_ipv6 = models.BooleanField("IPv6 capable", default=False, editable=False) created = models.DateTimeField(editable=False) + active = models.BooleanField(default=True) def address_families(self): hostname = urlparse(self.url).hostname -- cgit v1.2.3-54-g00ecf From e50471b9a7658f37b00aa3854d75a45007926802 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 31 May 2013 19:21:47 -0500 Subject: Honor mirror URL active attribute in several places Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 2 +- mirrors/management/commands/mirrorresolv.py | 2 +- mirrors/utils.py | 10 +++++++--- mirrors/views.py | 7 +++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index e7dd7b49..6faf294a 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -63,7 +63,7 @@ def handle_noargs(self, **options): timeout = options.get('timeout') urls = MirrorUrl.objects.select_related('protocol').filter( - mirror__active=True, mirror__public=True) + active=True, mirror__active=True, mirror__public=True) location = options.get('location', None) if location: diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index a6c2523e..85a3c654 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -39,7 +39,7 @@ def handle_noargs(self, **options): def resolve_mirrors(): logger.debug("requesting list of mirror URLs") - for mirrorurl in MirrorUrl.objects.filter(mirror__active=True): + for mirrorurl in MirrorUrl.objects.filter(active=True, mirror__active=True): try: # save old values, we can skip no-op updates this way oldvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) diff --git a/mirrors/utils.py b/mirrors/utils.py index ba45da5f..e98b5c9f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -116,7 +116,10 @@ def annotate_url(url, url_data): def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff - valid_urls = MirrorUrl.objects.filter( + # TODO: this prevents grabbing data points from any mirror that was active, + # receiving checks, and then marked private. we can probably be smarter and + # filter the data later? + valid_urls = MirrorUrl.objects.filter(active=True, mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).distinct() @@ -159,7 +162,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( - is_success=False, check_time__gte=cutoff_time, + is_success=False, check_time__gte=cutoff_time, url__active=True, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', 'url__mirror__tier', 'error').annotate( @@ -189,13 +192,14 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): min_sync_time = status_data['last_sync__max'] - timedelta(minutes=20) best_logs = MirrorLog.objects.filter(is_success=True, check_time__gte=min_check_time, last_sync__gte=min_sync_time, + url__active=True, url__mirror__public=True, url__mirror__active=True, url__protocol__default=True).order_by( 'duration')[:1] if best_logs: return MirrorUrl.objects.get(id=best_logs[0].url_id) - mirror_urls = MirrorUrl.objects.filter( + mirror_urls = MirrorUrl.objects.filter(active=True, mirror__public=True, mirror__active=True, protocol__default=True) # look first for a country-agnostic URL, then fall back to any HTTP URL filtered_urls = mirror_urls.filter(country='')[:1] diff --git a/mirrors/views.py b/mirrors/views.py index 22eb57b8..91cf6750 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -40,10 +40,13 @@ def __init__(self, *args, **kwargs): fields['protocol'].choices = protos fields['protocol'].initial = [p.protocol for p in initial] fields['ip_version'].initial = ['4'] + locations = [(l.id, l.source_ip) for l in + CheckLocation.objects.all()] + fields['check_location'].choices = locations def get_countries(self): country_codes = set() - country_codes.update(MirrorUrl.objects.filter( + country_codes.update(MirrorUrl.objects.filter(active=True, mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) countries = [(code, self.countries[code]) for code in country_codes] @@ -102,7 +105,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, else: protocols = MirrorProtocol.objects.filter(protocol__in=protocols) qset = MirrorUrl.objects.select_related().filter( - protocol__in=protocols, + protocol__in=protocols, active=True, mirror__public=True, mirror__active=True) if countries and 'all' not in countries: qset = qset.filter(country__in=countries) -- cgit v1.2.3-54-g00ecf From 31bc66e23c2ea0467d2201b86536f161fd1f32ca Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 31 May 2013 19:29:23 -0500 Subject: Whoops, committed a little bit too much Signed-off-by: Dan McGee --- mirrors/views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mirrors/views.py b/mirrors/views.py index 91cf6750..6f4ad838 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -40,9 +40,6 @@ def __init__(self, *args, **kwargs): fields['protocol'].choices = protos fields['protocol'].initial = [p.protocol for p in initial] fields['ip_version'].initial = ['4'] - locations = [(l.id, l.source_ip) for l in - CheckLocation.objects.all()] - fields['check_location'].choices = locations def get_countries(self): country_codes = set() -- cgit v1.2.3-54-g00ecf
Name: