diff options
Diffstat (limited to 'mirrors')
| -rw-r--r-- | mirrors/admin.py | 1 | ||||
| -rw-r--r-- | mirrors/fixtures/mirrorprotocols.json | 9 | ||||
| -rw-r--r-- | mirrors/management/commands/mirrorcheck.py | 2 | ||||
| -rw-r--r-- | mirrors/management/commands/mirrorresolv.py | 2 | ||||
| -rw-r--r-- | mirrors/migrations/0026_auto__add_field_mirrorurl_active.py | 83 | ||||
| -rw-r--r-- | mirrors/models.py | 3 | ||||
| -rw-r--r-- | mirrors/urls_mirrorlist.py | 4 | ||||
| -rw-r--r-- | mirrors/utils.py | 10 | ||||
| -rw-r--r-- | mirrors/views.py | 44 |
9 files changed, 114 insertions, 44 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py index 9c88207d..d0f2f475 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -10,6 +10,7 @@ from .models import (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, class MirrorUrlForm(forms.ModelForm): class Meta: model = MirrorUrl + def clean_url(self): # is this a valid-looking URL? url_parts = urlparse(self.cleaned_data["url"]) diff --git a/mirrors/fixtures/mirrorprotocols.json b/mirrors/fixtures/mirrorprotocols.json index 8822ef8e..1a07510b 100644 --- a/mirrors/fixtures/mirrorprotocols.json +++ b/mirrors/fixtures/mirrorprotocols.json @@ -9,15 +9,6 @@ } }, { - "pk": 2, - "model": "mirrors.mirrorprotocol", - "fields": { - "is_download": true, - "default": false, - "protocol": "ftp" - } - }, - { "pk": 3, "model": "mirrors.mirrorprotocol", "fields": { diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index e7dd7b49..6faf294a 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -63,7 +63,7 @@ class Command(NoArgsCommand): timeout = options.get('timeout') urls = MirrorUrl.objects.select_related('protocol').filter( - mirror__active=True, mirror__public=True) + active=True, mirror__active=True, mirror__public=True) location = options.get('location', None) if location: diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index a6c2523e..85a3c654 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -39,7 +39,7 @@ class Command(NoArgsCommand): def resolve_mirrors(): logger.debug("requesting list of mirror URLs") - for mirrorurl in MirrorUrl.objects.filter(mirror__active=True): + for mirrorurl in MirrorUrl.objects.filter(active=True, mirror__active=True): try: # save old values, we can skip no-op updates this way oldvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) diff --git a/mirrors/migrations/0026_auto__add_field_mirrorurl_active.py b/mirrors/migrations/0026_auto__add_field_mirrorurl_active.py new file mode 100644 index 00000000..f989435f --- /dev/null +++ b/mirrors/migrations/0026_auto__add_field_mirrorurl_active.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.add_column(u'mirrors_mirrorurl', 'active', + self.gf('django.db.models.fields.BooleanField')(default=True), + keep_default=True) + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorurl', 'active') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index d8ac7952..975ead39 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -47,7 +47,7 @@ class Mirror(models.Model): class MirrorProtocol(models.Model): protocol = models.CharField(max_length=10, unique=True) is_download = models.BooleanField(default=True, - help_text="Is protocol useful for end-users, e.g. FTP/HTTP") + help_text="Is protocol useful for end-users, e.g. HTTP") default = models.BooleanField(default=True, help_text="Included by default when building mirror list?") created = models.DateTimeField(editable=False) @@ -70,6 +70,7 @@ class MirrorUrl(models.Model): has_ipv6 = models.BooleanField("IPv6 capable", default=False, editable=False) created = models.DateTimeField(editable=False) + active = models.BooleanField(default=True) def address_families(self): hostname = urlparse(self.url).hostname diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index 1444eca9..bba54ec9 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -1,9 +1,11 @@ from django.conf.urls import patterns + urlpatterns = patterns('mirrors.views', (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), (r'^all/$', 'find_mirrors', {'countries': ['all']}), - (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple') + (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple', + {}, 'mirrorlist_simple') ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/utils.py b/mirrors/utils.py index ba45da5f..e98b5c9f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -116,7 +116,10 @@ def annotate_url(url, url_data): def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff - valid_urls = MirrorUrl.objects.filter( + # TODO: this prevents grabbing data points from any mirror that was active, + # receiving checks, and then marked private. we can probably be smarter and + # filter the data later? + valid_urls = MirrorUrl.objects.filter(active=True, mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).distinct() @@ -159,7 +162,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( - is_success=False, check_time__gte=cutoff_time, + is_success=False, check_time__gte=cutoff_time, url__active=True, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', 'url__mirror__tier', 'error').annotate( @@ -189,13 +192,14 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): min_sync_time = status_data['last_sync__max'] - timedelta(minutes=20) best_logs = MirrorLog.objects.filter(is_success=True, check_time__gte=min_check_time, last_sync__gte=min_sync_time, + url__active=True, url__mirror__public=True, url__mirror__active=True, url__protocol__default=True).order_by( 'duration')[:1] if best_logs: return MirrorUrl.objects.get(id=best_logs[0].url_id) - mirror_urls = MirrorUrl.objects.filter( + mirror_urls = MirrorUrl.objects.filter(active=True, mirror__public=True, mirror__active=True, protocol__default=True) # look first for a country-agnostic URL, then fall back to any HTTP URL filtered_urls = mirror_urls.filter(country='')[:1] diff --git a/mirrors/views.py b/mirrors/views.py index 73d40297..6f4ad838 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -8,7 +8,7 @@ from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404, render +from django.shortcuts import get_object_or_404, redirect, render from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES @@ -43,7 +43,7 @@ class MirrorlistForm(forms.Form): def get_countries(self): country_codes = set() - country_codes.update(MirrorUrl.objects.filter( + country_codes.update(MirrorUrl.objects.filter(active=True, mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) countries = [(code, self.countries[code]) for code in country_codes] @@ -78,18 +78,6 @@ def generate_mirrorlist(request): {'mirrorlist_form': form}) -def default_protocol_filter(original_urls): - key_func = attrgetter('country') - sorted_urls = sorted(original_urls, key=key_func) - urls = [] - for _, group in groupby(sorted_urls, key=key_func): - cntry_urls = list(group) - if any(url.protocol.default for url in cntry_urls): - cntry_urls = [url for url in cntry_urls if url.protocol.default] - urls.extend(cntry_urls) - return urls - - def status_filter(original_urls): status_info = get_mirror_statuses() scores = {u.id: u.score for u in status_info['urls']} @@ -105,7 +93,7 @@ def status_filter(original_urls): def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True, smart_protocol=False): + ipv4_supported=True, ipv6_supported=True): if not protocols: protocols = MirrorProtocol.objects.filter(is_download=True) elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: @@ -114,7 +102,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, else: protocols = MirrorProtocol.objects.filter(protocol__in=protocols) qset = MirrorUrl.objects.select_related().filter( - protocol__in=protocols, + protocol__in=protocols, active=True, mirror__public=True, mirror__active=True) if countries and 'all' not in countries: qset = qset.filter(country__in=countries) @@ -126,17 +114,12 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, ip_version |= Q(has_ipv6=True) qset = qset.filter(ip_version) - if smart_protocol: - urls = default_protocol_filter(qset) - else: - urls = qset - if not use_status: sort_key = attrgetter('country.name', 'mirror.name', 'url') - urls = sorted(urls, key=sort_key) + urls = sorted(qset, key=sort_key) template = 'mirrors/mirrorlist.txt' else: - urls = status_filter(urls) + urls = status_filter(qset) template = 'mirrors/mirrorlist_status.txt' context = { @@ -147,9 +130,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, def find_mirrors_simple(request, protocol): if protocol == 'smart': - # generate a 'smart' mirrorlist, one that only includes FTP mirrors if - # no HTTP mirror is available in that country. - return find_mirrors(request, smart_protocol=True) + return redirect('mirrorlist_simple', 'http', permanent=True) proto = get_object_or_404(MirrorProtocol, protocol=protocol) return find_mirrors(request, protocols=[proto]) @@ -175,6 +156,7 @@ def mirror_details(request, name): if not request.user.is_authenticated() and \ (not mirror.public or not mirror.active): raise Http404 + error_cutoff = timedelta(days=7) status_info = get_mirror_statuses(mirror_id=mirror.id) checked_urls = {url for url in status_info['urls'] \ @@ -188,9 +170,15 @@ def mirror_details(request, name): setattr(url, attr, None) all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) - return render(request, 'mirrors/mirror_details.html', - {'mirror': mirror, 'urls': all_urls}) + error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff) + context = { + 'mirror': mirror, + 'urls': all_urls, + 'cutoff': error_cutoff, + 'error_logs': error_logs, + } + return render(request, 'mirrors/mirror_details.html', context) def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) |