diff options
| -rw-r--r-- | devel/management/commands/reporead.py | 78 | ||||
| -rw-r--r-- | mirrors/models.py | 9 | ||||
| -rw-r--r-- | mirrors/urls.py | 1 | ||||
| -rw-r--r-- | mirrors/utils.py | 5 | ||||
| -rw-r--r-- | mirrors/views.py | 21 | ||||
| -rw-r--r-- | public/utils.py | 3 | ||||
| -rw-r--r-- | public/views.py | 8 | ||||
| -rw-r--r-- | templates/admin/index.html | 12 | ||||
| -rw-r--r-- | templates/devel/admin_log.html | 2 | ||||
| -rw-r--r-- | templates/mirrors/mirror_details.html | 10 | ||||
| -rw-r--r-- | templates/mirrors/url_details.html | 89 | ||||
| -rw-r--r-- | templates/public/index.html | 2 |
12 files changed, 201 insertions, 39 deletions
diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 8b591aeb..1945469f 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -82,8 +82,7 @@ class RepoPackage(object): 'md5sum', 'sha256sum', 'url', 'packager' ) number = ( 'csize', 'isize' ) collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends', - 'conflicts', 'provides', 'replaces', 'groups', 'license', - 'files' ) + 'conflicts', 'provides', 'replaces', 'groups', 'license') def __init__(self, repo): self.repo = repo @@ -98,7 +97,6 @@ class RepoPackage(object): setattr(self, k, ()) self.builddate = None self.files = None - self.has_files = False def populate(self, values): for k, v in values.iteritems(): @@ -122,14 +120,22 @@ class RepoPackage(object): logger.warning( 'Package %s had unparsable build date %s', self.name, v[0]) - elif k == 'files': - self.files = tuple(v) - self.has_files = True else: # anything left in collections setattr(self, k, tuple(v)) @property + def files_list(self): + data_file = io.TextIOWrapper(io.BytesIO(self.files), encoding='UTF-8') + try: + info = parse_info(data_file) + except UnicodeDecodeError: + logger.warn("Could not correctly decode files list for %s", + self.name) + return None + return info['files'] + + @property def full_version(self): '''Very similar to the main.models.Package method.''' if self.epoch > 0: @@ -265,6 +271,24 @@ def delete_pkg_files(dbpkg): cursor.execute('DELETE FROM package_files WHERE pkg_id = %s', [dbpkg.id]) +def batched_bulk_create(model, all_objects): + cutoff = 10000 + length = len(all_objects) + if length < cutoff: + return model.objects.bulk_create(all_objects) + + def chunks(): + offset = 0 + while offset < length: + yield all_objects[offset:offset + cutoff] + offset += cutoff + + for items in chunks(): + ret = model.objects.bulk_create(items) + + return ret + + def populate_files(dbpkg, repopkg, force=False): if not force: if not pkg_same_version(repopkg, dbpkg): @@ -278,15 +302,18 @@ def populate_files(dbpkg, repopkg, force=False): return # only delete files if we are reading a DB that contains them - if repopkg.has_files: + if repopkg.files: + files = repopkg.files_list + # we had files data, but it couldn't be parsed, so skip + if not files: + return delete_pkg_files(dbpkg) logger.info("adding %d files for package %s", - len(repopkg.files), dbpkg.pkgname) + len(files), dbpkg.pkgname) pkg_files = [] # sort in normal alpha-order that pacman uses, rather than makepkg's # default breadth-first, directory-first ordering - files = sorted(repopkg.files) - for f in files: + for f in sorted(files): if '/' in f: dirname, filename = f.rsplit('/', 1) dirname += '/' @@ -299,7 +326,7 @@ def populate_files(dbpkg, repopkg, force=False): directory=dirname, filename=filename) pkg_files.append(pkgfile) - PackageFile.objects.bulk_create(pkg_files) + batched_bulk_create(PackageFile, pkg_files) dbpkg.files_last_update = now() dbpkg.save() @@ -494,24 +521,27 @@ def parse_repo(repopath): repodb = tarfile.open(repopath, "r") logger.debug("Starting package parsing") - dbfiles = ('desc', 'depends', 'files') newpkg = lambda: RepoPackage(reponame) pkgs = defaultdict(newpkg) for tarinfo in repodb.getmembers(): if tarinfo.isreg(): pkgid, fname = os.path.split(tarinfo.name) - if fname not in dbfiles: - continue - data_file = repodb.extractfile(tarinfo) - data_file = io.TextIOWrapper(io.BytesIO(data_file.read()), - encoding='UTF-8') - try: - pkgs[pkgid].populate(parse_info(data_file)) - except UnicodeDecodeError: - logger.warn("Could not correctly decode %s, skipping file", - tarinfo.name) - data_file.close() - del data_file + if fname == 'files': + # don't parse yet for speed and memory consumption reasons + files_data = repodb.extractfile(tarinfo) + pkgs[pkgid].files = files_data.read() + del files_data + elif fname in ('desc', 'depends'): + data_file = repodb.extractfile(tarinfo) + data_file = io.TextIOWrapper(io.BytesIO(data_file.read()), + encoding='UTF-8') + try: + pkgs[pkgid].populate(parse_info(data_file)) + except UnicodeDecodeError: + logger.warn("Could not correctly decode %s, skipping file", + tarinfo.name) + data_file.close() + del data_file logger.debug("Done parsing file %s/%s", pkgid, fname) diff --git a/mirrors/models.py b/mirrors/models.py index 47e2051b..57664562 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,3 +1,4 @@ +from datetime import timedelta import socket from urlparse import urlparse @@ -159,6 +160,14 @@ class MirrorLog(models.Model): is_success = models.BooleanField(default=True) error = models.TextField(blank=True, default='') + def delay(self): + if self.last_sync is None: + return None + # sanity check, this shouldn't happen + if self.check_time < self.last_sync: + return timedelta() + return self.check_time - self.last_sync + def __unicode__(self): return "Check of %s at %s" % (self.url.url, self.check_time) diff --git a/mirrors/urls.py b/mirrors/urls.py index 7cf76aa1..b1054380 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -9,6 +9,7 @@ urlpatterns = patterns('mirrors.views', (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'), (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'), + (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', 'url_details'), ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/utils.py b/mirrors/utils.py index 9f40bca6..0dd26ae0 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -22,7 +22,8 @@ def dictfetchall(cursor): ] @cache_function(178) -def status_data(cutoff_time, mirror_id=None): +def status_data(cutoff=DEFAULT_CUTOFF, mirror_id=None): + cutoff_time = now() - cutoff if mirror_id is not None: params = [cutoff_time, mirror_id] mirror_where = 'AND u.mirror_id = %s' @@ -125,7 +126,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): valid_urls = valid_urls.filter(active=True, mirror__active=True, mirror__public=True) - url_data = status_data(cutoff_time, mirror_id) + url_data = status_data(cutoff, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') diff --git a/mirrors/views.py b/mirrors/views.py index 9e05e5fc..34336165 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -186,6 +186,7 @@ def mirror_details(request, name): } return render(request, 'mirrors/mirror_details.html', context) + def mirror_details_json(request, name): authorized = request.user.is_authenticated() mirror = get_object_or_404(Mirror, name=name) @@ -199,6 +200,26 @@ def mirror_details_json(request, name): return response +def url_details(request, name, url_id): + url = get_object_or_404(MirrorUrl.objects.select_related(), + id=url_id, mirror__name=name) + mirror = url.mirror + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active or not url.active): + raise Http404 + error_cutoff = timedelta(days=7) + cutoff_time = now() - error_cutoff + logs = MirrorLog.objects.select_related('location').filter( + url=url, check_time__gte=cutoff_time).order_by('-check_time') + + context = { + 'url': url, + 'logs': logs, + } + return render(request, 'mirrors/url_details.html', context) + + def status(request, tier=None): if tier is not None: tier = int(tier) diff --git a/public/utils.py b/public/utils.py index fcfd0f77..11091883 100644 --- a/public/utils.py +++ b/public/utils.py @@ -2,7 +2,7 @@ from collections import defaultdict from operator import attrgetter from main.models import Arch, Repo, Package -from main.utils import cache_function, groupby_preserve_order, PackageStandin +from main.utils import groupby_preserve_order, PackageStandin class RecentUpdate(object): def __init__(self, packages): @@ -58,7 +58,6 @@ class RecentUpdate(object): return "RecentUpdate '%s %s' <%d packages>" % ( self.pkgbase, self.version, len(self.packages)) -@cache_function(62) def get_recent_updates(number=15, testing=True, staging=False): repos = Repo.objects.all() if not testing: diff --git a/public/views.py b/public/views.py index a5f818dc..d1661afd 100644 --- a/public/views.py +++ b/public/views.py @@ -20,12 +20,14 @@ from .utils import get_recent_updates @cache_control(max_age=300) def index(request): if request.user.is_authenticated(): - pkgs = get_recent_updates(testing=True, staging=True) + def updates(): + return get_recent_updates(testing=True, staging=True) else: - pkgs = get_recent_updates() + def updates(): + return get_recent_updates() context = { 'news_updates': News.objects.order_by('-postdate', '-id')[:15], - 'pkg_updates': pkgs, + 'pkg_updates': updates, } return render(request, 'public/index.html', context) diff --git a/templates/admin/index.html b/templates/admin/index.html index 203206d5..fddd55e5 100644 --- a/templates/admin/index.html +++ b/templates/admin/index.html @@ -31,11 +31,15 @@ {% if app_list %} {% for app in app_list %} - <div class="module"> - <table summary="{% blocktrans with name=app.name %}Models available in the {{ name }} application.{% endblocktrans %}"> - <caption><a href="{{ app.app_url }}" class="section">{% blocktrans with name=app.name %}{{ name }}{% endblocktrans %}</a></caption> + <div class="app-{{ app.app_label }} module"> + <table> + <caption> + <a href="{{ app.app_url }}" class="section" title="{% blocktrans with name=app.name %}Models in the {{ name }} application{% endblocktrans %}"> + {% blocktrans with name=app.name %}{{ name }}{% endblocktrans %} + </a> + </caption> {% for model in app.models %} - <tr> + <tr class="model-{{ model.object_name|lower }}"> {% if model.admin_url %} <th scope="row"><a href="{{ model.admin_url }}">{{ model.name }}</a></th> {% else %} diff --git a/templates/devel/admin_log.html b/templates/devel/admin_log.html index 1629c104..05130491 100644 --- a/templates/devel/admin_log.html +++ b/templates/devel/admin_log.html @@ -48,7 +48,7 @@ {% if entry.is_deletion %} {{ entry.object_repr }} {% else %} - <a href="/admin/{{ entry.get_admin_url }}">{{ entry.object_repr }}</a> + <a href="{{ entry.get_admin_url }}">{{ entry.object_repr }}</a> {% endif %} </td> <td>{{ entry.change_message }}</td> diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index 0913a506..b75c5316 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -42,6 +42,10 @@ <td>{{ mirror.active|yesno|capfirst }}</td> </tr> <tr> + <th>Created:</th> + <td>{{ mirror.created }}</td> + </tr> + <tr> <th>Rsync IPs:</th> <td class="wrap">{{mirror.rsync_ips.all|join:', '}}</td> </tr> @@ -59,7 +63,7 @@ </tr> <tr> <th>Notes:</th> - <td>{{ mirror.notes|linebreaks }}</td> + <td class="wrap">{{ mirror.notes|linebreaks }}</td> </tr> <tr> <th>Upstream:</th> @@ -99,7 +103,8 @@ <th>μ Delay (hh:mm)</th> <th>μ Duration (secs)</th> <th>σ Duration (secs)</th> - <th>Mirror Score</th> + <th>Score</th> + <th>Details</th> </tr> </thead> <tbody> @@ -116,6 +121,7 @@ <td>{{ m_url.duration_avg|floatvalue:2 }}</td> <td>{{ m_url.duration_stddev|floatvalue:2 }}</td> <td>{{ m_url.score|floatvalue:1|default:'∞' }}</td> + <td><a href="{{ m_url.id }}/">Details</a></td> </tr> {% endfor %} </tbody> diff --git a/templates/mirrors/url_details.html b/templates/mirrors/url_details.html new file mode 100644 index 00000000..0b9d2916 --- /dev/null +++ b/templates/mirrors/url_details.html @@ -0,0 +1,89 @@ +{% extends "base.html" %} +{% load cycle from future %} +{% load static from staticfiles %} +{% load mirror_status %} +{% load flags %} + +{% block title %}Arch Linux - {{ url.url }} - URL Details{% endblock %} + +{% block head %}<link rel="stylesheet" type="text/css" href="{% static "flags/fam.css" %}" media="screen, projection" />{% endblock %} + +{% block content %} +<div class="box"> + <h2>URL Details: {{ url.url }}</h2> + + <table class="compact"> + <tr> + <th>URL:</th> + <td>{{ url.url }}</td> + </tr> + <tr> + <th>Protocol:</th> + <td>{{ url.protocol }}</td> + </tr> + <tr> + <th>Country:</th> + <td class="country">{% country_flag url.country %}{{ url.country.name }}</td> + </tr> + <tr> + <th>IPv4:</th> + <td>{{ url.has_ipv4|yesno|capfirst }}</td> + </tr> + <tr> + <th>IPv6:</th> + <td>{{ url.has_ipv6|yesno|capfirst }}</td> + </tr> + {% if user.is_authenticated %} + <tr> + <th>Active:</th> + <td>{{ url.active|yesno|capfirst }}</td> + </tr> + <tr> + <th>Created:</th> + <td>{{ url.created }}</td> + </tr> + {% endif %} + </table> + + <h3>Check Logs</h3> + + <table id="check_logs" class="results"> + <thead> + <tr> + <th>Check Time</th> + <th>Check Location</th> + <th>Check IP</th> + <th>Last Sync</th> + <th>Delay (hh:mm)</th> + <th>Duration (secs)</th> + <th>Success?</th> + <th>Error Message</th> + </tr> + </thead> + <tbody> + {% for log in logs %}<tr class="{% cycle 'odd' 'even' %}"> + <td>{{ log.check_time|date:'Y-m-d H:i' }}</td> + <td class="country">{% country_flag log.location.country %}{{ log.location.country.name }}</td> + <td>{{ log.location.source_ip }}</td> + <td>{{ log.last_sync|date:'Y-m-d H:i' }}</td> + <td>{{ log.delay|duration }}</td> + <td>{{ log.duration|floatvalue }}</td> + <td>{{ log.is_success|yesno|capfirst }}</td> + <td class="wrap">{{ log.error|linebreaksbr }}</td> + </tr>{% endfor %} + </tbody> + </table> +</div> +{% endblock %} + +{% block script_block %} +{% load cdn %}{% jquery %}{% jquery_tablesorter %} +<script type="text/javascript" src="{% static "archweb.js" %}"></script> +<script type="text/javascript"> +$(document).ready(function() { + $("#check_logs:has(tbody tr)").tablesorter( + {widgets: ['zebra'], sortList: [[0,1]], + headers: { 5: { sorter: 'mostlydigit' } } }); +}); +</script> +{% endblock %} diff --git a/templates/public/index.html b/templates/public/index.html index b263a308..bc0fae0c 100644 --- a/templates/public/index.html +++ b/templates/public/index.html @@ -106,7 +106,7 @@ </div> {% endcache %} -{% cache 59 main-page-right secure %} +{% cache 115 main-page-right secure %} <div id="nav-sidebar" class="widget"> <h4>Documentation</h4> |