From bf2cb9784fe726b163667aaeab541cf7ae7f21a9 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 10:58:22 -0400 Subject: Give things more consistent names. --- db-check-nonfree | 28 ++++++ db-check-package-libraries | 193 ++++++++++++++++++++++++++++++++++++++++++ db-check-package-libraries.py | 193 ------------------------------------------ db-check-unsigned-packages | 38 +++++++++ db-check-unsigned-packages.py | 96 +++++++++++++++++++++ db-import-pick-mirror | 25 ++++++ db-import.conf | 2 +- db-list-nonfree.py | 28 ------ db-list-unsigned-packages | 38 --------- db-list-unsigned-packages.py | 96 --------------------- db-pick-mirror | 25 ------ 11 files changed, 381 insertions(+), 381 deletions(-) create mode 100755 db-check-nonfree create mode 100755 db-check-package-libraries delete mode 100755 db-check-package-libraries.py create mode 100755 db-check-unsigned-packages create mode 100755 db-check-unsigned-packages.py create mode 100755 db-import-pick-mirror delete mode 100755 db-list-nonfree.py delete mode 100755 db-list-unsigned-packages delete mode 100755 db-list-unsigned-packages.py delete mode 100755 db-pick-mirror diff --git a/db-check-nonfree b/db-check-nonfree new file mode 100755 index 0000000..14a108f --- /dev/null +++ b/db-check-nonfree @@ -0,0 +1,28 @@ +#!/usr/bin/env python2 +#-*- encoding: utf-8 -*- +from filter import * +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="db-check-nonfree", + description="Cleans nonfree files on repo",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean", + required=True,) + + args=parser.parse_args() + + if not (args.blacklist_file and args.database): + parser.print_help() + exit(1) + + blacklist=listado(args.blacklist_file) + pkgs=get_pkginfo_from_db(args.database) + + print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) diff --git a/db-check-package-libraries b/db-check-package-libraries new file mode 100755 index 0000000..612fc4f --- /dev/null +++ b/db-check-package-libraries @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Check which libraries are provided or required by a package, store +this in a database, update and list broken packages. + +Dependencies: + +- Python 3.2 or later with SQLite 3 support + +- ``bsdtar`` + +- ``readelf`` +""" + + +import os.path +import re +import sqlite3 +import subprocess +import tempfile + + +#: Regexp matching an interesting dynamic entry. +_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" + "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") + + +def make_db(path): + """Make a new, empty, library database at *path*.""" + con = sqlite3.connect(path) + con.executescript(""" +create table provided( + library varchar not null, + package varchar not null +); +create table used( + library varchar not null, + package varchar not null +); +""") + con.close() + + +def begin(database): + """Connect to *database* and start a transaction.""" + con = sqlite3.connect(database) + con.execute("begin exclusive") + return con + + +def add_provided(con, package, libraries): + """Write that *package* provides *libraries*.""" + for library in libraries: + con.execute("insert into provided (package, library) values (?,?)", + (package, library)) + + +def add_used(con, package, libraries): + """Write that *package* uses *libraries*.""" + for library in libraries: + con.execute("insert into used (package, library) values (?,?)", + (package, library)) + + +def remove_package(con, package): + """Remove all entries for a package.""" + con.execute("delete from provided where package=?", (package,)) + con.execute("delete from used where package=?", (package,)) + + +def add_package(con, package): + """Add entries from a named *package*.""" + # Extract to a temporary directory. This could be done more + # efficiently, since there is no need to store more than one file + # at once. + with tempfile.TemporaryDirectory() as temp: + tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) + tar.communicate() + with open(os.path.join(temp, ".PKGINFO")) as pkginfo: + for line in pkginfo: + if line.startswith("pkgname ="): + pkgname = line[len("pkgname ="):].strip() + break + # Don't list previously removed libraries. + remove_package(con, pkgname) + provided = set() + used = set() + # Search for ELFs. + for dirname, dirnames, filenames in os.walk(temp): + assert dirnames is not None # unused, avoid pylint warning + for file_name in filenames: + path = os.path.join(dirname, file_name) + with open(path, "rb") as file_object: + if file_object.read(4) != b"\177ELF": + continue + readelf = subprocess.Popen(("readelf", "-d", path), + stdout=subprocess.PIPE) + for line in readelf.communicate()[0].split(b"\n"): + match = _DYNAMIC.match(line.decode("ascii")) + if match: + if match.group(1) == "SONAME": + provided.add(match.group(2)) + elif match.group(1) == "NEEDED": + used.add(match.group(2)) + else: + raise AssertionError("unknown entry type " + + match.group(1)) + add_provided(con, pkgname, provided) + add_used(con, pkgname, used) + + +def init(arguments): + """Initialize.""" + make_db(arguments.database) + + +def add(arguments): + """Add packages.""" + con = begin(arguments.database) + for package in arguments.packages: + add_package(con, package) + con.commit() + con.close() + + +def remove(arguments): + """Remove packages.""" + con = begin(arguments.database) + for package in arguments.packages: + remove_package(con, package) + con.commit() + con.close() + + +def check(arguments): + """List broken packages.""" + con = begin(arguments.database) + available = set(row[0] for row + in con.execute("select library from provided")) + for package, library in con.execute("select package, library from used"): + if library not in available: + print(package, "needs", library) + con.close() + + +def main(): + """Get arguments and run the command.""" + from argparse import ArgumentParser + parser = ArgumentParser(prog="db-check-package-libraries", + description="Check packages for " + "provided/needed libraries") + parser.add_argument("-d", "--database", type=str, + help="Database file to use", + default="package-libraries.sqlite") + subparsers = parser.add_subparsers() + subparser = subparsers.add_parser(name="init", + help="initialize the database") + subparser.set_defaults(command=init) + subparser = subparsers.add_parser(name="add", + help="add packages to database") + subparser.add_argument("packages", nargs="+", type=str, + help="package files to add") + subparser.set_defaults(command=add) + subparser = subparsers.add_parser(name="remove", + help="remove packages from database") + subparser.add_argument("packages", nargs="+", type=str, + help="package names to remove") + subparser.set_defaults(command=remove) + subparser = subparsers.add_parser(name="check", + help="list broken packages") + subparser.set_defaults(command=check) + arguments = parser.parse_args() + arguments.command(arguments) + + +if __name__ == "__main__": + main() diff --git a/db-check-package-libraries.py b/db-check-package-libraries.py deleted file mode 100755 index bc2349b..0000000 --- a/db-check-package-libraries.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Check which libraries are provided or required by a package, store -this in a database, update and list broken packages. - -Dependencies: - -- Python 3.2 or later with SQLite 3 support - -- ``bsdtar`` - -- ``readelf`` -""" - - -import os.path -import re -import sqlite3 -import subprocess -import tempfile - - -#: Regexp matching an interesting dynamic entry. -_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" - "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") - - -def make_db(path): - """Make a new, empty, library database at *path*.""" - con = sqlite3.connect(path) - con.executescript(""" -create table provided( - library varchar not null, - package varchar not null -); -create table used( - library varchar not null, - package varchar not null -); -""") - con.close() - - -def begin(database): - """Connect to *database* and start a transaction.""" - con = sqlite3.connect(database) - con.execute("begin exclusive") - return con - - -def add_provided(con, package, libraries): - """Write that *package* provides *libraries*.""" - for library in libraries: - con.execute("insert into provided (package, library) values (?,?)", - (package, library)) - - -def add_used(con, package, libraries): - """Write that *package* uses *libraries*.""" - for library in libraries: - con.execute("insert into used (package, library) values (?,?)", - (package, library)) - - -def remove_package(con, package): - """Remove all entries for a package.""" - con.execute("delete from provided where package=?", (package,)) - con.execute("delete from used where package=?", (package,)) - - -def add_package(con, package): - """Add entries from a named *package*.""" - # Extract to a temporary directory. This could be done more - # efficiently, since there is no need to store more than one file - # at once. - with tempfile.TemporaryDirectory() as temp: - tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) - tar.communicate() - with open(os.path.join(temp, ".PKGINFO")) as pkginfo: - for line in pkginfo: - if line.startswith("pkgname ="): - pkgname = line[len("pkgname ="):].strip() - break - # Don't list previously removed libraries. - remove_package(con, pkgname) - provided = set() - used = set() - # Search for ELFs. - for dirname, dirnames, filenames in os.walk(temp): - assert dirnames is not None # unused, avoid pylint warning - for file_name in filenames: - path = os.path.join(dirname, file_name) - with open(path, "rb") as file_object: - if file_object.read(4) != b"\177ELF": - continue - readelf = subprocess.Popen(("readelf", "-d", path), - stdout=subprocess.PIPE) - for line in readelf.communicate()[0].split(b"\n"): - match = _DYNAMIC.match(line.decode("ascii")) - if match: - if match.group(1) == "SONAME": - provided.add(match.group(2)) - elif match.group(1) == "NEEDED": - used.add(match.group(2)) - else: - raise AssertionError("unknown entry type " - + match.group(1)) - add_provided(con, pkgname, provided) - add_used(con, pkgname, used) - - -def init(arguments): - """Initialize.""" - make_db(arguments.database) - - -def add(arguments): - """Add packages.""" - con = begin(arguments.database) - for package in arguments.packages: - add_package(con, package) - con.commit() - con.close() - - -def remove(arguments): - """Remove packages.""" - con = begin(arguments.database) - for package in arguments.packages: - remove_package(con, package) - con.commit() - con.close() - - -def check(arguments): - """List broken packages.""" - con = begin(arguments.database) - available = set(row[0] for row - in con.execute("select library from provided")) - for package, library in con.execute("select package, library from used"): - if library not in available: - print(package, "needs", library) - con.close() - - -def main(): - """Get arguments and run the command.""" - from argparse import ArgumentParser - parser = ArgumentParser(prog="check-package-libraries.py", - description="Check packages for " - "provided/needed libraries") - parser.add_argument("-d", "--database", type=str, - help="Database file to use", - default="package-libraries.sqlite") - subparsers = parser.add_subparsers() - subparser = subparsers.add_parser(name="init", - help="initialize the database") - subparser.set_defaults(command=init) - subparser = subparsers.add_parser(name="add", - help="add packages to database") - subparser.add_argument("packages", nargs="+", type=str, - help="package files to add") - subparser.set_defaults(command=add) - subparser = subparsers.add_parser(name="remove", - help="remove packages from database") - subparser.add_argument("packages", nargs="+", type=str, - help="package names to remove") - subparser.set_defaults(command=remove) - subparser = subparsers.add_parser(name="check", - help="list broken packages") - subparser.set_defaults(command=check) - arguments = parser.parse_args() - arguments.command(arguments) - - -if __name__ == "__main__": - main() diff --git a/db-check-unsigned-packages b/db-check-unsigned-packages new file mode 100755 index 0000000..0fc053b --- /dev/null +++ b/db-check-unsigned-packages @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -e + +# Output a list of repo/package-name-and-version pairs representing +# unsigned packages available for architecture $1 and specified for +# architecture $2 (usually $1 or any, default is to list all). + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -lt 1 ]; then + msg "usage: %s " "${0##*/}" + exit 1 +fi + +arch=$1 +shift + +for repo in "${PKGREPOS[@]}" +do + db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-check-unsigned-packages.py" "$repo" "$@" < "$db" +done diff --git a/db-check-unsigned-packages.py b/db-check-unsigned-packages.py new file mode 100755 index 0000000..80cff51 --- /dev/null +++ b/db-check-unsigned-packages.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Output a list of repo/package-name-and-version pairs representing +unsigned packages in the database at standard input of repo named in +the first argument and specified for architectures listed in the +following arguments (usually the one of the database or any, default +is to list all). + +If the --keyset argument is passed, print the key fingerprint of every +signed package. +""" + + +import base64 +import subprocess +import sys +import tarfile + + +def main(): + """Do the job.""" + check_keys = False + if "--keyset" in sys.argv: + sys.argv.remove("--keyset") + check_keys = True + repo = sys.argv[1] + pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) + packages = [] + keys = [] + with tarfile.open(fileobj=sys.stdin.buffer) as archive: + for entry in archive: + if entry.name.endswith("/desc"): + content = archive.extractfile(entry) + skip = False + is_arch = False + key = None + for line in content: + if is_arch: + is_arch = False + if pkgarches and line.strip() not in pkgarches: + skip = True # different architecture + break + if line == b"%PGPSIG%\n": + skip = True # signed + key = b"" + if check_keys: + continue + else: + break + if line == b"%ARCH%\n": + is_arch = True + continue + if key is not None: + if line.strip(): + key += line.strip() + else: + break + if check_keys and key: + key_binary = base64.b64decode(key) + keys.append(key_binary) + packages.append(repo + "/" + entry.name[:-5]) + if skip: + continue + print(repo + "/" + entry.name[:-5]) + if check_keys and keys: + # We have collected all signed package names in packages and + # all keys in keys. Let's now ask gpg to list all signatures + # and find which keys made them. + packets = subprocess.check_output(("gpg", "--list-packets"), + input=b"".join(keys)) + i = 0 + for line in packets.decode("latin1").split("\n"): + if line.startswith(":signature packet:"): + keyid = line[line.index("keyid ") + len("keyid "):] + print(packages[i], keyid) + i += 1 + + +if __name__ == "__main__": + main() diff --git a/db-import-pick-mirror b/db-import-pick-mirror new file mode 100755 index 0000000..4d01b95 --- /dev/null +++ b/db-import-pick-mirror @@ -0,0 +1,25 @@ +#!/usr/bin/env ruby + +require 'json' +require 'net/http' + +protocol = ARGV[0] +jsonurl = ARGV[1] + +data = JSON::parse(Net::HTTP.get(URI(jsonurl))) + +if data["version"] != 3 + print "Data format version != 3" + exit 1 +end + +# Filter out URLs with incomplete information +urls = data["urls"].select{|a| a.none?{|k,v|v.nil?}} +rsync_urls = urls.select{|a| a["protocol"]==protocol} + +# By score ( (delay+speed)/completion ) +#best = rsync_urls.sort{|a,b| (a["score"] || Float::INFINITY) <=> (b["score"] || Float::INFINITY) }.first +# By delay/completion +best = rsync_urls.sort{|a,b| a["delay"]/a["completion_pct"] <=> b["delay"]/b["completion_pct"] }.first + +puts best["url"] diff --git a/db-import.conf b/db-import.conf index aaf7b0f..e00e7b0 100644 --- a/db-import.conf +++ b/db-import.conf @@ -15,7 +15,7 @@ case "$USER" in );; esac -_archpkgmirror=$(db-pick-mirror rsync https://www.archlinux.org/mirrors/status/tier/1/json/) +_archpkgmirror=$(db-import-pick-mirror rsync https://www.archlinux.org/mirrors/status/tier/1/json/) # name pkgmirror absmirror repo-arch... IMPORTS=("archlinux ${_archpkgmirror} rsync.archlinux.org ${_archrepos[*]}") diff --git a/db-list-nonfree.py b/db-list-nonfree.py deleted file mode 100755 index a486fa5..0000000 --- a/db-list-nonfree.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python2 -#-*- encoding: utf-8 -*- -from filter import * -import argparse - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="nonfree_in_db", - description="Cleans nonfree files on repo",) - - parser.add_argument("-k", "--blacklist-file", type=str, - help="File containing blacklisted names", - required=True,) - - parser.add_argument("-b", "--database", type=str, - help="dabatase to clean", - required=True,) - - args=parser.parse_args() - - if not (args.blacklist_file and args.database): - parser.print_help() - exit(1) - - blacklist=listado(args.blacklist_file) - pkgs=get_pkginfo_from_db(args.database) - - print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages deleted file mode 100755 index 095e1e6..0000000 --- a/db-list-unsigned-packages +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -set -e - -# Output a list of repo/package-name-and-version pairs representing -# unsigned packages available for architecture $1 and specified for -# architecture $2 (usually $1 or any, default is to list all). - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -lt 1 ]; then - msg "usage: %s " "${0##*/}" - exit 1 -fi - -arch=$1 -shift - -for repo in "${PKGREPOS[@]}" -do - db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" -done diff --git a/db-list-unsigned-packages.py b/db-list-unsigned-packages.py deleted file mode 100755 index 80cff51..0000000 --- a/db-list-unsigned-packages.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Output a list of repo/package-name-and-version pairs representing -unsigned packages in the database at standard input of repo named in -the first argument and specified for architectures listed in the -following arguments (usually the one of the database or any, default -is to list all). - -If the --keyset argument is passed, print the key fingerprint of every -signed package. -""" - - -import base64 -import subprocess -import sys -import tarfile - - -def main(): - """Do the job.""" - check_keys = False - if "--keyset" in sys.argv: - sys.argv.remove("--keyset") - check_keys = True - repo = sys.argv[1] - pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) - packages = [] - keys = [] - with tarfile.open(fileobj=sys.stdin.buffer) as archive: - for entry in archive: - if entry.name.endswith("/desc"): - content = archive.extractfile(entry) - skip = False - is_arch = False - key = None - for line in content: - if is_arch: - is_arch = False - if pkgarches and line.strip() not in pkgarches: - skip = True # different architecture - break - if line == b"%PGPSIG%\n": - skip = True # signed - key = b"" - if check_keys: - continue - else: - break - if line == b"%ARCH%\n": - is_arch = True - continue - if key is not None: - if line.strip(): - key += line.strip() - else: - break - if check_keys and key: - key_binary = base64.b64decode(key) - keys.append(key_binary) - packages.append(repo + "/" + entry.name[:-5]) - if skip: - continue - print(repo + "/" + entry.name[:-5]) - if check_keys and keys: - # We have collected all signed package names in packages and - # all keys in keys. Let's now ask gpg to list all signatures - # and find which keys made them. - packets = subprocess.check_output(("gpg", "--list-packets"), - input=b"".join(keys)) - i = 0 - for line in packets.decode("latin1").split("\n"): - if line.startswith(":signature packet:"): - keyid = line[line.index("keyid ") + len("keyid "):] - print(packages[i], keyid) - i += 1 - - -if __name__ == "__main__": - main() diff --git a/db-pick-mirror b/db-pick-mirror deleted file mode 100755 index 4d01b95..0000000 --- a/db-pick-mirror +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env ruby - -require 'json' -require 'net/http' - -protocol = ARGV[0] -jsonurl = ARGV[1] - -data = JSON::parse(Net::HTTP.get(URI(jsonurl))) - -if data["version"] != 3 - print "Data format version != 3" - exit 1 -end - -# Filter out URLs with incomplete information -urls = data["urls"].select{|a| a.none?{|k,v|v.nil?}} -rsync_urls = urls.select{|a| a["protocol"]==protocol} - -# By score ( (delay+speed)/completion ) -#best = rsync_urls.sort{|a,b| (a["score"] || Float::INFINITY) <=> (b["score"] || Float::INFINITY) }.first -# By delay/completion -best = rsync_urls.sort{|a,b| a["delay"]/a["completion_pct"] <=> b["delay"]/b["completion_pct"] }.first - -puts best["url"] -- cgit v1.2.3