diff options
| -rw-r--r-- | README | 7 | ||||
| -rw-r--r-- | config | 47 | ||||
| -rw-r--r-- | config.local.import-community | 5 | ||||
| -rw-r--r-- | config.local.import-packages | 5 | ||||
| -rw-r--r-- | config.local.parabola | 14 | ||||
| -rw-r--r-- | config.local.svn-community | 13 | ||||
| -rw-r--r-- | config.local.svn-packages | 13 | ||||
| -rwxr-xr-x | db-import | 288 | ||||
| -rwxr-xr-x | db-import-pick-mirror | 25 | ||||
| -rw-r--r-- | db-import.conf | 23 | ||||
| -rwxr-xr-x | db-update | 10 |
11 files changed, 429 insertions, 21 deletions
@@ -16,10 +16,12 @@ The executables that you (might) care about are: ├── db-check-repo-sanity [Parabola only] ├── db-check-unsigned-packages [Parabola only] ├── db-check-unsigned-packages.py [Parabola only] + ├── db-import [Parabola only] ├── db-import-archlinux-any-to-ours [Parabola only] ├── db-import-archlinux-pkg [Parabola only] ├── db-import-archlinux-src [Parabola only] ├── db-import-archlinuxarm-pkg [Parabola only] + ├── db-import-pick-mirror [Parabola only] ├── db-init [Parabola only] ├── db-move ├── db-remove @@ -72,7 +74,10 @@ packages from elsewhere: - db-import-${source}* Unfortunately, these scripts are horribly duct-taped together and are -aweful. I hate them. +aweful. I hate them. Hopefully, one day, they can be entirely +replaced by + + - db-import Things that haven't been mentioned yet: @@ -3,38 +3,53 @@ # Instead, create separate ${toolname}.conf files. Only add a # variable here if multiple tools start needing the option. -FTP_BASE="/srv/repo/main" - -PKGREPOS=( - 'core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing' - 'libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing' - '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' - 'nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') -PKGPOOL='pool/parabola_gnu+linux' -SRCPOOL='src/parabola_gnu+linux' +case "$USER" in + db-import-packages) _name=import-packages;; + db-import-community) _name=import-community;; + *) _name=parabola;; +esac -CLEANUP_DESTDIR="$FTP_BASE/old/pkg" +FTP_BASE="/srv/repo/main" +PKGREPOS=() +# PKGREPOS=( +# 'core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing' +# 'libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing' +# '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' +# 'nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') +PKGPOOL='' +SRCPOOL='' + +CLEANUP_DESTDIR="/srv/repo/private/${_name}/package-cleanup" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/src" -SOURCE_CLEANUP_DRYRUN=true +SOURCE_CLEANUP_DESTDIR="/srv/repo/private/${_name}/source-cleanup" +SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=30 +SOURCE_CLEANUP_KEEP=14 REQUIRE_SIGNATURE=true LOCK_DELAY=10 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" -TMPDIR="/tmp" +export TMPDIR="${TMPDIR:-/tmp}" ARCHES=(i686 x86_64 armv7h) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" +# Allowed licenses: get sourceballs only for licenses in this array +# Empty (commented out) to get sourceballs for all packages +#ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') + # Where to send error emails, and who they are from -LIST="maintenance@lists.parabola.nu" -FROM="repo@parabola.nu" +LIST="dev@lists.parabola.nu" +FROM="dbscripts+${_name}@$(hostname -f)" + +# Override default config with config.local +[ -f "$(dirname "${BASH_SOURCE[0]}")/config.local" ] && . "$(dirname "${BASH_SOURCE[0]}")/config.local" +[ -f "$(dirname "${BASH_SOURCE[0]}")/config.local.${_name}" ] && . "$(dirname "${BASH_SOURCE[0]}")/config.local.${_name}" +unset _name diff --git a/config.local.import-community b/config.local.import-community new file mode 100644 index 0000000..393fd57 --- /dev/null +++ b/config.local.import-community @@ -0,0 +1,5 @@ +#!/hint/bash + +PKGREPOS=({community,multilib}{,-testing,-staging}) +PKGPOOL='pool/community' +SRCPOOL='sources/community' diff --git a/config.local.import-packages b/config.local.import-packages new file mode 100644 index 0000000..c699b28 --- /dev/null +++ b/config.local.import-packages @@ -0,0 +1,5 @@ +#!/hint/bash + +PKGREPOS=('core' 'extra' 'testing' 'staging' {kde,gnome}-unstable) +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' diff --git a/config.local.parabola b/config.local.parabola new file mode 100644 index 0000000..648dfbb --- /dev/null +++ b/config.local.parabola @@ -0,0 +1,14 @@ +#!/hint/bash + +PKGREPOS=( + # Main repos + libre{,-testing} + libre-multilib{,-testing} + # Community project repos + {nonsystemd,nonprism}{,-testing} + pcr kernels cross java + # User repos + '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' +) +PKGPOOL='pool/parabola' +SRCPOOL='sources/parabola' diff --git a/config.local.svn-community b/config.local.svn-community new file mode 100644 index 0000000..5d61b5e --- /dev/null +++ b/config.local.svn-community @@ -0,0 +1,13 @@ +#!/hint/bash + +PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') +PKGPOOL='pool/community' +SRCPOOL='sources/community' +SVNREPO='file:///srv/repos/svn-community/svn' +SVNUSER='svn-community' +TESTING_REPO='community-testing' +STABLE_REPOS=('community') + +CLEANUP_DESTDIR="/srv/repos/svn-community/package-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-community/source-cleanup" +TMPDIR="/srv/repos/svn-community/tmp" diff --git a/config.local.svn-packages b/config.local.svn-packages new file mode 100644 index 0000000..34aab35 --- /dev/null +++ b/config.local.svn-packages @@ -0,0 +1,13 @@ +#!/hint/bash + +PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' +SVNREPO='file:///srv/repos/svn-packages/svn' +SVNUSER='svn-packages' +TESTING_REPO='testing' +STABLE_REPOS=('core' 'extra') + +CLEANUP_DESTDIR="/srv/repos/svn-packages/package-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup" +TMPDIR="/srv/repos/svn-packages/tmp" diff --git a/db-import b/db-import new file mode 100755 index 0000000..a8a073d --- /dev/null +++ b/db-import @@ -0,0 +1,288 @@ +#!/bin/bash +set -euE +# Imports Arch-like repos, running them through a blacklist +# License: GPLv3 + +. "$(dirname "$(readlink -e "$0")")/config" # for: FTP_BASE DBEXT +. "$(dirname "$(readlink -e "$0")")/db-import.conf" # for: IMPORTDIR IMPORTS +. "$(librelib messages)" +. "$(librelib blacklist)" + +# DBs = pacman DataBases + +# This replaces two scripts: +# - abslibre : imported ABS tree from Arch +# - db-sync : imported pacman DBs from Arch + +# The flow here is: +# 1. "${IMPORTDIR}/cache/${name}/dbs/" # Download the pacman databases +# 2. "${IMPORTDIR}/cache/${name}/abs/" # Download the ABS tree +# 3. "${IMPORTDIR}/clean/${name}/dbs/" # Run the pacman DBs through the blacklist +# 4. "${IMPORTDIR}/clean/${name}/pkgs/" # Download the pkg files mentioned in "clean/${name}/dbs/" +# 5. "${IMPORTDIR}/staging/${tag}" # Copy all the package files we just downloaded to here +# 6. Run `db-update on` with STAGING="${IMPORTDIR}/staging/${tag}" + +# generic arguments to pass to rsync, borrowed from `abs` +SYNCARGS='-mrtvlH --no-motd --no-p --no-o --no-g' + +main() { + blacklist-update + + local importStr + for importStr in "${IMPORTS[@]}"; do + local importAry=($importStr) + local name=${importAry[0]} + local pkgmirror=${importAry[1]} + local absmirror=${importAry[2]} + local tags=("${importAry[@]:3}") + + msg "Fetching remote package source: %s" "$name" + fetch_dbs "$name" "$pkgmirror" + fetch_abs "$name" "$absmirror" "${tags[@]}" + msg "Filtering blacklisted packages from remote package source: %s" "$name" + clean_dbs "$name" "${tags[@]}" + fetch_pkgs "$name" "${tags[@]}" + msg "Publishing changes from remote package source: %s" "$name" + publish "$name" "${tags[@]}" + done + return $r +} + +fetch_dbs() { + local name=$1 + local pkgmirror=$2 + + msg2 'Synchronizing package databases...' + + mkdir -p -- "${IMPORTDIR}/cache/${name}/dbs" + # Grab just the .db files from $pkgmirror + rsync $SYNCARGS --delete-after \ + --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --exclude="*" \ + "rsync://${pkgmirror}/" "${IMPORTDIR}/cache/${name}/dbs" +} + +fetch_abs() { + local name=$1 + local absmirror=$2 + local tags=("${@:3}") + + local fake_home + local absroot + + # Sync the ABS tree from $absmirror + local arch + for arch in $(list_arches "${tags[@]}"); do + msg2 'Synchronizing %s ABS tree...' "$arch" + + absroot="${IMPORTDIR}/cache/${name}/abs/${arch}" + mkdir -p -- "$absroot" + + # Configure `abs` for this mirror + fake_home="${IMPORTDIR}/homes/${name}/${arch}" + mkdir -p -- "$fake_home" + { + printf "ABSROOT='%s'\n" "$absroot" + printf "SYNCSERVER='%s'\n" "$absmirror" + printf "ARCH='%s'\n" "$arch" + printf 'REPOS=(\n' + list_repos "$arch" "${tags[@]}" + printf ')\n' + } > "${fake_home}/.abs.conf" + + # Run `abs` + HOME=$fake_home abs + done +} + +clean_dbs() { + local name=$1 + local tags=("${@:2}") + + rm -rf -- "${IMPORTDIR}/clean/$name" + + local tag + for tag in "${tags[@]}"; do + msg2 'Creating clean version of %s package database...' "$tag" + + local cache="${IMPORTDIR}/cache/$name/dbs/$(db_file "$tag")" + local clean="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + install -Dm644 "$cache" "$clean" + + blacklist-cat | blacklist-get-pkg | xargs -d '\n' repo-remove "$clean" + done +} + +fetch_pkgs() { + local name=$1 + local tags=("${@:2}") + + local repo arch dbfile whitelist + + local tag + for tag in "${tags[@]}"; do + msg2 'Syncronizing package files for %s...' "$tag" + repo=${tag%-*} + arch=${tag##*-} + + dbfile="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + whitelist="${IMPORTDIR}/clean/$name/dbs/$tag.whitelist" + + list_pkgs "$dbfile" > "$whitelist" + + # fetch the architecture-specific packages + rsync $SYNCARGS --delete-after --delete-excluded \ + --delay-updates \ + --include-from=<(sed "s|\$|-$arch.tar.?z|" "$whitelist") \ + --exclude='*' \ + "rsync://${pkgmirror}/$(db_dir "$tag")/" \ + "${IMPORTDIR}/clean/${name}/pkgs/${tag}/" + + # fetch the architecture-independent packages + rsync $SYNCARGS --delete-after --delete-excluded \ + --delay-updates \ + --include-from=<(sed "s|\$|-any.tar.?z|" "$whitelist") \ + --exclude='*' \ + "rsync://${pkgmirror}/$(db_dir "$tag")/" \ + "${IMPORTDIR}/clean/${name}/pkgs/${repo}-any/" + done +} + +publish() { + local name=$1 + local tags=("${@:2}") + + local tag + for tag in "${tags[@]}"; do + msg2 'Publishing changes to %s...' "$tag" + publish_tag "$name" "$tag" + done +} + +publish_tag() { + local name=$1 + local tag=$2 + + local repo=${tag%-*} + local arch=${tag##*-} + local dir="${IMPORTDIR}/clean/${name}/pkgs/${tag}" + + local found + local error=false + local files=() + + local pkgid pkgarch + for pkgid in $(list_added_pkgs "$name" "$tag"); do + found=false + + for pkgarch in "${arch}" any; do + file="${dir}/${pkgid}-${arch}".pkg.tar.?z + if ! $found && [[ -r $file ]]; then + files+=("$file") + found=true + fi + done + + if ! $found; then + error 'Could not find package file for %s' "$pkgid" + error=true + fi + done + + if $error; then + error 'Quitting...' + return 1 + fi + + mkdir -p -- "${IMPORTDIR}/staging/${tag}/${repo}" + cp -al -- "${files[@]}" "${IMPORTDIR}/staging/${tag}/${repo}/" + STAGING="${IMPORTDIR}/staging/${tag}" db-update + + # XXX: db-remove wants pkgbase, not pkgname + list_removed_pkgs "$name" "$tag" | xargs -d '\n' db-remove "$repo" "$arch" +} + +################################################################################ + +# Usage: list_arches repo-arch... +# Returns a list of the architectures mentioned in a list of "repo-arch" pairs. +list_arches() { + local tags=("$@") + printf '%s\n' "${tags[@]##*-}" | sort -u +} + +# Usage: list_repos arch repo-arch... +# Returns a list of all the repositories mentioned for a given architecture in a +# list of "repo-arch" pairs. +list_repos() { + local arch=$1 + local tags=("${@:2}") + printf '%s\n' "${tags[@]}" | sed -n "s/-$arch\$//p" +} + +# Usage: db_dir repo-arch +db_dir() { + local tag=$1 + local repo=${tag%-*} + local arch=${tag##*-} + echo "${repo}/os/${arch}" +} + +# Usage; db_file repo-arch +db_file() { + local tag=$1 + local repo=${tag%-*} + local arch=${tag##*-} + echo "${repo}/os/${arch}/${repo}${DBEXT}" +} + +# Usage: list_pkgs dbfile +# Prints "$pkgname-$(get_full_version "$pkgname")" for every package in $dbfile +list_pkgs() { + local dbfile=$1 + bsdtar tf "$dbfile" | cut -d/ -f1 +} + +# Usage: list_pkgs | sep_ver +# Separates the pkgname from the version (replaces the '-' with ' ') for the +# list provided on stdin. +sep_ver() { + sed -r 's/-([^-]*-[^-]*)$/ \1/' +} + +# Usage: list_removed_pkgs importsrc repo-arch +# Prints "$pkgname-$(get_full_version "$pkgname")" for every removed package. +list_removed_pkgs() { + local name=$1 + local tag=$2 + + local old="${FTP_BASE}/$(db_file "$tag")" + local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + + # make a list of: + # pkgname oldver[ newver] + # It will include removed or updated packages (changed packages) + join -a1 \ + <(list_pkgs "$old"|sep_ver|sort) \ + <(list_pkgs "$new"|sep_ver|sort) + | grep -v ' .* ' # remove updated packages + | sed 's/ /-/' # re-combine the pkgname and version +} + +# Usage: list_added_pkgs importsrc repo-arch +# slightly a misnomer; added and updated +# Prints "$pkgname-$(get_full_version "$pkgname")" for every added or updated +# package. +list_added_pkgs() { + local name=$1 + local tag=$2 + + local old="${FTP_BASE}/$(db_file "$tag")" + local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + + comm -13 <(list_pkgs "$old") <(list_pkgs "$new") +} + +main "$@" diff --git a/db-import-pick-mirror b/db-import-pick-mirror new file mode 100755 index 0000000..4d01b95 --- /dev/null +++ b/db-import-pick-mirror @@ -0,0 +1,25 @@ +#!/usr/bin/env ruby + +require 'json' +require 'net/http' + +protocol = ARGV[0] +jsonurl = ARGV[1] + +data = JSON::parse(Net::HTTP.get(URI(jsonurl))) + +if data["version"] != 3 + print "Data format version != 3" + exit 1 +end + +# Filter out URLs with incomplete information +urls = data["urls"].select{|a| a.none?{|k,v|v.nil?}} +rsync_urls = urls.select{|a| a["protocol"]==protocol} + +# By score ( (delay+speed)/completion ) +#best = rsync_urls.sort{|a,b| (a["score"] || Float::INFINITY) <=> (b["score"] || Float::INFINITY) }.first +# By delay/completion +best = rsync_urls.sort{|a,b| a["delay"]/a["completion_pct"] <=> b["delay"]/b["completion_pct"] }.first + +puts best["url"] diff --git a/db-import.conf b/db-import.conf new file mode 100644 index 0000000..e00e7b0 --- /dev/null +++ b/db-import.conf @@ -0,0 +1,23 @@ +#!/hint/bash + +IMPORTDIR=/srv/repo/import + +case "$USER" in + db-import-packages) + _archrepos=( + {core,extra,testing,staging}-{i686,x86_64} + {gnome,kde}-unstable-{i686,x86_64} + );; + db-import-community) + _archrepos=( + community{,-testing,-staging}-{i686,x86_64} + multilib{,-testing,-staging}-x86_64 + );; +esac + +_archpkgmirror=$(db-import-pick-mirror rsync https://www.archlinux.org/mirrors/status/tier/1/json/) + +# name pkgmirror absmirror repo-arch... +IMPORTS=("archlinux ${_archpkgmirror} rsync.archlinux.org ${_archrepos[*]}") + +unset _archrepos _archpkgmirror @@ -46,6 +46,9 @@ for repo in "${repos[@]}"; do if "${REQUIRE_SIGNATURE}" && ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then die "Package %s does not have a valid signature" "${repo}/${pkg##*/}" fi + if ! check_pkgxbs "${pkg}" "${repo}"; then + die "Package %s is not consistent with %s" "${repo}/${pkg##*/}" "$(xbs name)" + fi if ! check_pkgrepos "${pkg}"; then die "Package %s already exists in another repository" "${repo}/${pkg##*/}" fi @@ -59,10 +62,9 @@ for repo in "${repos[@]}"; do die "Package ${repo}/${pkg##*/} was not built in a chroot" fi done - # This is fucking obnoxious - #if ! check_splitpkgs ${repo} "${pkgs[@]}"; then - # die "Missing split packages for %s" "${repo}" - #fi + if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then + die "Missing split packages for %s" "${repo}" + fi else die "Could not read %s" "${STAGING}" fi |