summaryrefslogtreecommitdiff
path: root/db-import
diff options
context:
space:
mode:
Diffstat (limited to 'db-import')
-rwxr-xr-xdb-import288
1 files changed, 288 insertions, 0 deletions
diff --git a/db-import b/db-import
new file mode 100755
index 0000000..a8a073d
--- /dev/null
+++ b/db-import
@@ -0,0 +1,288 @@
+#!/bin/bash
+set -euE
+# Imports Arch-like repos, running them through a blacklist
+# License: GPLv3
+
+. "$(dirname "$(readlink -e "$0")")/config" # for: FTP_BASE DBEXT
+. "$(dirname "$(readlink -e "$0")")/db-import.conf" # for: IMPORTDIR IMPORTS
+. "$(librelib messages)"
+. "$(librelib blacklist)"
+
+# DBs = pacman DataBases
+
+# This replaces two scripts:
+# - abslibre : imported ABS tree from Arch
+# - db-sync : imported pacman DBs from Arch
+
+# The flow here is:
+# 1. "${IMPORTDIR}/cache/${name}/dbs/" # Download the pacman databases
+# 2. "${IMPORTDIR}/cache/${name}/abs/" # Download the ABS tree
+# 3. "${IMPORTDIR}/clean/${name}/dbs/" # Run the pacman DBs through the blacklist
+# 4. "${IMPORTDIR}/clean/${name}/pkgs/" # Download the pkg files mentioned in "clean/${name}/dbs/"
+# 5. "${IMPORTDIR}/staging/${tag}" # Copy all the package files we just downloaded to here
+# 6. Run `db-update on` with STAGING="${IMPORTDIR}/staging/${tag}"
+
+# generic arguments to pass to rsync, borrowed from `abs`
+SYNCARGS='-mrtvlH --no-motd --no-p --no-o --no-g'
+
+main() {
+ blacklist-update
+
+ local importStr
+ for importStr in "${IMPORTS[@]}"; do
+ local importAry=($importStr)
+ local name=${importAry[0]}
+ local pkgmirror=${importAry[1]}
+ local absmirror=${importAry[2]}
+ local tags=("${importAry[@]:3}")
+
+ msg "Fetching remote package source: %s" "$name"
+ fetch_dbs "$name" "$pkgmirror"
+ fetch_abs "$name" "$absmirror" "${tags[@]}"
+ msg "Filtering blacklisted packages from remote package source: %s" "$name"
+ clean_dbs "$name" "${tags[@]}"
+ fetch_pkgs "$name" "${tags[@]}"
+ msg "Publishing changes from remote package source: %s" "$name"
+ publish "$name" "${tags[@]}"
+ done
+ return $r
+}
+
+fetch_dbs() {
+ local name=$1
+ local pkgmirror=$2
+
+ msg2 'Synchronizing package databases...'
+
+ mkdir -p -- "${IMPORTDIR}/cache/${name}/dbs"
+ # Grab just the .db files from $pkgmirror
+ rsync $SYNCARGS --delete-after \
+ --include="*/" \
+ --include="*.db" \
+ --include="*${DBEXT}" \
+ --exclude="*" \
+ "rsync://${pkgmirror}/" "${IMPORTDIR}/cache/${name}/dbs"
+}
+
+fetch_abs() {
+ local name=$1
+ local absmirror=$2
+ local tags=("${@:3}")
+
+ local fake_home
+ local absroot
+
+ # Sync the ABS tree from $absmirror
+ local arch
+ for arch in $(list_arches "${tags[@]}"); do
+ msg2 'Synchronizing %s ABS tree...' "$arch"
+
+ absroot="${IMPORTDIR}/cache/${name}/abs/${arch}"
+ mkdir -p -- "$absroot"
+
+ # Configure `abs` for this mirror
+ fake_home="${IMPORTDIR}/homes/${name}/${arch}"
+ mkdir -p -- "$fake_home"
+ {
+ printf "ABSROOT='%s'\n" "$absroot"
+ printf "SYNCSERVER='%s'\n" "$absmirror"
+ printf "ARCH='%s'\n" "$arch"
+ printf 'REPOS=(\n'
+ list_repos "$arch" "${tags[@]}"
+ printf ')\n'
+ } > "${fake_home}/.abs.conf"
+
+ # Run `abs`
+ HOME=$fake_home abs
+ done
+}
+
+clean_dbs() {
+ local name=$1
+ local tags=("${@:2}")
+
+ rm -rf -- "${IMPORTDIR}/clean/$name"
+
+ local tag
+ for tag in "${tags[@]}"; do
+ msg2 'Creating clean version of %s package database...' "$tag"
+
+ local cache="${IMPORTDIR}/cache/$name/dbs/$(db_file "$tag")"
+ local clean="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
+ install -Dm644 "$cache" "$clean"
+
+ blacklist-cat | blacklist-get-pkg | xargs -d '\n' repo-remove "$clean"
+ done
+}
+
+fetch_pkgs() {
+ local name=$1
+ local tags=("${@:2}")
+
+ local repo arch dbfile whitelist
+
+ local tag
+ for tag in "${tags[@]}"; do
+ msg2 'Syncronizing package files for %s...' "$tag"
+ repo=${tag%-*}
+ arch=${tag##*-}
+
+ dbfile="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
+ whitelist="${IMPORTDIR}/clean/$name/dbs/$tag.whitelist"
+
+ list_pkgs "$dbfile" > "$whitelist"
+
+ # fetch the architecture-specific packages
+ rsync $SYNCARGS --delete-after --delete-excluded \
+ --delay-updates \
+ --include-from=<(sed "s|\$|-$arch.tar.?z|" "$whitelist") \
+ --exclude='*' \
+ "rsync://${pkgmirror}/$(db_dir "$tag")/" \
+ "${IMPORTDIR}/clean/${name}/pkgs/${tag}/"
+
+ # fetch the architecture-independent packages
+ rsync $SYNCARGS --delete-after --delete-excluded \
+ --delay-updates \
+ --include-from=<(sed "s|\$|-any.tar.?z|" "$whitelist") \
+ --exclude='*' \
+ "rsync://${pkgmirror}/$(db_dir "$tag")/" \
+ "${IMPORTDIR}/clean/${name}/pkgs/${repo}-any/"
+ done
+}
+
+publish() {
+ local name=$1
+ local tags=("${@:2}")
+
+ local tag
+ for tag in "${tags[@]}"; do
+ msg2 'Publishing changes to %s...' "$tag"
+ publish_tag "$name" "$tag"
+ done
+}
+
+publish_tag() {
+ local name=$1
+ local tag=$2
+
+ local repo=${tag%-*}
+ local arch=${tag##*-}
+ local dir="${IMPORTDIR}/clean/${name}/pkgs/${tag}"
+
+ local found
+ local error=false
+ local files=()
+
+ local pkgid pkgarch
+ for pkgid in $(list_added_pkgs "$name" "$tag"); do
+ found=false
+
+ for pkgarch in "${arch}" any; do
+ file="${dir}/${pkgid}-${arch}".pkg.tar.?z
+ if ! $found && [[ -r $file ]]; then
+ files+=("$file")
+ found=true
+ fi
+ done
+
+ if ! $found; then
+ error 'Could not find package file for %s' "$pkgid"
+ error=true
+ fi
+ done
+
+ if $error; then
+ error 'Quitting...'
+ return 1
+ fi
+
+ mkdir -p -- "${IMPORTDIR}/staging/${tag}/${repo}"
+ cp -al -- "${files[@]}" "${IMPORTDIR}/staging/${tag}/${repo}/"
+ STAGING="${IMPORTDIR}/staging/${tag}" db-update
+
+ # XXX: db-remove wants pkgbase, not pkgname
+ list_removed_pkgs "$name" "$tag" | xargs -d '\n' db-remove "$repo" "$arch"
+}
+
+################################################################################
+
+# Usage: list_arches repo-arch...
+# Returns a list of the architectures mentioned in a list of "repo-arch" pairs.
+list_arches() {
+ local tags=("$@")
+ printf '%s\n' "${tags[@]##*-}" | sort -u
+}
+
+# Usage: list_repos arch repo-arch...
+# Returns a list of all the repositories mentioned for a given architecture in a
+# list of "repo-arch" pairs.
+list_repos() {
+ local arch=$1
+ local tags=("${@:2}")
+ printf '%s\n' "${tags[@]}" | sed -n "s/-$arch\$//p"
+}
+
+# Usage: db_dir repo-arch
+db_dir() {
+ local tag=$1
+ local repo=${tag%-*}
+ local arch=${tag##*-}
+ echo "${repo}/os/${arch}"
+}
+
+# Usage; db_file repo-arch
+db_file() {
+ local tag=$1
+ local repo=${tag%-*}
+ local arch=${tag##*-}
+ echo "${repo}/os/${arch}/${repo}${DBEXT}"
+}
+
+# Usage: list_pkgs dbfile
+# Prints "$pkgname-$(get_full_version "$pkgname")" for every package in $dbfile
+list_pkgs() {
+ local dbfile=$1
+ bsdtar tf "$dbfile" | cut -d/ -f1
+}
+
+# Usage: list_pkgs | sep_ver
+# Separates the pkgname from the version (replaces the '-' with ' ') for the
+# list provided on stdin.
+sep_ver() {
+ sed -r 's/-([^-]*-[^-]*)$/ \1/'
+}
+
+# Usage: list_removed_pkgs importsrc repo-arch
+# Prints "$pkgname-$(get_full_version "$pkgname")" for every removed package.
+list_removed_pkgs() {
+ local name=$1
+ local tag=$2
+
+ local old="${FTP_BASE}/$(db_file "$tag")"
+ local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
+
+ # make a list of:
+ # pkgname oldver[ newver]
+ # It will include removed or updated packages (changed packages)
+ join -a1 \
+ <(list_pkgs "$old"|sep_ver|sort) \
+ <(list_pkgs "$new"|sep_ver|sort)
+ | grep -v ' .* ' # remove updated packages
+ | sed 's/ /-/' # re-combine the pkgname and version
+}
+
+# Usage: list_added_pkgs importsrc repo-arch
+# slightly a misnomer; added and updated
+# Prints "$pkgname-$(get_full_version "$pkgname")" for every added or updated
+# package.
+list_added_pkgs() {
+ local name=$1
+ local tag=$2
+
+ local old="${FTP_BASE}/$(db_file "$tag")"
+ local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
+
+ comm -13 <(list_pkgs "$old") <(list_pkgs "$new")
+}
+
+main "$@"