From a4fb5765c0fe8f6d03985d5375b09c0cd68903f2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 13:49:03 -0400 Subject: switch up the naming schem for the import scripts --- db-import-archlinuxarm-pkg | 204 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100755 db-import-archlinuxarm-pkg (limited to 'db-import-archlinuxarm-pkg') diff --git a/db-import-archlinuxarm-pkg b/db-import-archlinuxarm-pkg new file mode 100755 index 0000000..8d8cb3d --- /dev/null +++ b/db-import-archlinuxarm-pkg @@ -0,0 +1,204 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * make a tarball of files used for forensics + +set -e + +# Run as `V=true db-import-pkg-archlinux` to get verbose output +VERBOSE=${V} +extra=() +${VERBOSE} && extra+=(-v) + +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + +# Returns contents of a repo +get_repos() { + # Exclude everything but db files + rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" +} + +get_repo_content() { + # Return all contents + bsdtar tf "${1}" | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + echo "${WORKDIR}/${2}/${1}/${1}" +} + +# Process the databases and get the libre packages +init() { + + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _arch in "${OURARCHES[@]}"; do + for _repo in "${ARMREPOS[@]}"; do + msg "Processing %s-%s" "${_repo}" "${_arch}" + + db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} + files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this arch-repo" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this arch-repo" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content "${db_file}")) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" + + msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + # Sync excluding everything but whitelist + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ + "${FTP_BASE}/${PKGPOOLARM}/" + + msg "Putting databases back in place" + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + "${WORKDIR}/${_arch}/${_repo}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Cleanup + unset db + done + done + + + msg "Generating symbolic links to pool" + + for _arch in "${OURARCHES[@]}"; do + for _repo in "${ARMREPOS[@]}"; do + # Modify whitelist to search packages and create symlinks + sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist" + + msg "Putting symlinks in ${_repo}/os/${_arch}" + + while read _pkgfile; do + # Symlink to package + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" + elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}" + fi + + # Symlink to signature + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" + elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig" + fi + done < "/tmp/${_repo}-${_arch}.whitelist" + done + done + + date -u +%s > "${FTP_BASE}/lastsync" + + # Cleanup + unset blacklist whitelists _arch _repo repo_file _pkgfile +} + +trap_exit() { + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" +} + +fatal_error() { + error "$@" + exit 1 +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-archlinuxarm.conf" +source "$(librelib messages)" + +# Check variables presence +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE; do + test -z "${!var}" && fatal_error "Empty %s" "${var}" +done + +# From makepkg +set -E +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR + +init -- cgit v1.2.3