diff options
author | Parabola <dev@list.parabolagnulinux.org> | 2014-01-16 04:02:30 +0000 |
---|---|---|
committer | Parabola <dev@list.parabolagnulinux.org> | 2014-01-16 04:02:30 +0000 |
commit | 0f9c53d616116cac705b01bfabb2186506aac52a (patch) | |
tree | 30cbe9934c9026ffe7be380856966e9b1e6764b1 | |
parent | 0684ad038134d7c9ccbaebc709c4267a904b98b9 (diff) |
fix new db-sync
-rwxr-xr-x | db-sync | 281 | ||||
-rwxr-xr-x | db-sync.orig | 191 |
2 files changed, 144 insertions, 328 deletions
@@ -13,174 +13,181 @@ # TODO # * make a tarball of files used for forensics -# * get files db # Run as `V=true db-sync` to get verbose output VERBOSE=${V} ${VERBOSE} && extra="-v" +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/$0.$$.cache -# Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache + # Exclude everything but db files + rsync ${extra} --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ "$WORKDIR" } get_repo_content() { -# Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u + # Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u } # Prints blacklisted packages get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" + cut -d ':' -f 1 "${BLACKLIST_FILE}" } # repo # arch get_repo_file() { -# shopt -s nullglob - - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}" + echo "${WORKDIR}/${1}/os/${2}/${1}" } # Process the databases and get the libre packages init() { -# Get the blacklisted packages - blacklist=($(get_blacklist)) -# Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - -# Sync the repos databases - get_repos - -# Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - repo_file=$(get_repo_file ${_repo} ${_arch}) - - if [ ! -f "${repo_file}" ]; then - warning "${repo_file} doesn't exist, skipping this repo-arch" - continue - fi - -# Remove blacklisted packages and count them -# TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages: $( - LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ - grep "\-> Removing" 2>/dev/null| wc -l)" - -# Get db contents - db=($(get_repo_content ${repo_file})) - - msg2 "Process clean db for syncing..." - -# Create a whitelist, add * wildcard to end -# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if -# we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - -# Sync excluding everything but whitelist -# We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Cleanup - unset db - done - done - - - msg "Syncing package pool" -# Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - -# Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - -# Cleanup - unset blacklist whitelists _arch _repo repo_file + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content ${db_file})) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + printf '%s\n' "${db[@]}" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync ${extra} --no-motd -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + ${WORKDIR}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done + + # Cleanup + unset blacklist whitelists _arch _repo repo_file } trap_exit() { - echo - error "$@" - exit 1 + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" } - -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR init - -rm -r ${TMPDIR}/$0.$$.cache diff --git a/db-sync.orig b/db-sync.orig deleted file mode 100755 index 0e07757..0000000 --- a/db-sync.orig +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics -# * get files db - -# Run as `V=true db-sync` to get verbose output -VERBOSE=${V} -${VERBOSE} && extra="-v" - -# Returns contents of a repo -get_repos() { - mkdir -p ${TMPDIR}/${0##*/}.$$.cache - # Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache -} - -get_repo_content() { - # Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from .db database..." - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" - msg2 "Removing blacklisted packages from .files database..." - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" - - # Get db contents - db=($(get_repo_content ${db_file})) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - # Cleanup - unset blacklist whitelists _arch _repo repo_file -} - -trap_exit() { - echo - error "$@" - exit 1 -} - - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" -source "$(dirname "$(readlink -e "$0")")/libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -init - -rm -r ${TMPDIR}/${0##*/}.$$.cache |