summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorParabola <dev@list.parabolagnulinux.org>2014-01-16 04:02:30 +0000
committerParabola <dev@list.parabolagnulinux.org>2014-01-16 04:02:30 +0000
commit0f9c53d616116cac705b01bfabb2186506aac52a (patch)
tree30cbe9934c9026ffe7be380856966e9b1e6764b1
parent0684ad038134d7c9ccbaebc709c4267a904b98b9 (diff)
fix new db-sync
-rwxr-xr-xdb-sync281
-rwxr-xr-xdb-sync.orig191
2 files changed, 144 insertions, 328 deletions
diff --git a/db-sync b/db-sync
index 81dee24..c90f89b 100755
--- a/db-sync
+++ b/db-sync
@@ -13,174 +13,181 @@
# TODO
# * make a tarball of files used for forensics
-# * get files db
# Run as `V=true db-sync` to get verbose output
VERBOSE=${V}
${VERBOSE} && extra="-v"
+WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
+
# Returns contents of a repo
get_repos() {
- mkdir -p ${TMPDIR}/$0.$$.cache
-# Exclude everything but db files
- rsync ${extra} -mrtlH --no-p --include="*/" \
- --include="*.db" \
- --include="*${DBEXT}" \
- --exclude="*" \
- --delete-after \
- rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache
+ # Exclude everything but db files
+ rsync ${extra} --no-motd -mrtlH --no-p --include="*/" \
+ --include="*.db" \
+ --include="*${DBEXT}" \
+ --include="*.files" \
+ --include="*${FILESEXT}" \
+ --exclude="*" \
+ --delete-after \
+ rsync://${mirror}/${mirrorpath}/ "$WORKDIR"
}
get_repo_content() {
-# Return all contents
- bsdtar tf ${1} | \
- cut -d "/" -f 1 | \
- sort -u
+ # Return all contents
+ bsdtar tf ${1} | \
+ cut -d "/" -f 1 | \
+ sort -u
}
# Prints blacklisted packages
get_blacklist() {
- cut -d ':' -f 1 "${BLACKLIST_FILE}"
+ cut -d ':' -f 1 "${BLACKLIST_FILE}"
}
# repo
# arch
get_repo_file() {
-# shopt -s nullglob
-
- echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}"
+ echo "${WORKDIR}/${1}/os/${2}/${1}"
}
# Process the databases and get the libre packages
init() {
-# Get the blacklisted packages
- blacklist=($(get_blacklist))
-# Store all the whitelist files
- whitelists=()
-
- msg "${#blacklist[@]} packages in blacklist"
-
-# Sync the repos databases
- get_repos
-
-# Traverse all repo-arch pairs
- for _repo in ${ARCHREPOS[@]}; do
- for _arch in ${ARCHARCHES[@]}; do
- msg "Processing ${_repo}-${_arch}"
-
- repo_file=$(get_repo_file ${_repo} ${_arch})
-
- if [ ! -f "${repo_file}" ]; then
- warning "${repo_file} doesn't exist, skipping this repo-arch"
- continue
- fi
-
-# Remove blacklisted packages and count them
-# TODO capture all removed packages for printing on debug mode
- msg2 "Removing blacklisted packages: $(
- LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \
- grep "\-> Removing" 2>/dev/null| wc -l)"
-
-# Get db contents
- db=($(get_repo_content ${repo_file}))
-
- msg2 "Process clean db for syncing..."
-
-# Create a whitelist, add * wildcard to end
-# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
-# we aren't syncing them
- echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
-
- msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
-
-# Sync excluding everything but whitelist
-# We delete here for cleanup
- rsync ${extra} -rtlH \
- --delete-after \
- --delete-excluded \
- --delay-updates \
- --include-from=/tmp/${_repo}-${_arch}.whitelist \
- --exclude="*" \
- rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
- ${FTP_BASE}/${_repo}/os/${_arch}/
-
-# Add a new whitelist
- whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
-
- msg "Putting databases back in place"
- rsync ${extra} -rtlH \
- --delay-updates \
- --safe-links \
- ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \
- ${FTP_BASE}/${_repo}/os/${_arch}/
-
-# Cleanup
- unset db
- done
- done
-
-
- msg "Syncing package pool"
-# Concatenate all whitelists
- cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
-
- msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
-
-# Sync
-# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-# packages too
- for PKGPOOL in ${PKGPOOLS[@]}; do
- rsync ${extra} -rtlH \
- --delay-updates \
- --safe-links \
- --include-from=/tmp/any.whitelist \
- --exclude="*" \
- rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
- ${FTP_BASE}/${PKGPOOL}/
- done
-
-# Sync sources
- msg "Syncing source pool"
- #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
-
- #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool"
-# Sync
-# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-# packages too
- for SRCPOOL in ${SRCPOOLS[@]}; do
- rsync ${extra} -rtlH \
- --delay-updates \
- --safe-links \
- --include-from=/tmp/any.whitelist \
- --exclude="*" \
- rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
- ${FTP_BASE}/${SRCPOOL}/
- done
-
-
-# Cleanup
- unset blacklist whitelists _arch _repo repo_file
+ # Get the blacklisted packages
+ blacklist=($(get_blacklist))
+ # Store all the whitelist files
+ whitelists=()
+
+ msg "%d packages in blacklist" ${#blacklist[@]}
+
+ # Sync the repos databases
+ get_repos
+
+ # Traverse all repo-arch pairs
+ for _repo in ${ARCHREPOS[@]}; do
+ for _arch in ${ARCHARCHES[@]}; do
+ msg "Processing ${_repo}-${_arch}"
+
+ db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT}
+ files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT}
+
+ if [ ! -f "${db_file}" ]; then
+ warning "%s doesn't exist, skipping this repo-arch" "${db_file}"
+ continue
+ fi
+ if [ ! -f "${files_file}" ]; then
+ warning "%s doesn't exist, skipping this repo-arch" "${files_file}"
+ continue
+ fi
+
+ # Remove blacklisted packages and count them
+ # TODO capture all removed packages for printing on debug mode
+ msg2 "Removing blacklisted packages from %s database..." .db
+ LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
+ |& sed -n 's/-> Removing/ &/p'
+ msg2 "Removing blacklisted packages from %s database..." .files
+ LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \
+ |& sed -n 's/-> Removing/ &/p'
+ # Get db contents
+ db=($(get_repo_content ${db_file}))
+
+ msg2 "Process clean db for syncing..."
+
+ # Create a whitelist, add * wildcard to end
+ # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+ # we aren't syncing them
+ printf '%s\n' "${db[@]}" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
+
+ msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
+
+ # Sync excluding everything but whitelist
+ # We delete here for cleanup
+ rsync ${extra} --no-motd -rtlH \
+ --delete-after \
+ --delete-excluded \
+ --delay-updates \
+ --include-from=/tmp/${_repo}-${_arch}.whitelist \
+ --exclude="*" \
+ rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
+ ${FTP_BASE}/${_repo}/os/${_arch}/
+
+ # Add a new whitelist
+ whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
+
+ msg "Putting databases back in place"
+ rsync ${extra} --no-motd -rtlH \
+ --delay-updates \
+ --safe-links \
+ ${WORKDIR}/${_repo}/os/${_arch}/ \
+ ${FTP_BASE}/${_repo}/os/${_arch}/
+
+ # Cleanup
+ unset db
+ done
+ done
+
+
+ msg "Syncing package pool"
+ # Concatenate all whitelists
+ cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
+
+ msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
+
+ # Sync
+ # *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+ # packages too
+ for PKGPOOL in ${PKGPOOLS[@]}; do
+ rsync ${extra} --no-motd -rtlH \
+ --delay-updates \
+ --safe-links \
+ --include-from=/tmp/any.whitelist \
+ --exclude="*" \
+ rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
+ ${FTP_BASE}/${PKGPOOL}/
+ done
+
+ # Sync sources
+ msg "Syncing source pool"
+ #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
+ #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist)
+
+ # Sync
+ # *Don't delete-after*, this is the job of cleanup scripts. It will remove our
+ # packages too
+ for SRCPOOL in ${SRCPOOLS[@]}; do
+ rsync ${extra} --no-motd -rtlH \
+ --delay-updates \
+ --safe-links \
+ --include-from=/tmp/any.whitelist \
+ --exclude="*" \
+ rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
+ ${FTP_BASE}/${SRCPOOL}/
+ done
+
+ # Cleanup
+ unset blacklist whitelists _arch _repo repo_file
}
trap_exit() {
- echo
- error "$@"
- exit 1
+ local signal=$1; shift
+ echo
+ error "$@"
+ trap -- "$signal"
+ kill "-$signal" "$$"
}
-
-source $(dirname $0)/config
-source $(dirname $0)/local_config
-source $(dirname $0)/libremessages
+source "$(dirname "$(readlink -e "$0")")/config"
+source "$(dirname "$(readlink -e "$0")")/local_config"
+source "$(dirname "$(readlink -e "$0")")/libremessages"
# From makepkg
set -E
-
-trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
-trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
-trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
+for signal in TERM HUP QUIT; do
+ trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal
+done
+trap 'trap_exit INT "Aborted by user! Exiting..."' INT
+trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR
init
-
-rm -r ${TMPDIR}/$0.$$.cache
diff --git a/db-sync.orig b/db-sync.orig
deleted file mode 100755
index 0e07757..0000000
--- a/db-sync.orig
+++ /dev/null
@@ -1,191 +0,0 @@
-#!/bin/bash
-# Syncs Arch repos based on info contained in repo.db files
-# License: GPLv3
-
-# Principles
-# * Get repo.db from an Arch-like repo
-# * Generate a list of available packages
-# * Create sync whitelist (based on package blacklist)
-# * Get packages
-# * Check package signatures
-# * Check database signatures
-# * Sync repo => repo
-
-# TODO
-# * make a tarball of files used for forensics
-# * get files db
-
-# Run as `V=true db-sync` to get verbose output
-VERBOSE=${V}
-${VERBOSE} && extra="-v"
-
-# Returns contents of a repo
-get_repos() {
- mkdir -p ${TMPDIR}/${0##*/}.$$.cache
- # Exclude everything but db files
- rsync ${extra} -mrtlH --no-p --include="*/" \
- --include="*.db" \
- --include="*${DBEXT}" \
- --include="*.files" \
- --include="*${FILESEXT}" \
- --exclude="*" \
- --delete-after \
- rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache
-}
-
-get_repo_content() {
- # Return all contents
- bsdtar tf ${1} | \
- cut -d "/" -f 1 | \
- sort -u
-}
-
-# Prints blacklisted packages
-get_blacklist() {
- cut -d ':' -f 1 "${BLACKLIST_FILE}"
-}
-
-# repo
-# arch
-get_repo_file() {
- echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}"
-}
-
-# Process the databases and get the libre packages
-init() {
-
- # Get the blacklisted packages
- blacklist=($(get_blacklist))
- # Store all the whitelist files
- whitelists=()
-
- msg "${#blacklist[@]} packages in blacklist"
-
- # Sync the repos databases
- get_repos
-
- # Traverse all repo-arch pairs
- for _repo in ${ARCHREPOS[@]}; do
- for _arch in ${ARCHARCHES[@]}; do
- msg "Processing ${_repo}-${_arch}"
-
- db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT}
- files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT}
-
- if [ ! -f "${db_file}" ]; then
- warning "%s doesn't exist, skipping this repo-arch" "${db_file}"
- continue
- fi
- if [ ! -f "${files_file}" ]; then
- warning "%s doesn't exist, skipping this repo-arch" "${files_file}"
- continue
- fi
-
- # Remove blacklisted packages and count them
- # TODO capture all removed packages for printing on debug mode
- msg2 "Removing blacklisted packages from .db database..."
- LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}"
- msg2 "Removing blacklisted packages from .files database..."
- LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}"
-
- # Get db contents
- db=($(get_repo_content ${db_file}))
-
- msg2 "Process clean db for syncing..."
-
- # Create a whitelist, add * wildcard to end
- # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
- # we aren't syncing them
- echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist
-
- msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist"
-
- # Sync excluding everything but whitelist
- # We delete here for cleanup
- rsync ${extra} -rtlH \
- --delete-after \
- --delete-excluded \
- --delay-updates \
- --include-from=/tmp/${_repo}-${_arch}.whitelist \
- --exclude="*" \
- rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \
- ${FTP_BASE}/${_repo}/os/${_arch}/
-
- # Add a new whitelist
- whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
-
- msg "Putting databases back in place"
- rsync ${extra} -rtlH \
- --delay-updates \
- --safe-links \
- ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \
- ${FTP_BASE}/${_repo}/os/${_arch}/
-
- # Cleanup
- unset db
- done
- done
-
-
- msg "Syncing package pool"
- # Concatenate all whitelists
- cat ${whitelists[@]} | sort -u > /tmp/any.whitelist
-
- msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool"
-
- # Sync
- # *Don't delete-after*, this is the job of cleanup scripts. It will remove our
- # packages too
- for PKGPOOL in ${PKGPOOLS[@]}; do
- rsync ${extra} -rtlH \
- --delay-updates \
- --safe-links \
- --include-from=/tmp/any.whitelist \
- --exclude="*" \
- rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \
- ${FTP_BASE}/${PKGPOOL}/
- done
-
- # Sync sources
- msg "Syncing source pool"
- #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
-
- #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool"
- # Sync
- # *Don't delete-after*, this is the job of cleanup scripts. It will remove our
- # packages too
- for SRCPOOL in ${SRCPOOLS[@]}; do
- rsync ${extra} -rtlH \
- --delay-updates \
- --safe-links \
- --include-from=/tmp/any.whitelist \
- --exclude="*" \
- rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \
- ${FTP_BASE}/${SRCPOOL}/
- done
-
- # Cleanup
- unset blacklist whitelists _arch _repo repo_file
-}
-
-trap_exit() {
- echo
- error "$@"
- exit 1
-}
-
-
-source "$(dirname "$(readlink -e "$0")")/config"
-source "$(dirname "$(readlink -e "$0")")/local_config"
-source "$(dirname "$(readlink -e "$0")")/libremessages"
-
-# From makepkg
-set -E
-
-trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
-trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
-trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
-
-init
-
-rm -r ${TMPDIR}/${0##*/}.$$.cache