summaryrefslogtreecommitdiff
path: root/db-sync-arm
diff options
context:
space:
mode:
authorLuke Shumaker <lukeshu@sbcglobal.net>2016-04-17 01:56:43 -0400
committerLuke Shumaker <lukeshu@sbcglobal.net>2016-04-17 01:56:43 -0400
commitff2983c207e6ab2a9381e20312355c42307187f6 (patch)
treeba4a2aca77c4f0b9963dab8754e5b1d58f711a88 /db-sync-arm
parent6ad57ac9f185dcfd4e02174ffdaff9bca1b0516c (diff)
Give things more consistent names
Diffstat (limited to 'db-sync-arm')
-rwxr-xr-xdb-sync-arm199
1 files changed, 0 insertions, 199 deletions
diff --git a/db-sync-arm b/db-sync-arm
deleted file mode 100755
index 952b546..0000000
--- a/db-sync-arm
+++ /dev/null
@@ -1,199 +0,0 @@
-#!/bin/bash
-# Syncs Arch repos based on info contained in repo.db files
-# License: GPLv3
-
-# Principles
-# * Get repo.db from an Arch-like repo
-# * Generate a list of available packages
-# * Create sync whitelist (based on package blacklist)
-# * Get packages
-# * Check package signatures
-# * Check database signatures
-# * Sync repo => repo
-
-# TODO
-# * make a tarball of files used for forensics
-
-set -e
-
-# Run as `V=true db-sync` to get verbose output
-VERBOSE=${V}
-extra=()
-${VERBOSE} && extra+=(-v)
-
-WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
-trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
-
-# Returns contents of a repo
-get_repos() {
- # Exclude everything but db files
- rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \
- --include="*.db" \
- --include="*${DBEXT}" \
- --include="*.files" \
- --include="*${FILESEXT}" \
- --exclude="*" \
- --delete-after \
- "rsync://${mirror}/${mirrorpath}/" "$WORKDIR"
-}
-
-get_repo_content() {
- # Return all contents
- bsdtar tf "${1}" | \
- cut -d "/" -f 1 | \
- sort -u
-}
-
-# Prints blacklisted packages
-get_blacklist() {
- cut -d ':' -f 1 "${BLACKLIST_FILE}"
-}
-
-# repo
-# arch
-get_repo_file() {
- echo "${WORKDIR}/${2}/${1}/${1}"
-}
-
-# Process the databases and get the libre packages
-init() {
-
- # Get the blacklisted packages
- blacklist=($(get_blacklist))
- # Store all the whitelist files
- whitelists=()
-
- msg "%d packages in blacklist" ${#blacklist[@]}
-
- test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist"
-
- # Sync the repos databases
- get_repos
-
- # Traverse all repo-arch pairs
- for _arch in "${OURARCHES[@]}"; do
- for _repo in "${ARMREPOS[@]}"; do
- msg "Processing %s-%s" "${_repo}" "${_arch}"
-
- db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT}
- files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT}
-
- if [ ! -f "${db_file}" ]; then
- warning "%s doesn't exist, skipping this arch-repo" "${db_file}"
- continue
- fi
- if [ ! -f "${files_file}" ]; then
- warning "%s doesn't exist, skipping this arch-repo" "${files_file}"
- continue
- fi
-
- # Remove blacklisted packages and count them
- # TODO capture all removed packages for printing on debug mode
- msg2 "Removing blacklisted packages from %s database..." .db
- LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
- |& sed -n 's/-> Removing/ &/p'
- msg2 "Removing blacklisted packages from %s database..." .files
- LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \
- |& sed -n 's/-> Removing/ &/p'
- # Get db contents
- db=($(get_repo_content "${db_file}"))
-
- msg2 "Process clean db for syncing..."
-
- # Create a whitelist, add * wildcard to end
- # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
- # we aren't syncing them
- # IMPORTANT: the . in the sed command is needed because an empty
- # whitelist would consist of a single * allowing any package to
- # pass through
- printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist"
-
- msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
-
- msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
-
- # Sync excluding everything but whitelist
- rsync "${extra[@]}" --no-motd -rtlH \
- --delay-updates \
- --safe-links \
- --include-from="/tmp/${_repo}-${_arch}.whitelist" \
- --exclude="*" \
- "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \
- "${FTP_BASE}/${PKGPOOLARM}/"
-
- msg "Putting databases back in place"
- rsync "${extra[@]}" --no-motd -rtlH \
- --delay-updates \
- --safe-links \
- "${WORKDIR}/${_arch}/${_repo}/" \
- "${FTP_BASE}/${_repo}/os/${_arch}/"
-
- # Cleanup
- unset db
- done
- done
-
-
- msg "Generating symbolic links to pool"
-
- for _arch in "${OURARCHES[@]}"; do
- for _repo in "${ARMREPOS[@]}"; do
- # Modify whitelist to search packages and create symlinks
- sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist"
-
- msg "Putting symlinks in ${_repo}/os/${_arch}"
-
- while read _pkgfile; do
- # Symlink to package
- if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then
- ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \
- "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}"
- elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}" ]; then
- ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}" \
- "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}"
- fi
-
- # Symlink to signature
- if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then
- ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \
- "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig"
- elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" ]; then
- ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" \
- "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig"
- fi
- done < "/tmp/${_repo}-${_arch}.whitelist"
- done
- done
-
- date -u +%s > "${FTP_BASE}/lastsync"
-
- # Cleanup
- unset blacklist whitelists _arch _repo repo_file _pkgfile
-}
-
-trap_exit() {
- local signal=$1; shift
- echo
- error "$@"
- trap -- "$signal"
- kill "-$signal" "$$"
-}
-
-source "$(dirname "$(readlink -e "$0")")/config"
-source "$(dirname "$(readlink -e "$0")")/db-sync-arm.conf"
-source "$(dirname "$(readlink -e "$0")")/db-libremessages"
-
-# Check variables presence
-for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do
- test -z "${!var}" && fatal_error "Empty %s" "${var}"
-done
-
-# From makepkg
-set -E
-for signal in TERM HUP QUIT; do
- trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal"
-done
-trap 'trap_exit INT "Aborted by user! Exiting..."' INT
-trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR
-
-init