diff options
-rw-r--r-- | config | 16 | ||||
-rwxr-xr-x | createrepos | 8 | ||||
-rwxr-xr-x | db-sync | 171 |
3 files changed, 187 insertions, 8 deletions
@@ -1,10 +1,10 @@ #!/bin/bash -FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +FTP_BASE="/tmp/repo" +ARCH_BASE="/tmp/repo" +SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') +ARCHREPOS=('core') #'extra' 'community' 'testing' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos @@ -31,13 +31,13 @@ LOCK_DELAY=10 LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" -TMPDIR="$HOME/tmp" -ARCHARCHES=(i686 x86_64) +TMPDIR="/tmp" +ARCHARCHES=(i686) # x86_64) ARCHES=(${ARCHARCHES[@]} mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" -MAKEPKGCONF="$HOME/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" +MAKEPKGCONF="/etc/makepkg.conf" +BLACKLIST_FILE="$FTP_BASE/blacklist.txt" diff --git a/createrepos b/createrepos new file mode 100755 index 0000000..4ee057b --- /dev/null +++ b/createrepos @@ -0,0 +1,8 @@ +#!/bin/bash +# Creates the repo structure defined in config + +source $(dirname $0)/config + +mkdir -p ${FTP_BASE}/{${PKGPOOL},${SRCPOOL}} ${ARCH_BASE} ${CLEANUP_DESTDIR} ${SOURCE_CLEANUP_DESTDIR} ${STAGING} + +$(dirname $0)/create-repo ${PKGREPOS[@]} @@ -0,0 +1,171 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * verbose mode +# * make a tarball of files used for forensics +# * get files db + +# Returns contents of a repo +get_repos() { +# Exclude everything but db files + rsync -avm --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ cache/ +} + +get_repo_content() { +# Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + [ ! -r "cache/${1}/os/${2}/${1}${DBEXT}" ] && return 1 + + echo "cache/${1}/os/${2}/${1}${DBEXT}" +} + +# Process the databases and get the libre packages +init() { + +# Get the blacklisted packages + blacklist=($(get_blacklist)) +# Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" + +# Sync the repos databases + get_repos + +# Traverse all repo-arch pairs + for _arch in ${ARCHARCHES[@]}; do + for _repo in ${ARCHREPOS[@]}; do + msg "Processing ${_repo}-${_arch}" + + repo_file=$(get_repo_file ${_repo} ${_arch}) + +# Remove blacklisted packages and count them +# TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages: $( + LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ + grep "\-> Removing" 2>/dev/null| wc -l)" + +# Get db contents + db=($(get_repo_content ${repo_file})) + + msg2 "Process clean db for syncing..." + +# Create a whitelist, add * wildcard to end +# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if +# we aren't syncing them + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + +# Sync excluding everything but whitelist +# We delete here for cleanup + rsync -vrtlH \ + --delete-after \ + --safe-links \ + --delay-updates \ + --max-delete=1000 \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + +# Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + +# Cleanup + unset db + done + done + + msg "Putting databases back in place" + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + cache/ \ + ${FTP_BASE}/ + + msg "Syncing package pool" +# Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + +# Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + + +# Cleanup + unset blacklist whitelists _arch _repo repo_file +} + +trap_exit() { + echo + error "$@" + exit 1 +} + + +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +init |