From 0af2e33b4e5c41bf88022fa1127cad1637369963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Thu, 20 Aug 2015 13:56:54 -0300 Subject: add armv7h architecture on OURARCHES --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 287f5b0..be502cb 100644 --- a/config +++ b/config @@ -43,7 +43,7 @@ LOCK_TIMEOUT=300 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) -OURARCHES=() +OURARCHES=(armv7h) ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -- cgit v1.2.3 From 3a2360afd7c530370501038fb7f4d6088b96f8c6 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Fri, 25 Sep 2015 14:43:15 -0500 Subject: Adding db-sync-arm untested --- config | 1 + db-sync-arm | 168 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ db-sync-arm.conf | 6 ++ 3 files changed, 175 insertions(+) create mode 100755 db-sync-arm create mode 100644 db-sync-arm.conf diff --git a/config b/config index be502cb..6899db4 100644 --- a/config +++ b/config @@ -4,6 +4,7 @@ FTP_BASE="/srv/repo/main" # Repos from Arch ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') +ARMREPOS=('core' 'extra' 'community') # Official Parabola repos OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') # User repos diff --git a/db-sync-arm b/db-sync-arm new file mode 100755 index 0000000..7391fbc --- /dev/null +++ b/db-sync-arm @@ -0,0 +1,168 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * make a tarball of files used for forensics + +# Run as `V=true db-sync` to get verbose output +VERBOSE=${V} +extra=() +${VERBOSE} && extra+=(-v) + +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + +# Returns contents of a repo +get_repos() { + # Exclude everything but db files + rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" +} + +get_repo_content() { + # Return all contents + bsdtar tf "${1}" | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + echo "${WORKDIR}/${2}/${1}/${1}" +} + +# Process the databases and get the libre packages +init() { + + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _arch in "${OURARCHES[@]}"; do + for _repo in "${ARMREPOS[@]}"; do + msg "Processing %s-%s" "${_repo}-${_arch}" + + db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} + files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this arch-repo" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this arch-repo" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content "${db_file}")) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" + + msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync "${extra[@]}" --no-motd -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + "${WORKDIR}/${_arch}/${_repo}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Cleanup + unset db + done + done + + date -u +%s > "${FTP_BASE}/lastsync" + + # Cleanup + unset blacklist whitelists _arch _repo repo_file +} + +trap_exit() { + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-sync-arm.conf" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" + +# Check variables presence +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do + test -z "${!var}" && fatal_error "Empty %s" "${var}" +done + +# From makepkg +set -E +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR + +init diff --git a/db-sync-arm.conf b/db-sync-arm.conf new file mode 100644 index 0000000..91e401a --- /dev/null +++ b/db-sync-arm.conf @@ -0,0 +1,6 @@ +#mirror="mirror.yandex.ru" +mirror="ftp.halifax.rwth-aachen.de" + +## mirrors without sources folder + +mirrorpath="archlinux-arm" -- cgit v1.2.3 From 2db40ba76f088c89b87f0c1b943e50116a9db735 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Fri, 25 Sep 2015 16:42:24 -0500 Subject: Note to mirror.yandex.ru --- db-sync-arm.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/db-sync-arm.conf b/db-sync-arm.conf index 91e401a..eaa170f 100644 --- a/db-sync-arm.conf +++ b/db-sync-arm.conf @@ -2,5 +2,6 @@ mirror="ftp.halifax.rwth-aachen.de" ## mirrors without sources folder +## use "archlinuxarm" instead "archlinux-arm" to mirror.yandex.ru mirrorpath="archlinux-arm" -- cgit v1.2.3 From ad836d45cc1e29b9671bad0fcfeba99514f746a8 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Fri, 25 Sep 2015 20:12:01 -0500 Subject: Adding sync to pool --- db-sync-arm | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/db-sync-arm b/db-sync-arm index 7391fbc..fa2589a 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -134,6 +134,32 @@ init() { done done + + msg "Syncing package pool" + + # Traverse all repo-arch pairs + local pkgpool + for _arch in "${OURARCHES[@]}"; do + for _repo in "${ARMREPOS[@]}"; do + if [ "${_repo}" = "community" ]; then + pkgpool="pool/community" + else + pkgpool="pool/packages" + fi + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ + --exclude="*" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${pkgpool}/" + done + done + date -u +%s > "${FTP_BASE}/lastsync" # Cleanup -- cgit v1.2.3 From 3bc03bbdce0ad13d6db6c2a35ea3a2e7333004b7 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Sat, 26 Sep 2015 10:02:23 -0500 Subject: Fix Symlink to pool --- db-sync-arm | 54 +++++++++++++++++++++--------------------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/db-sync-arm b/db-sync-arm index fa2589a..853fee4 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -69,6 +69,7 @@ init() { get_repos # Traverse all repo-arch pairs + local pkgpool for _arch in "${OURARCHES[@]}"; do for _repo in "${ARMREPOS[@]}"; do msg "Processing %s-%s" "${_repo}-${_arch}" @@ -108,19 +109,21 @@ init() { msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + # Set pkgpool to sync packages + if [ "${_repo}" = "community" ]; then + pkgpool="pool/community" + else + pkgpool="pool/packages" + fi + # Sync excluding everything but whitelist - # We delete here for cleanup rsync "${extra[@]}" --no-motd -rtlH \ - --delete-after \ - --delete-excluded \ --delay-updates \ + --safe-links \ --include-from="/tmp/${_repo}-${_arch}.whitelist" \ --exclude="*" \ "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + "${FTP_BASE}/${pkgpool}/" msg "Putting databases back in place" rsync "${extra[@]}" --no-motd -rtlH \ @@ -129,41 +132,26 @@ init() { "${WORKDIR}/${_arch}/${_repo}/" \ "${FTP_BASE}/${_repo}/os/${_arch}/" - # Cleanup - unset db - done - done + # Search packages in whitelist to create symlinks + sed -i 's/^/\^/' "/tmp/${_repo}-${_arch}.whitelist" + ls ${FTP_BASE}/${pkgpool}/ | grep -w -f "/tmp/${_repo}-${_arch}.whitelist" \ + > "/tmp/${_repo}-${_arch}.pkgwhitelist" - msg "Syncing package pool" + msg "Putting symlinks" + while read _pkgfile; do + ln -sf "../../../${pkgpool}/${_pkgfile}" "${FTP_BASE}/${_repo}/os/${_arch}" + done < "/tmp/${_repo}-${_arch}.pkgwhitelist" - # Traverse all repo-arch pairs - local pkgpool - for _arch in "${OURARCHES[@]}"; do - for _repo in "${ARMREPOS[@]}"; do - if [ "${_repo}" = "community" ]; then - pkgpool="pool/community" - else - pkgpool="pool/packages" - fi - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from="/tmp/${_repo}-${_arch}.whitelist" \ - --exclude="*" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${pkgpool}/" + # Cleanup + unset db done done date -u +%s > "${FTP_BASE}/lastsync" # Cleanup - unset blacklist whitelists _arch _repo repo_file + unset blacklist whitelists _arch _repo repo_file _pkgfile } trap_exit() { -- cgit v1.2.3 From 56694a1e73a5c91df075fa3307bdd40a9d1e520f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Sat, 26 Sep 2015 14:43:03 -0300 Subject: import changes from repo --- README-WORKING-ON | 10 ++++++++++ abslibre | 9 ++------- cron-jobs/db-update-mailer | 26 ++++++++++++++++++++++++++ cron-jobs/db-update-mailer~ | 28 ++++++++++++++++++++++++++++ db-sync | 4 +++- db-sync.conf | 4 ++-- 6 files changed, 71 insertions(+), 10 deletions(-) create mode 100644 README-WORKING-ON create mode 100644 cron-jobs/db-update-mailer create mode 100644 cron-jobs/db-update-mailer~ diff --git a/README-WORKING-ON b/README-WORKING-ON new file mode 100644 index 0000000..fdc0978 --- /dev/null +++ b/README-WORKING-ON @@ -0,0 +1,10 @@ +Message from aurelien. + +Working on #bug 5 https://labs.parabola.nu/issues/5 + +db-update~old is the untouched file + +db-update is the testing file + + +db-update-mailer is another (testing) one for the moment. in cron/ diff --git a/abslibre b/abslibre index 6b21d24..72171f1 100755 --- a/abslibre +++ b/abslibre @@ -52,13 +52,8 @@ function get_blacklist() { function sync_abs_libre() { # Clone ABSLibre git repo - if [ -d /tmp/abslibre/.git ]; then - pushd /tmp/abslibre >/dev/null 2>&1 - git pull - popd >/dev/null 2>&1 - else - git clone "$ABSGIT" /tmp/abslibre - fi + rm -rf /tmp/abslibre + git clone "$ABSGIT" /tmp/abslibre # Sync from ABS and then sync from ABSLibre printf ":: Syncing ABSLibre...\t" diff --git a/cron-jobs/db-update-mailer b/cron-jobs/db-update-mailer new file mode 100644 index 0000000..35ff204 --- /dev/null +++ b/cron-jobs/db-update-mailer @@ -0,0 +1,26 @@ +#!/bin/bash +# Dummy helper to send email to parabola-dev +# It does nothing if no output +# Aurélien DESBRIERES +# GPL v3 or later. +# testing version !!! + +LIST="maintenance@lists.parabolagnulinux.org" +FROM="maintenance@lists.parabolagnulinux.org" + +SUBJECT="Database Updated $(date +"%d-%m-%Y")" +if [ $db-update 1 ]; then + SUBJECT="$1 $(date +"%d-%m-%Y")" +fi + +stdin="$(cat)" +#echo used to strip whitespace for checking for actual data +if [ -n "$(echo $stdin)" ]; then + +echo "Subject: $SUBJECT +To: $LIST +From: $FROM + +$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" + +fi diff --git a/cron-jobs/db-update-mailer~ b/cron-jobs/db-update-mailer~ new file mode 100644 index 0000000..ca2e46b --- /dev/null +++ b/cron-jobs/db-update-mailer~ @@ -0,0 +1,28 @@ +#!/bin/bash +#Dummy helper to send email to arch-dev +# It does nothing if no output + +LIST="arch-dev-public@archlinux.org" +#LIST="aaronmgriffin@gmail.com" +FROM="repomaint@archlinux.org" + +SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" +if [ $# -ge 1 ]; then + SUBJECT="$1 $(date +"%d-%m-%Y")" +fi + +if [ $# -ge 2 ]; then + LIST="$2" +fi + +stdin="$(cat)" +#echo used to strip whitespace for checking for actual data +if [ -n "$(echo $stdin)" ]; then + +echo "Subject: $SUBJECT +To: $LIST +From: $FROM + +$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" + +fi diff --git a/db-sync b/db-sync index 2194fe6..facfae9 100755 --- a/db-sync +++ b/db-sync @@ -14,6 +14,8 @@ # TODO # * make a tarball of files used for forensics +set -e + # Run as `V=true db-sync` to get verbose output VERBOSE=${V} extra=() @@ -71,7 +73,7 @@ init() { # Traverse all repo-arch pairs for _repo in "${ARCHREPOS[@]}"; do for _arch in "${ARCHARCHES[@]}"; do - msg "Processing %s-%s" "${_repo}-${_arch}" + msg "Processing %s-%s" "${_repo}" "${_arch}" db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} diff --git a/db-sync.conf b/db-sync.conf index f7748c3..24fc44d 100644 --- a/db-sync.conf +++ b/db-sync.conf @@ -1,7 +1,7 @@ -#mirror="mirrors.kernel.org" -mirror="mirrors.niyawe.de" +mirror="mirrors.kernel.org" ## mirrors without sources folder +#mirror="mirrors.niyawe.de" #mirror="mirror.nl.leaseweb.net" #mirror="mirror.one.com" #mirror="mirror.us.leaseweb.net" -- cgit v1.2.3 From f16b5bfab292c9947dfe342b27f1c31cf6045728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Sat, 26 Sep 2015 14:45:07 -0300 Subject: add set command to db-sync-arm --- db-sync-arm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db-sync-arm b/db-sync-arm index 853fee4..2e58802 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -14,6 +14,8 @@ # TODO # * make a tarball of files used for forensics +set -e + # Run as `V=true db-sync` to get verbose output VERBOSE=${V} extra=() @@ -72,7 +74,7 @@ init() { local pkgpool for _arch in "${OURARCHES[@]}"; do for _repo in "${ARMREPOS[@]}"; do - msg "Processing %s-%s" "${_repo}-${_arch}" + msg "Processing %s-%s" "${_repo}" "${_arch}" db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} -- cgit v1.2.3 From 67061580ebd4dde6e94d281a60e1322dc6e9b587 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Sat, 26 Sep 2015 15:24:27 -0500 Subject: Fixing symlinks to db-syn-arm --- db-sync-arm | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/db-sync-arm b/db-sync-arm index 2e58802..08a0905 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -118,6 +118,8 @@ init() { pkgpool="pool/packages" fi + msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + # Sync excluding everything but whitelist rsync "${extra[@]}" --no-motd -rtlH \ --delay-updates \ @@ -134,10 +136,28 @@ init() { "${WORKDIR}/${_arch}/${_repo}/" \ "${FTP_BASE}/${_repo}/os/${_arch}/" + # Cleanup + unset db + done + done + + + msg "Generating symbolic links to pool" + + for _arch in "${OURARCHES[@]}"; do + for _repo in "${ARMREPOS[@]}"; do + # Set pkgpool to symlink + if [ "${_repo}" = "community" ]; then + pkgpool="pool/community" + else + pkgpool="pool/packages" + fi + # Search packages in whitelist to create symlinks sed -i 's/^/\^/' "/tmp/${_repo}-${_arch}.whitelist" - ls ${FTP_BASE}/${pkgpool}/ | grep -w -f "/tmp/${_repo}-${_arch}.whitelist" \ + ls "${FTP_BASE}/${pkgpool}/" | grep "${_arch}" \ + grep -w -f "/tmp/${_repo}-${_arch}.whitelist" \ > "/tmp/${_repo}-${_arch}.pkgwhitelist" msg "Putting symlinks" @@ -145,8 +165,6 @@ init() { ln -sf "../../../${pkgpool}/${_pkgfile}" "${FTP_BASE}/${_repo}/os/${_arch}" done < "/tmp/${_repo}-${_arch}.pkgwhitelist" - # Cleanup - unset db done done -- cgit v1.2.3 From 67cb9c3f9eb7a5c68dfd775366573a0eccfc0da0 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Sat, 26 Sep 2015 17:16:36 -0500 Subject: Fixing "any" packages to db-syn-arm --- db-sync-arm | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/db-sync-arm b/db-sync-arm index 08a0905..180869f 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -101,13 +101,13 @@ init() { msg2 "Process clean db for syncing..." - # Create a whitelist, add * wildcard to end + # Create a whitelist, add arch and * wildcard to end # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if # we aren't syncing them # IMPORTANT: the . in the sed command is needed because an empty # whitelist would consist of a single * allowing any package to # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" + printf '%s\n' "${db[@]}" | sed "s|.$|&-${_arch}*|g" > "/tmp/${_repo}-${_arch}.whitelist" msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" @@ -156,15 +156,22 @@ init() { # Search packages in whitelist to create symlinks sed -i 's/^/\^/' "/tmp/${_repo}-${_arch}.whitelist" - ls "${FTP_BASE}/${pkgpool}/" | grep "${_arch}" \ + ls "${FTP_BASE}/${pkgpool}/" \ grep -w -f "/tmp/${_repo}-${_arch}.whitelist" \ > "/tmp/${_repo}-${_arch}.pkgwhitelist" - msg "Putting symlinks" + msg "Putting symlinks in ${_repo}/os/${_arch}" while read _pkgfile; do ln -sf "../../../${pkgpool}/${_pkgfile}" "${FTP_BASE}/${_repo}/os/${_arch}" done < "/tmp/${_repo}-${_arch}.pkgwhitelist" + # Putting symlinks to "any" packages + ls "${FTP_BASE}/${_repo}/os/${_arch}/" | grep '\-any\.' \ + > "/tmp/${_repo}-${_arch}-any.pkgwhitelist" + + while read _pkgfile; do + ln -sf "../../../${pkgpool}/${_pkgfile}" "${FTP_BASE}/${_repo}/os/${_arch}" + done < "/tmp/${_repo}-${_arch}-any.pkgwhitelist" done done -- cgit v1.2.3 From 4c88b9e238aac2c242a9bd7f0a75f95e88bdff80 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Sat, 26 Sep 2015 17:18:42 -0500 Subject: Minor fix to db-sync-arm --- db-sync-arm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-sync-arm b/db-sync-arm index 180869f..fc6a080 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -166,7 +166,7 @@ init() { done < "/tmp/${_repo}-${_arch}.pkgwhitelist" # Putting symlinks to "any" packages - ls "${FTP_BASE}/${_repo}/os/${_arch}/" | grep '\-any\.' \ + ls "${FTP_BASE}/${_repo}/os/i686/" | grep '\-any\.' \ > "/tmp/${_repo}-${_arch}-any.pkgwhitelist" while read _pkgfile; do -- cgit v1.2.3 From f46dad7eb27fa71836dbda144e256454aa63784f Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Sat, 26 Sep 2015 17:39:56 -0500 Subject: Minor fix to db-sync-arm --- db-sync-arm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/db-sync-arm b/db-sync-arm index fc6a080..155731a 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -156,8 +156,7 @@ init() { # Search packages in whitelist to create symlinks sed -i 's/^/\^/' "/tmp/${_repo}-${_arch}.whitelist" - ls "${FTP_BASE}/${pkgpool}/" \ - grep -w -f "/tmp/${_repo}-${_arch}.whitelist" \ + ls "${FTP_BASE}/${pkgpool}/" | grep -w -f "/tmp/${_repo}-${_arch}.whitelist" \ > "/tmp/${_repo}-${_arch}.pkgwhitelist" msg "Putting symlinks in ${_repo}/os/${_arch}" -- cgit v1.2.3 From 59d8594bfd89cdf37a4cae7fa893da4a6bc980e4 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Sat, 26 Sep 2015 18:56:02 -0500 Subject: Fixing symlinks to db-sync-arm --- db-sync-arm | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/db-sync-arm b/db-sync-arm index 155731a..3b1e3df 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -102,8 +102,6 @@ init() { msg2 "Process clean db for syncing..." # Create a whitelist, add arch and * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them # IMPORTANT: the . in the sed command is needed because an empty # whitelist would consist of a single * allowing any package to # pass through @@ -154,23 +152,41 @@ init() { fi # Search packages in whitelist to create symlinks - sed -i 's/^/\^/' "/tmp/${_repo}-${_arch}.whitelist" - - ls "${FTP_BASE}/${pkgpool}/" | grep -w -f "/tmp/${_repo}-${_arch}.whitelist" \ - > "/tmp/${_repo}-${_arch}.pkgwhitelist" + sed -i 's/\*/\.pkg\.tar\.xz/' "/tmp/${_repo}-${_arch}.whitelist" msg "Putting symlinks in ${_repo}/os/${_arch}" + while read _pkgfile; do - ln -sf "../../../${pkgpool}/${_pkgfile}" "${FTP_BASE}/${_repo}/os/${_arch}" - done < "/tmp/${_repo}-${_arch}.pkgwhitelist" + # Symlink to package + if [ -f "${FTP_BASE}/${pkgpool}/${_pkgfile}" ]; then + ln -sf "../../../${pkgpool}/${_pkgfile}" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" + fi + + # Symlink to signature + if [ -f "${FTP_BASE}/${pkgpool}/${_pkgfile}.sig" ]; then + ln -sf "../../../${pkgpool}/${_pkgfile}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" + fi + done < "/tmp/${_repo}-${_arch}.whitelist" # Putting symlinks to "any" packages ls "${FTP_BASE}/${_repo}/os/i686/" | grep '\-any\.' \ - > "/tmp/${_repo}-${_arch}-any.pkgwhitelist" + > "/tmp/${_repo}-any.pkgwhitelist" while read _pkgfile; do - ln -sf "../../../${pkgpool}/${_pkgfile}" "${FTP_BASE}/${_repo}/os/${_arch}" - done < "/tmp/${_repo}-${_arch}-any.pkgwhitelist" + # Symlink to package + if [ -f "${FTP_BASE}/${pkgpool}/${_pkgfile}" ]; then + ln -sf "../../../${pkgpool}/${_pkgfile}" \ + "${FTP_BASE}/${_repo}/os/${_arch}" + fi + + # Symlink to signature + if [ -f "${FTP_BASE}/${pkgpool}/${_pkgfile}.sig" ]; then + ln -sf "../../../${pkgpool}/${_pkgfile}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" + fi + done < "/tmp/${_repo}-any.pkgwhitelist" done done -- cgit v1.2.3 From 1896e9b9e11cdb70094d79ba369ca3f3b3d0d969 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Sat, 26 Sep 2015 19:05:45 -0500 Subject: Minor fix to db-sync-arm --- db-sync-arm | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/db-sync-arm b/db-sync-arm index 3b1e3df..0c28a37 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -175,16 +175,10 @@ init() { > "/tmp/${_repo}-any.pkgwhitelist" while read _pkgfile; do - # Symlink to package + # Symlink to package and signature if [ -f "${FTP_BASE}/${pkgpool}/${_pkgfile}" ]; then ln -sf "../../../${pkgpool}/${_pkgfile}" \ - "${FTP_BASE}/${_repo}/os/${_arch}" - fi - - # Symlink to signature - if [ -f "${FTP_BASE}/${pkgpool}/${_pkgfile}.sig" ]; then - ln -sf "../../../${pkgpool}/${_pkgfile}.sig" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" fi done < "/tmp/${_repo}-any.pkgwhitelist" done -- cgit v1.2.3 From 5c153b813718f34b12a2bdc3b226ce573365a8e2 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Mon, 28 Sep 2015 01:45:32 -0500 Subject: Modify pool to arm packages --- config | 1 + db-sync-arm | 46 +++++++++++++++++++--------------------------- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/config b/config index 6899db4..8171ae1 100644 --- a/config +++ b/config @@ -14,6 +14,7 @@ PROJREPOS=('nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") PKGPOOL='pool/parabola' +PKGPOOLARM='pool/arch_gnu+linux_arm' SRCPOOL='sources/parabola' # Directories where packages are shared between repos diff --git a/db-sync-arm b/db-sync-arm index 0c28a37..8bc55ce 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -71,7 +71,6 @@ init() { get_repos # Traverse all repo-arch pairs - local pkgpool for _arch in "${OURARCHES[@]}"; do for _repo in "${ARMREPOS[@]}"; do msg "Processing %s-%s" "${_repo}" "${_arch}" @@ -101,21 +100,16 @@ init() { msg2 "Process clean db for syncing..." - # Create a whitelist, add arch and * wildcard to end + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them # IMPORTANT: the . in the sed command is needed because an empty # whitelist would consist of a single * allowing any package to # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&-${_arch}*|g" > "/tmp/${_repo}-${_arch}.whitelist" + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - # Set pkgpool to sync packages - if [ "${_repo}" = "community" ]; then - pkgpool="pool/community" - else - pkgpool="pool/packages" - fi - msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" # Sync excluding everything but whitelist @@ -125,7 +119,7 @@ init() { --include-from="/tmp/${_repo}-${_arch}.whitelist" \ --exclude="*" \ "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ - "${FTP_BASE}/${pkgpool}/" + "${FTP_BASE}/${PKGPOOLARM}/" msg "Putting databases back in place" rsync "${extra[@]}" --no-motd -rtlH \ @@ -144,40 +138,38 @@ init() { for _arch in "${OURARCHES[@]}"; do for _repo in "${ARMREPOS[@]}"; do - # Set pkgpool to symlink - if [ "${_repo}" = "community" ]; then - pkgpool="pool/community" - else - pkgpool="pool/packages" - fi - - # Search packages in whitelist to create symlinks - sed -i 's/\*/\.pkg\.tar\.xz/' "/tmp/${_repo}-${_arch}.whitelist" + # Modify whitelist to search packages and create symlinks + sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist" msg "Putting symlinks in ${_repo}/os/${_arch}" while read _pkgfile; do # Symlink to package - if [ -f "${FTP_BASE}/${pkgpool}/${_pkgfile}" ]; then - ln -sf "../../../${pkgpool}/${_pkgfile}" \ + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then + ln -sf "../../../${PKGPOOLARM}/${_pkgfile}" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" fi # Symlink to signature - if [ -f "${FTP_BASE}/${pkgpool}/${_pkgfile}.sig" ]; then - ln -sf "../../../${pkgpool}/${_pkgfile}.sig" \ + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then + ln -sf "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" fi done < "/tmp/${_repo}-${_arch}.whitelist" + # Modify whitelist to search "any" packages and create symlinks + sed -i "s/^/\^/g" "/tmp/${_repo}-${_arch}.whitelist" + sed -i "s/-i686./-any./g" "/tmp/${_repo}-${_arch}.whitelist" + # Putting symlinks to "any" packages - ls "${FTP_BASE}/${_repo}/os/i686/" | grep '\-any\.' \ + ls "${FTP_BASE}/${PKGPOOLARM}/" | grep '\-any\.' \ + | grep -w -f "/tmp/${_repo}-${_arch}.whitelist" \ > "/tmp/${_repo}-any.pkgwhitelist" while read _pkgfile; do # Symlink to package and signature - if [ -f "${FTP_BASE}/${pkgpool}/${_pkgfile}" ]; then - ln -sf "../../../${pkgpool}/${_pkgfile}" \ + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then + ln -sf "../../../${PKGPOOLARM}/${_pkgfile}" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" fi done < "/tmp/${_repo}-any.pkgwhitelist" -- cgit v1.2.3 From e62bfa20ada102246056ea12405049dbc6a825d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Mon, 28 Sep 2015 04:32:46 -0300 Subject: update *POOL*, *CLEANUP_DESTDIR and WEB_DIR --- config | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/config b/config index 8171ae1..8a38b71 100644 --- a/config +++ b/config @@ -13,26 +13,26 @@ USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jor PROJREPOS=('nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") -PKGPOOL='pool/parabola' +PKGPOOL='pool/parabola_gnu+linux' PKGPOOLARM='pool/arch_gnu+linux_arm' -SRCPOOL='sources/parabola' +SRCPOOL='src/parabola_gnu+linux' # Directories where packages are shared between repos # *relative to FTP_BASE* -ARCHPKGPOOLS=(pool/{packages,community}) -OURPKGPOOLS=(pool/parabola) +ARCHPKGPOOLS=(pool/arch_gnu+linux_x86{,_community}) +OURPKGPOOLS=(pool/parabola_gnu+linux) PKGPOOLS=(${OURPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) # Directories where sources are stored -ARCHSRCPOOLS=(sources/{packages,community}) -OURPKGPOOLS=(sources/parabola) +ARCHSRCPOOLS=(src/arch_gnu+linux_x86{,_community}) +OURPKGPOOLS=(src/parabola_gnu+linux) SRCPOOLS=(${OURSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) -CLEANUP_DESTDIR="$FTP_BASE/old/packages" +CLEANUP_DESTDIR="$FTP_BASE/old/pkg" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/src" SOURCE_CLEANUP_DRYRUN=true # Time in days to keep moved sourcepackages SOURCE_CLEANUP_KEEP=30 @@ -56,4 +56,4 @@ MAKEPKGCONF="~/.makepkg.conf" BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" # parabolaweb root -WEB_DIR=/srv/http/parabolagnulinux.org/web +WEB_DIR=/srv/http/parabola.nu/web -- cgit v1.2.3 From c1c48223db40547b6d62f699494922c8aee78941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Mon, 28 Sep 2015 04:56:48 -0300 Subject: ARCH*POOLS: change back to legacy directories --- config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config b/config index 8a38b71..d82889e 100644 --- a/config +++ b/config @@ -19,11 +19,11 @@ SRCPOOL='src/parabola_gnu+linux' # Directories where packages are shared between repos # *relative to FTP_BASE* -ARCHPKGPOOLS=(pool/arch_gnu+linux_x86{,_community}) +ARCHPKGPOOLS=(pool/{packages,community}) OURPKGPOOLS=(pool/parabola_gnu+linux) PKGPOOLS=(${OURPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) # Directories where sources are stored -ARCHSRCPOOLS=(src/arch_gnu+linux_x86{,_community}) +ARCHSRCPOOLS=(sources/{packages,community}) OURPKGPOOLS=(src/parabola_gnu+linux) SRCPOOLS=(${OURSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) -- cgit v1.2.3 From b38f808a7cd46902ff4254a1a4c795abbfdf3dfb Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Mon, 28 Sep 2015 09:55:26 -0500 Subject: Minor fix to db-sync-arm --- db-sync-arm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-sync-arm b/db-sync-arm index 8bc55ce..9653d00 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -159,7 +159,7 @@ init() { # Modify whitelist to search "any" packages and create symlinks sed -i "s/^/\^/g" "/tmp/${_repo}-${_arch}.whitelist" - sed -i "s/-i686./-any./g" "/tmp/${_repo}-${_arch}.whitelist" + sed -i "s/-${_arch}./-any./g" "/tmp/${_repo}-${_arch}.whitelist" # Putting symlinks to "any" packages ls "${FTP_BASE}/${PKGPOOLARM}/" | grep '\-any\.' \ -- cgit v1.2.3 From 0ebafbe6539b45b706e8509918f72bcf6b3bed0e Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Mon, 28 Sep 2015 11:53:06 -0500 Subject: Fixing symlinks to db-sync-arm --- db-sync-arm | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/db-sync-arm b/db-sync-arm index 9653d00..0f82ea9 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -139,7 +139,8 @@ init() { for _arch in "${OURARCHES[@]}"; do for _repo in "${ARMREPOS[@]}"; do # Modify whitelist to search packages and create symlinks - sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist" + sed -i "s/*/-${_arch}.pkg.tar.xz/g" > "/tmp/${_repo}-${_arch}.whitelist" + cp "/tmp/${_repo}-${_arch}.whitelist" "/tmp/${_repo}-any.whitelist" msg "Putting symlinks in ${_repo}/os/${_arch}" @@ -148,31 +149,40 @@ init() { if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then ln -sf "../../../${PKGPOOLARM}/${_pkgfile}" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" + + # Delete entry to "any" whitelist + sed -i "s/${_pkgfile}//g" "/tmp/${_repo}-any.whitelist" fi # Symlink to signature if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then ln -sf "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" + + # Delete entry to "any" whitelist + sed -i "s/${_pkgfile}//g" "/tmp/${_repo}-any.whitelist" fi done < "/tmp/${_repo}-${_arch}.whitelist" - # Modify whitelist to search "any" packages and create symlinks - sed -i "s/^/\^/g" "/tmp/${_repo}-${_arch}.whitelist" - sed -i "s/-${_arch}./-any./g" "/tmp/${_repo}-${_arch}.whitelist" + msg "Putting symlinks to packages with arch 'any' in ${_repo}/os/${_arch}" - # Putting symlinks to "any" packages - ls "${FTP_BASE}/${PKGPOOLARM}/" | grep '\-any\.' \ - | grep -w -f "/tmp/${_repo}-${_arch}.whitelist" \ - > "/tmp/${_repo}-any.pkgwhitelist" + # Modify whitelist to search "any" packages and create symlinks + sed -i "s/-${_arch}./-any./g" "/tmp/${_repo}-any.whitelist" + sed -i '/^$/d' "/tmp/${_repo}-any.whitelist" while read _pkgfile; do - # Symlink to package and signature + # Symlink to package if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then ln -sf "../../../${PKGPOOLARM}/${_pkgfile}" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" fi - done < "/tmp/${_repo}-any.pkgwhitelist" + + # Symlink to signature + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then + ln -sf "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" + fi + done < "/tmp/${_repo}-any.whitelist" done done -- cgit v1.2.3 From af4aaf7e469738df873d7f8bf89399f21e94db87 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Mon, 28 Sep 2015 11:57:23 -0500 Subject: Minor fix to db-sync-arm --- db-sync-arm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-sync-arm b/db-sync-arm index 0f82ea9..90dc293 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -139,7 +139,7 @@ init() { for _arch in "${OURARCHES[@]}"; do for _repo in "${ARMREPOS[@]}"; do # Modify whitelist to search packages and create symlinks - sed -i "s/*/-${_arch}.pkg.tar.xz/g" > "/tmp/${_repo}-${_arch}.whitelist" + sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist" cp "/tmp/${_repo}-${_arch}.whitelist" "/tmp/${_repo}-any.whitelist" msg "Putting symlinks in ${_repo}/os/${_arch}" -- cgit v1.2.3 From 70e08c0826ea4736e767f835da15c1daf8c0f81b Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Tue, 6 Oct 2015 00:21:19 -0500 Subject: Adding verbose mode to symlinks --- db-sync-arm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-sync-arm b/db-sync-arm index 90dc293..ff88c93 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -147,7 +147,7 @@ init() { while read _pkgfile; do # Symlink to package if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then - ln -sf "../../../${PKGPOOLARM}/${_pkgfile}" \ + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" # Delete entry to "any" whitelist @@ -156,7 +156,7 @@ init() { # Symlink to signature if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then - ln -sf "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" # Delete entry to "any" whitelist @@ -173,13 +173,13 @@ init() { while read _pkgfile; do # Symlink to package if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then - ln -sf "../../../${PKGPOOLARM}/${_pkgfile}" \ + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" fi # Symlink to signature if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then - ln -sf "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" fi done < "/tmp/${_repo}-any.whitelist" -- cgit v1.2.3 From fc0e5942ab8a3340bd80f4a788f26bb261a078f6 Mon Sep 17 00:00:00 2001 From: Omar Vega Ramos Date: Tue, 6 Oct 2015 01:36:12 -0500 Subject: Fixing symlinks to db-sync-arm --- db-sync-arm | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/db-sync-arm b/db-sync-arm index ff88c93..952b546 100755 --- a/db-sync-arm +++ b/db-sync-arm @@ -140,7 +140,6 @@ init() { for _repo in "${ARMREPOS[@]}"; do # Modify whitelist to search packages and create symlinks sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist" - cp "/tmp/${_repo}-${_arch}.whitelist" "/tmp/${_repo}-any.whitelist" msg "Putting symlinks in ${_repo}/os/${_arch}" @@ -149,40 +148,20 @@ init() { if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" - - # Delete entry to "any" whitelist - sed -i "s/${_pkgfile}//g" "/tmp/${_repo}-any.whitelist" + elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}" fi # Symlink to signature if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" - - # Delete entry to "any" whitelist - sed -i "s/${_pkgfile}//g" "/tmp/${_repo}-any.whitelist" + elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig" fi done < "/tmp/${_repo}-${_arch}.whitelist" - - msg "Putting symlinks to packages with arch 'any' in ${_repo}/os/${_arch}" - - # Modify whitelist to search "any" packages and create symlinks - sed -i "s/-${_arch}./-any./g" "/tmp/${_repo}-any.whitelist" - sed -i '/^$/d' "/tmp/${_repo}-any.whitelist" - - while read _pkgfile; do - # Symlink to package - if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" - fi - - # Symlink to signature - if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" - fi - done < "/tmp/${_repo}-any.whitelist" done done -- cgit v1.2.3 From 4822b418896d2c5d91b50f00476f39f0441f5b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 14 Oct 2015 12:42:11 -0300 Subject: add ARMPKGPOOLS and ARMSRCPOOLS to config --- config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config b/config index d82889e..f41e6f6 100644 --- a/config +++ b/config @@ -20,12 +20,14 @@ SRCPOOL='src/parabola_gnu+linux' # Directories where packages are shared between repos # *relative to FTP_BASE* ARCHPKGPOOLS=(pool/{packages,community}) +ARMPKGPOOLS=(pool/arch_gnu+linux_arm) OURPKGPOOLS=(pool/parabola_gnu+linux) -PKGPOOLS=(${OURPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) +PKGPOOLS=(${OURPKGPOOLS[@]} ${ARMPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) # Directories where sources are stored ARCHSRCPOOLS=(sources/{packages,community}) +ARMSRCPOOLS=(src/arch_gnu+linux_arm) OURPKGPOOLS=(src/parabola_gnu+linux) -SRCPOOLS=(${OURSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) +SRCPOOLS=(${OURSRCPOOLS[@]} ${ARMSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) CLEANUP_DESTDIR="$FTP_BASE/old/pkg" CLEANUP_DRYRUN=false -- cgit v1.2.3 From 86b3d7c0cdff042fd33e7705e6ef756415b86bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 24 Nov 2015 10:28:21 -0300 Subject: prevent duplicate repos from locking themselves --- db-update | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index 4830791..a1552c0 100755 --- a/db-update +++ b/db-update @@ -9,7 +9,7 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)) +staging_repos=($(find "${STAGING}" -mindepth 1 -maxdepth 3 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)) if [ $? -ge 1 ]; then die "Could not read %s" "${STAGING}" fi @@ -20,6 +20,7 @@ for staging_repo in "${staging_repos[@]##*/}"; do repos+=("${staging_repo}") fi done +repos=($(echo "${repos[@]}" | tr " " "\n" | sort -u)) # TODO: this might lock too much (architectures) for repo in "${repos[@]}"; do -- cgit v1.2.3 From 8daed438c73d0354cf45ffbd304489ee26b5d9a2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 01:12:04 -0400 Subject: remove editor backup file --- cron-jobs/db-update-mailer~ | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 cron-jobs/db-update-mailer~ diff --git a/cron-jobs/db-update-mailer~ b/cron-jobs/db-update-mailer~ deleted file mode 100644 index ca2e46b..0000000 --- a/cron-jobs/db-update-mailer~ +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -#Dummy helper to send email to arch-dev -# It does nothing if no output - -LIST="arch-dev-public@archlinux.org" -#LIST="aaronmgriffin@gmail.com" -FROM="repomaint@archlinux.org" - -SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" -if [ $# -ge 1 ]; then - SUBJECT="$1 $(date +"%d-%m-%Y")" -fi - -if [ $# -ge 2 ]; then - LIST="$2" -fi - -stdin="$(cat)" -#echo used to strip whitespace for checking for actual data -if [ -n "$(echo $stdin)" ]; then - -echo "Subject: $SUBJECT -To: $LIST -From: $FROM - -$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" - -fi -- cgit v1.2.3 From 6ad57ac9f185dcfd4e02174ffdaff9bca1b0516c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 01:18:09 -0400 Subject: merge README-WORKING-ON and TODO into HACKING.org --- HACKING.org | 22 ++++++++++++++++++++++ README-WORKING-ON | 10 ---------- TODO | 10 ---------- 3 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 HACKING.org delete mode 100644 README-WORKING-ON delete mode 100644 TODO diff --git a/HACKING.org b/HACKING.org new file mode 100644 index 0000000..90db1a9 --- /dev/null +++ b/HACKING.org @@ -0,0 +1,22 @@ +* README-WORKING-ON.txt +Message from aurelien. + +Working on #bug 5 https://labs.parabola.nu/issues/5 + +db-update~old is the untouched file + +db-update is the testing file + + +db-update-mailer is another (testing) one for the moment. in cron/ +* TODO.txt +** Test Suite for clean_repo.py + + - Review all repo + - Remove all blacklisted packages + - Get pending list right + - Extract licenses all right + +** Fix db-move + + - Make it use abslibre diff --git a/README-WORKING-ON b/README-WORKING-ON deleted file mode 100644 index fdc0978..0000000 --- a/README-WORKING-ON +++ /dev/null @@ -1,10 +0,0 @@ -Message from aurelien. - -Working on #bug 5 https://labs.parabola.nu/issues/5 - -db-update~old is the untouched file - -db-update is the testing file - - -db-update-mailer is another (testing) one for the moment. in cron/ diff --git a/TODO b/TODO deleted file mode 100644 index 9dd4b52..0000000 --- a/TODO +++ /dev/null @@ -1,10 +0,0 @@ -* Test Suite for clean_repo.py - - - Review all repo - - Remove all blacklisted packages - - Get pending list right - - Extract licenses all right - -* Fix db-move - - - Make it use abslibre -- cgit v1.2.3 From ff2983c207e6ab2a9381e20312355c42307187f6 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 01:56:43 -0400 Subject: Give things more consistent names --- abslibre | 124 ------------------------ any-to-ours | 71 -------------- check-package-libraries.py | 193 ------------------------------------ createrepos | 8 -- cron-jobs/db-cleanup | 69 +++++++++++++ db-check-nonfree-in-db | 28 ++++++ db-check-package-libraries | 193 ++++++++++++++++++++++++++++++++++++ db-check-unsigned-packages | 38 ++++++++ db-check-unsigned-packages.py | 96 ++++++++++++++++++ db-cleanup | 69 ------------- db-import-any-to-ours | 71 ++++++++++++++ db-import-pkg-archlinux | 210 ++++++++++++++++++++++++++++++++++++++++ db-import-pkg-archlinux.conf | 11 +++ db-import-pkg-archlinuxarm | 199 +++++++++++++++++++++++++++++++++++++ db-import-pkg-archlinuxarm.conf | 7 ++ db-import-src-archlinux | 124 ++++++++++++++++++++++++ db-init | 8 ++ db-libremessages | 0 db-list-unsigned-packages | 38 -------- db-list-unsigned-packages.py | 96 ------------------ db-sync | 210 ---------------------------------------- db-sync-arm | 199 ------------------------------------- db-sync-arm.conf | 7 -- db-sync.conf | 11 --- list_nonfree_in_db.py | 28 ------ mkrepo | 15 --- 26 files changed, 1054 insertions(+), 1069 deletions(-) delete mode 100755 abslibre delete mode 100755 any-to-ours delete mode 100755 check-package-libraries.py delete mode 100755 createrepos create mode 100755 cron-jobs/db-cleanup create mode 100755 db-check-nonfree-in-db create mode 100755 db-check-package-libraries create mode 100755 db-check-unsigned-packages create mode 100755 db-check-unsigned-packages.py delete mode 100755 db-cleanup create mode 100755 db-import-any-to-ours create mode 100755 db-import-pkg-archlinux create mode 100644 db-import-pkg-archlinux.conf create mode 100755 db-import-pkg-archlinuxarm create mode 100644 db-import-pkg-archlinuxarm.conf create mode 100755 db-import-src-archlinux create mode 100755 db-init mode change 100755 => 100644 db-libremessages delete mode 100755 db-list-unsigned-packages delete mode 100755 db-list-unsigned-packages.py delete mode 100755 db-sync delete mode 100755 db-sync-arm delete mode 100644 db-sync-arm.conf delete mode 100644 db-sync.conf delete mode 100755 list_nonfree_in_db.py delete mode 100755 mkrepo diff --git a/abslibre b/abslibre deleted file mode 100755 index 72171f1..0000000 --- a/abslibre +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -set -e - -FTP_BASE=/srv/repo/main -ABSLIBRE=/srv/abslibre -ABSGIT=/srv/git/abslibre/abslibre.git -# Remote -# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git -BLACKLIST=/home/repo/blacklist/blacklist.txt -SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' -BLFILE=/tmp/blacklist.txt - -# Variables from abs.conf -ABSROOT="/srv/abs/" -# DON'T CHANGE. WE NEED IT FOR ABSLIBRE -SYNCSERVER="rsync.archlinux.org" -ARCH="i686" -MIRRORLIST="/etc/pacman.d/mirrorlist" -REPOS=(core extra community testing community-testing !staging !community-staging) - -# Steps -# * Sync abs -# * Download blacklist.txt -# * Sync abslibre from abs excluding from blacklist -# * Create repo.abs.tar.gz tarballs - -function sync_abs() { - for ARCH in any i686 x86_64; do - rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? - done - - # fix some permissions - find "${ABSROOT}" -type d -print0 | xargs -0 chmod 755 - find "${ABSROOT}" -type f -print0 | xargs -0 chmod 644 -} - -function get_blacklist() { - printf ":: Updating blacklist...\t" - cat "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ - sed "s/^/**\//" > ${BLFILE} || { - printf "[FAILED]\n" - return 1 - } - - # Prevent using an empty blacklist - [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 - - printf "[OK]\n" -} - -function sync_abs_libre() { - - # Clone ABSLibre git repo - rm -rf /tmp/abslibre - git clone "$ABSGIT" /tmp/abslibre - - # Sync from ABS and then sync from ABSLibre - printf ":: Syncing ABSLibre...\t" - (rsync ${SYNCARGS} --delete-excluded \ - --exclude-from=${BLFILE} \ - ${ABSROOT} \ - ${ABSLIBRE} \ - && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { - printf "[FAILED]\n" - return 1 - } - - # fix some permissions - find "${ABSLIBRE}" -type d -print0 | xargs -0 chmod 755 - find "${ABSLIBRE}" -type f -print0 | xargs -0 chmod 644 - - printf "[OK]\n" -} - -# This part is very hacky and particular to the current setup :P -sync_pre_mips64el() { - pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null - - sudo -u fauno sh -c " - rsync ${SYNCARGS} \ - --exclude=.git* \ - --exclude=community-staging \ - --exclude=community-testing \ - --exclude=gnome-unstable \ - --exclude=kde-unstable \ - --exclude=multilib \ - --exclude=multilib-testing \ - --exclude=multilib-staging \ - --exclude=staging \ - --exclude=testing \ - ${ABSLIBRE}/x86_64/ \ - /home/fauno/Repos/abslibre-pre-mips64el/ && - git add . && - git commit -m \"$(date)\" -a - git push origin master - git gc - " -} - -# Create .abs.tar.gz tarballs -create_tarballs() { - for repo in ${ABSLIBRE}/{i686,x86_64}/*; do - baserepo=${repo##*/} - arch=$(basename $(dirname $repo)) - - # Remove the old one - mkdir -p $FTP_BASE/$baserepo/os/$arch/ - rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz - # Create a new one joining arch and any - # Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ - -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ - $repo/* ${ABSLIBRE}/any/${baserepo}/* - - done -} - -sync_abs -get_blacklist -sync_abs_libre -#sync_pre_mips64el -create_tarballs diff --git a/any-to-ours b/any-to-ours deleted file mode 100755 index a901d54..0000000 --- a/any-to-ours +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -# Releases 'any' packages from Arch arches to ours - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -# The architecture to compare with -BASEARCH='x86_64' - -# Traverse all Arch repos -for _repo in "${ARCHREPOS[@]}"; do - msg "Processing %s..." "${_repo}" - - # Find 'any' packages - # This is hardcoded but it could release other arches... - PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - -iname '*-any.pkg.tar.?z' \ - -printf "%f ")) - - if [ ${#PKGS[@]} -eq 0 ]; then - msg2 "No '%s' packages here" any - continue - fi - - for _arch in "${OURARCHES[@]}"; do - msg2 "Syncing %s..." "${_arch}" - - # Sync 'any' only and extract the synced packages - SYNCED=($( - rsync -av \ - --include='*-any.pkg.tar.?z' \ - --include='*-any.pkg.tar.?z.sig' \ - --exclude='*' \ - "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \ - grep 'any\.pkg\.tar\..z$' | \ - cut -d ' ' -f 1 )) - - if [ ${#SYNCED[@]} -eq 0 ]; then - msg2 "Already synced (or error happened)" - continue - fi - - msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" - - msg2 "Adding to db..." - - pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null - - # Add the packages to the db - repo-add "${_repo}${DBEXT}" "${SYNCED[@]}" - - popd >/dev/null - - # Avoid mixups - unset SYNCED PKGS - done -done diff --git a/check-package-libraries.py b/check-package-libraries.py deleted file mode 100755 index bc2349b..0000000 --- a/check-package-libraries.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Check which libraries are provided or required by a package, store -this in a database, update and list broken packages. - -Dependencies: - -- Python 3.2 or later with SQLite 3 support - -- ``bsdtar`` - -- ``readelf`` -""" - - -import os.path -import re -import sqlite3 -import subprocess -import tempfile - - -#: Regexp matching an interesting dynamic entry. -_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" - "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") - - -def make_db(path): - """Make a new, empty, library database at *path*.""" - con = sqlite3.connect(path) - con.executescript(""" -create table provided( - library varchar not null, - package varchar not null -); -create table used( - library varchar not null, - package varchar not null -); -""") - con.close() - - -def begin(database): - """Connect to *database* and start a transaction.""" - con = sqlite3.connect(database) - con.execute("begin exclusive") - return con - - -def add_provided(con, package, libraries): - """Write that *package* provides *libraries*.""" - for library in libraries: - con.execute("insert into provided (package, library) values (?,?)", - (package, library)) - - -def add_used(con, package, libraries): - """Write that *package* uses *libraries*.""" - for library in libraries: - con.execute("insert into used (package, library) values (?,?)", - (package, library)) - - -def remove_package(con, package): - """Remove all entries for a package.""" - con.execute("delete from provided where package=?", (package,)) - con.execute("delete from used where package=?", (package,)) - - -def add_package(con, package): - """Add entries from a named *package*.""" - # Extract to a temporary directory. This could be done more - # efficiently, since there is no need to store more than one file - # at once. - with tempfile.TemporaryDirectory() as temp: - tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) - tar.communicate() - with open(os.path.join(temp, ".PKGINFO")) as pkginfo: - for line in pkginfo: - if line.startswith("pkgname ="): - pkgname = line[len("pkgname ="):].strip() - break - # Don't list previously removed libraries. - remove_package(con, pkgname) - provided = set() - used = set() - # Search for ELFs. - for dirname, dirnames, filenames in os.walk(temp): - assert dirnames is not None # unused, avoid pylint warning - for file_name in filenames: - path = os.path.join(dirname, file_name) - with open(path, "rb") as file_object: - if file_object.read(4) != b"\177ELF": - continue - readelf = subprocess.Popen(("readelf", "-d", path), - stdout=subprocess.PIPE) - for line in readelf.communicate()[0].split(b"\n"): - match = _DYNAMIC.match(line.decode("ascii")) - if match: - if match.group(1) == "SONAME": - provided.add(match.group(2)) - elif match.group(1) == "NEEDED": - used.add(match.group(2)) - else: - raise AssertionError("unknown entry type " - + match.group(1)) - add_provided(con, pkgname, provided) - add_used(con, pkgname, used) - - -def init(arguments): - """Initialize.""" - make_db(arguments.database) - - -def add(arguments): - """Add packages.""" - con = begin(arguments.database) - for package in arguments.packages: - add_package(con, package) - con.commit() - con.close() - - -def remove(arguments): - """Remove packages.""" - con = begin(arguments.database) - for package in arguments.packages: - remove_package(con, package) - con.commit() - con.close() - - -def check(arguments): - """List broken packages.""" - con = begin(arguments.database) - available = set(row[0] for row - in con.execute("select library from provided")) - for package, library in con.execute("select package, library from used"): - if library not in available: - print(package, "needs", library) - con.close() - - -def main(): - """Get arguments and run the command.""" - from argparse import ArgumentParser - parser = ArgumentParser(prog="check-package-libraries.py", - description="Check packages for " - "provided/needed libraries") - parser.add_argument("-d", "--database", type=str, - help="Database file to use", - default="package-libraries.sqlite") - subparsers = parser.add_subparsers() - subparser = subparsers.add_parser(name="init", - help="initialize the database") - subparser.set_defaults(command=init) - subparser = subparsers.add_parser(name="add", - help="add packages to database") - subparser.add_argument("packages", nargs="+", type=str, - help="package files to add") - subparser.set_defaults(command=add) - subparser = subparsers.add_parser(name="remove", - help="remove packages from database") - subparser.add_argument("packages", nargs="+", type=str, - help="package names to remove") - subparser.set_defaults(command=remove) - subparser = subparsers.add_parser(name="check", - help="list broken packages") - subparser.set_defaults(command=check) - arguments = parser.parse_args() - arguments.command(arguments) - - -if __name__ == "__main__": - main() diff --git a/createrepos b/createrepos deleted file mode 100755 index 8da2455..0000000 --- a/createrepos +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Creates the repo structure defined in config - -source "$(dirname "$(readlink -e "$0")")/config" - -mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" - -"$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/cron-jobs/db-cleanup b/cron-jobs/db-cleanup new file mode 100755 index 0000000..ffa2601 --- /dev/null +++ b/cron-jobs/db-cleanup @@ -0,0 +1,69 @@ +#!/bin/bash +# Syncs pools against themselves using database contents as filter to cleanup +# them up +# License: GPLv3 + +# Principles +# * Get repos dbs contents +# * Make them a include list +# * Rsync pools against themselves removing excluded files +# * Instant cleanup! + +trap_exit() { + echo + error "$@" + exit 1 +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +EXTRAFLAGS=() +"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) + +filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") +trap "rm -f -- $(printf %q "$filter")" EXIT + +for _repo in "${PKGREPOS[@]}"; do + for _arch in "${ARCHES[@]}"; do + msg "Getting %s-%s database" "${_repo}" "${_arch}" + + dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" + + if [ ! -r "${dbfile}" ]; then + warning "Not found" + continue + fi + + # Echo the contents into a filter file + bsdtar tf "${dbfile}" | \ + cut -d'/' -f1 | \ + sort -u | \ + sed "s|$|*|" >> "$filter" + + done +done + +msg "Removing old files:" + +for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do + msg2 '%s' "${POOL}" + + rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ + --include-from="$filter" \ + --exclude="*" \ + "${FTP_BASE}/${POOL}/" \ + "${FTP_BASE}/${POOL}/" +done + +msg "Removing dead symlinks:" +actions=(-print) +"${CLEANUP_DRYRUN}" || actions+=(-delete) +find -L "${FTP_BASE}/" -type l "${actions[@]}" diff --git a/db-check-nonfree-in-db b/db-check-nonfree-in-db new file mode 100755 index 0000000..a486fa5 --- /dev/null +++ b/db-check-nonfree-in-db @@ -0,0 +1,28 @@ +#!/usr/bin/env python2 +#-*- encoding: utf-8 -*- +from filter import * +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="nonfree_in_db", + description="Cleans nonfree files on repo",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean", + required=True,) + + args=parser.parse_args() + + if not (args.blacklist_file and args.database): + parser.print_help() + exit(1) + + blacklist=listado(args.blacklist_file) + pkgs=get_pkginfo_from_db(args.database) + + print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) diff --git a/db-check-package-libraries b/db-check-package-libraries new file mode 100755 index 0000000..612fc4f --- /dev/null +++ b/db-check-package-libraries @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Check which libraries are provided or required by a package, store +this in a database, update and list broken packages. + +Dependencies: + +- Python 3.2 or later with SQLite 3 support + +- ``bsdtar`` + +- ``readelf`` +""" + + +import os.path +import re +import sqlite3 +import subprocess +import tempfile + + +#: Regexp matching an interesting dynamic entry. +_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" + "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") + + +def make_db(path): + """Make a new, empty, library database at *path*.""" + con = sqlite3.connect(path) + con.executescript(""" +create table provided( + library varchar not null, + package varchar not null +); +create table used( + library varchar not null, + package varchar not null +); +""") + con.close() + + +def begin(database): + """Connect to *database* and start a transaction.""" + con = sqlite3.connect(database) + con.execute("begin exclusive") + return con + + +def add_provided(con, package, libraries): + """Write that *package* provides *libraries*.""" + for library in libraries: + con.execute("insert into provided (package, library) values (?,?)", + (package, library)) + + +def add_used(con, package, libraries): + """Write that *package* uses *libraries*.""" + for library in libraries: + con.execute("insert into used (package, library) values (?,?)", + (package, library)) + + +def remove_package(con, package): + """Remove all entries for a package.""" + con.execute("delete from provided where package=?", (package,)) + con.execute("delete from used where package=?", (package,)) + + +def add_package(con, package): + """Add entries from a named *package*.""" + # Extract to a temporary directory. This could be done more + # efficiently, since there is no need to store more than one file + # at once. + with tempfile.TemporaryDirectory() as temp: + tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) + tar.communicate() + with open(os.path.join(temp, ".PKGINFO")) as pkginfo: + for line in pkginfo: + if line.startswith("pkgname ="): + pkgname = line[len("pkgname ="):].strip() + break + # Don't list previously removed libraries. + remove_package(con, pkgname) + provided = set() + used = set() + # Search for ELFs. + for dirname, dirnames, filenames in os.walk(temp): + assert dirnames is not None # unused, avoid pylint warning + for file_name in filenames: + path = os.path.join(dirname, file_name) + with open(path, "rb") as file_object: + if file_object.read(4) != b"\177ELF": + continue + readelf = subprocess.Popen(("readelf", "-d", path), + stdout=subprocess.PIPE) + for line in readelf.communicate()[0].split(b"\n"): + match = _DYNAMIC.match(line.decode("ascii")) + if match: + if match.group(1) == "SONAME": + provided.add(match.group(2)) + elif match.group(1) == "NEEDED": + used.add(match.group(2)) + else: + raise AssertionError("unknown entry type " + + match.group(1)) + add_provided(con, pkgname, provided) + add_used(con, pkgname, used) + + +def init(arguments): + """Initialize.""" + make_db(arguments.database) + + +def add(arguments): + """Add packages.""" + con = begin(arguments.database) + for package in arguments.packages: + add_package(con, package) + con.commit() + con.close() + + +def remove(arguments): + """Remove packages.""" + con = begin(arguments.database) + for package in arguments.packages: + remove_package(con, package) + con.commit() + con.close() + + +def check(arguments): + """List broken packages.""" + con = begin(arguments.database) + available = set(row[0] for row + in con.execute("select library from provided")) + for package, library in con.execute("select package, library from used"): + if library not in available: + print(package, "needs", library) + con.close() + + +def main(): + """Get arguments and run the command.""" + from argparse import ArgumentParser + parser = ArgumentParser(prog="db-check-package-libraries", + description="Check packages for " + "provided/needed libraries") + parser.add_argument("-d", "--database", type=str, + help="Database file to use", + default="package-libraries.sqlite") + subparsers = parser.add_subparsers() + subparser = subparsers.add_parser(name="init", + help="initialize the database") + subparser.set_defaults(command=init) + subparser = subparsers.add_parser(name="add", + help="add packages to database") + subparser.add_argument("packages", nargs="+", type=str, + help="package files to add") + subparser.set_defaults(command=add) + subparser = subparsers.add_parser(name="remove", + help="remove packages from database") + subparser.add_argument("packages", nargs="+", type=str, + help="package names to remove") + subparser.set_defaults(command=remove) + subparser = subparsers.add_parser(name="check", + help="list broken packages") + subparser.set_defaults(command=check) + arguments = parser.parse_args() + arguments.command(arguments) + + +if __name__ == "__main__": + main() diff --git a/db-check-unsigned-packages b/db-check-unsigned-packages new file mode 100755 index 0000000..0fc053b --- /dev/null +++ b/db-check-unsigned-packages @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -e + +# Output a list of repo/package-name-and-version pairs representing +# unsigned packages available for architecture $1 and specified for +# architecture $2 (usually $1 or any, default is to list all). + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -lt 1 ]; then + msg "usage: %s " "${0##*/}" + exit 1 +fi + +arch=$1 +shift + +for repo in "${PKGREPOS[@]}" +do + db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-check-unsigned-packages.py" "$repo" "$@" < "$db" +done diff --git a/db-check-unsigned-packages.py b/db-check-unsigned-packages.py new file mode 100755 index 0000000..80cff51 --- /dev/null +++ b/db-check-unsigned-packages.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Output a list of repo/package-name-and-version pairs representing +unsigned packages in the database at standard input of repo named in +the first argument and specified for architectures listed in the +following arguments (usually the one of the database or any, default +is to list all). + +If the --keyset argument is passed, print the key fingerprint of every +signed package. +""" + + +import base64 +import subprocess +import sys +import tarfile + + +def main(): + """Do the job.""" + check_keys = False + if "--keyset" in sys.argv: + sys.argv.remove("--keyset") + check_keys = True + repo = sys.argv[1] + pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) + packages = [] + keys = [] + with tarfile.open(fileobj=sys.stdin.buffer) as archive: + for entry in archive: + if entry.name.endswith("/desc"): + content = archive.extractfile(entry) + skip = False + is_arch = False + key = None + for line in content: + if is_arch: + is_arch = False + if pkgarches and line.strip() not in pkgarches: + skip = True # different architecture + break + if line == b"%PGPSIG%\n": + skip = True # signed + key = b"" + if check_keys: + continue + else: + break + if line == b"%ARCH%\n": + is_arch = True + continue + if key is not None: + if line.strip(): + key += line.strip() + else: + break + if check_keys and key: + key_binary = base64.b64decode(key) + keys.append(key_binary) + packages.append(repo + "/" + entry.name[:-5]) + if skip: + continue + print(repo + "/" + entry.name[:-5]) + if check_keys and keys: + # We have collected all signed package names in packages and + # all keys in keys. Let's now ask gpg to list all signatures + # and find which keys made them. + packets = subprocess.check_output(("gpg", "--list-packets"), + input=b"".join(keys)) + i = 0 + for line in packets.decode("latin1").split("\n"): + if line.startswith(":signature packet:"): + keyid = line[line.index("keyid ") + len("keyid "):] + print(packages[i], keyid) + i += 1 + + +if __name__ == "__main__": + main() diff --git a/db-cleanup b/db-cleanup deleted file mode 100755 index ffa2601..0000000 --- a/db-cleanup +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Syncs pools against themselves using database contents as filter to cleanup -# them up -# License: GPLv3 - -# Principles -# * Get repos dbs contents -# * Make them a include list -# * Rsync pools against themselves removing excluded files -# * Instant cleanup! - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -EXTRAFLAGS=() -"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) - -filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") -trap "rm -f -- $(printf %q "$filter")" EXIT - -for _repo in "${PKGREPOS[@]}"; do - for _arch in "${ARCHES[@]}"; do - msg "Getting %s-%s database" "${_repo}" "${_arch}" - - dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" - - if [ ! -r "${dbfile}" ]; then - warning "Not found" - continue - fi - - # Echo the contents into a filter file - bsdtar tf "${dbfile}" | \ - cut -d'/' -f1 | \ - sort -u | \ - sed "s|$|*|" >> "$filter" - - done -done - -msg "Removing old files:" - -for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do - msg2 '%s' "${POOL}" - - rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ - --include-from="$filter" \ - --exclude="*" \ - "${FTP_BASE}/${POOL}/" \ - "${FTP_BASE}/${POOL}/" -done - -msg "Removing dead symlinks:" -actions=(-print) -"${CLEANUP_DRYRUN}" || actions+=(-delete) -find -L "${FTP_BASE}/" -type l "${actions[@]}" diff --git a/db-import-any-to-ours b/db-import-any-to-ours new file mode 100755 index 0000000..a901d54 --- /dev/null +++ b/db-import-any-to-ours @@ -0,0 +1,71 @@ +#!/bin/bash +# Releases 'any' packages from Arch arches to ours + +trap_exit() { + echo + error "$@" + exit 1 +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +# The architecture to compare with +BASEARCH='x86_64' + +# Traverse all Arch repos +for _repo in "${ARCHREPOS[@]}"; do + msg "Processing %s..." "${_repo}" + + # Find 'any' packages + # This is hardcoded but it could release other arches... + PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + -iname '*-any.pkg.tar.?z' \ + -printf "%f ")) + + if [ ${#PKGS[@]} -eq 0 ]; then + msg2 "No '%s' packages here" any + continue + fi + + for _arch in "${OURARCHES[@]}"; do + msg2 "Syncing %s..." "${_arch}" + + # Sync 'any' only and extract the synced packages + SYNCED=($( + rsync -av \ + --include='*-any.pkg.tar.?z' \ + --include='*-any.pkg.tar.?z.sig' \ + --exclude='*' \ + "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \ + grep 'any\.pkg\.tar\..z$' | \ + cut -d ' ' -f 1 )) + + if [ ${#SYNCED[@]} -eq 0 ]; then + msg2 "Already synced (or error happened)" + continue + fi + + msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" + + msg2 "Adding to db..." + + pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null + + # Add the packages to the db + repo-add "${_repo}${DBEXT}" "${SYNCED[@]}" + + popd >/dev/null + + # Avoid mixups + unset SYNCED PKGS + done +done diff --git a/db-import-pkg-archlinux b/db-import-pkg-archlinux new file mode 100755 index 0000000..81221fd --- /dev/null +++ b/db-import-pkg-archlinux @@ -0,0 +1,210 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * make a tarball of files used for forensics + +set -e + +# Run as `V=true db-import-pkg-archlinux` to get verbose output +VERBOSE=${V} +extra=() +${VERBOSE} && extra+=(-v) + +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + +# Returns contents of a repo +get_repos() { + # Exclude everything but db files + rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" +} + +get_repo_content() { + # Return all contents + bsdtar tf "${1}" | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + echo "${WORKDIR}/${1}/os/${2}/${1}" +} + +# Process the databases and get the libre packages +init() { + + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in "${ARCHREPOS[@]}"; do + for _arch in "${ARCHARCHES[@]}"; do + msg "Processing %s-%s" "${_repo}" "${_arch}" + + db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} + files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content "${db_file}")) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" + + msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync "${extra[@]}" --no-motd -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + "${WORKDIR}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists, check for single *s just in case + cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist + + msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + local pkgpool + for pkgpool in "${ARCHPKGPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ + "${FTP_BASE}/${pkgpool}/" + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + local srcpool + for srcpool in "${ARCHSRCPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ + "${FTP_BASE}/${srcpool}/" + done + + date -u +%s > "${FTP_BASE}/lastsync" + + # Cleanup + unset blacklist whitelists _arch _repo repo_file +} + +trap_exit() { + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-pkg-archlinux.conf" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" + +# Check variables presence +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do + test -z "${!var}" && fatal_error "Empty %s" "${var}" +done + +# From makepkg +set -E +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR + +init diff --git a/db-import-pkg-archlinux.conf b/db-import-pkg-archlinux.conf new file mode 100644 index 0000000..24fc44d --- /dev/null +++ b/db-import-pkg-archlinux.conf @@ -0,0 +1,11 @@ +mirror="mirrors.kernel.org" + +## mirrors without sources folder +#mirror="mirrors.niyawe.de" +#mirror="mirror.nl.leaseweb.net" +#mirror="mirror.one.com" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.bytemark.co.uk" +#mirror="mirror.de.leaseweb.net" + +mirrorpath="archlinux" diff --git a/db-import-pkg-archlinuxarm b/db-import-pkg-archlinuxarm new file mode 100755 index 0000000..c707e5b --- /dev/null +++ b/db-import-pkg-archlinuxarm @@ -0,0 +1,199 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * make a tarball of files used for forensics + +set -e + +# Run as `V=true db-import-pkg-archlinux` to get verbose output +VERBOSE=${V} +extra=() +${VERBOSE} && extra+=(-v) + +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + +# Returns contents of a repo +get_repos() { + # Exclude everything but db files + rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" +} + +get_repo_content() { + # Return all contents + bsdtar tf "${1}" | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + echo "${WORKDIR}/${2}/${1}/${1}" +} + +# Process the databases and get the libre packages +init() { + + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _arch in "${OURARCHES[@]}"; do + for _repo in "${ARMREPOS[@]}"; do + msg "Processing %s-%s" "${_repo}" "${_arch}" + + db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} + files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this arch-repo" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this arch-repo" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content "${db_file}")) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" + + msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + # Sync excluding everything but whitelist + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ + "${FTP_BASE}/${PKGPOOLARM}/" + + msg "Putting databases back in place" + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + "${WORKDIR}/${_arch}/${_repo}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Cleanup + unset db + done + done + + + msg "Generating symbolic links to pool" + + for _arch in "${OURARCHES[@]}"; do + for _repo in "${ARMREPOS[@]}"; do + # Modify whitelist to search packages and create symlinks + sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist" + + msg "Putting symlinks in ${_repo}/os/${_arch}" + + while read _pkgfile; do + # Symlink to package + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" + elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}" + fi + + # Symlink to signature + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" + elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig" + fi + done < "/tmp/${_repo}-${_arch}.whitelist" + done + done + + date -u +%s > "${FTP_BASE}/lastsync" + + # Cleanup + unset blacklist whitelists _arch _repo repo_file _pkgfile +} + +trap_exit() { + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-pkg-archlinuxarm.conf" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" + +# Check variables presence +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do + test -z "${!var}" && fatal_error "Empty %s" "${var}" +done + +# From makepkg +set -E +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR + +init diff --git a/db-import-pkg-archlinuxarm.conf b/db-import-pkg-archlinuxarm.conf new file mode 100644 index 0000000..eaa170f --- /dev/null +++ b/db-import-pkg-archlinuxarm.conf @@ -0,0 +1,7 @@ +#mirror="mirror.yandex.ru" +mirror="ftp.halifax.rwth-aachen.de" + +## mirrors without sources folder +## use "archlinuxarm" instead "archlinux-arm" to mirror.yandex.ru + +mirrorpath="archlinux-arm" diff --git a/db-import-src-archlinux b/db-import-src-archlinux new file mode 100755 index 0000000..72171f1 --- /dev/null +++ b/db-import-src-archlinux @@ -0,0 +1,124 @@ +#!/bin/bash + +set -e + +FTP_BASE=/srv/repo/main +ABSLIBRE=/srv/abslibre +ABSGIT=/srv/git/abslibre/abslibre.git +# Remote +# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git +BLACKLIST=/home/repo/blacklist/blacklist.txt +SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' +BLFILE=/tmp/blacklist.txt + +# Variables from abs.conf +ABSROOT="/srv/abs/" +# DON'T CHANGE. WE NEED IT FOR ABSLIBRE +SYNCSERVER="rsync.archlinux.org" +ARCH="i686" +MIRRORLIST="/etc/pacman.d/mirrorlist" +REPOS=(core extra community testing community-testing !staging !community-staging) + +# Steps +# * Sync abs +# * Download blacklist.txt +# * Sync abslibre from abs excluding from blacklist +# * Create repo.abs.tar.gz tarballs + +function sync_abs() { + for ARCH in any i686 x86_64; do + rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? + done + + # fix some permissions + find "${ABSROOT}" -type d -print0 | xargs -0 chmod 755 + find "${ABSROOT}" -type f -print0 | xargs -0 chmod 644 +} + +function get_blacklist() { + printf ":: Updating blacklist...\t" + cat "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ + sed "s/^/**\//" > ${BLFILE} || { + printf "[FAILED]\n" + return 1 + } + + # Prevent using an empty blacklist + [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 + + printf "[OK]\n" +} + +function sync_abs_libre() { + + # Clone ABSLibre git repo + rm -rf /tmp/abslibre + git clone "$ABSGIT" /tmp/abslibre + + # Sync from ABS and then sync from ABSLibre + printf ":: Syncing ABSLibre...\t" + (rsync ${SYNCARGS} --delete-excluded \ + --exclude-from=${BLFILE} \ + ${ABSROOT} \ + ${ABSLIBRE} \ + && + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + printf "[FAILED]\n" + return 1 + } + + # fix some permissions + find "${ABSLIBRE}" -type d -print0 | xargs -0 chmod 755 + find "${ABSLIBRE}" -type f -print0 | xargs -0 chmod 644 + + printf "[OK]\n" +} + +# This part is very hacky and particular to the current setup :P +sync_pre_mips64el() { + pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null + + sudo -u fauno sh -c " + rsync ${SYNCARGS} \ + --exclude=.git* \ + --exclude=community-staging \ + --exclude=community-testing \ + --exclude=gnome-unstable \ + --exclude=kde-unstable \ + --exclude=multilib \ + --exclude=multilib-testing \ + --exclude=multilib-staging \ + --exclude=staging \ + --exclude=testing \ + ${ABSLIBRE}/x86_64/ \ + /home/fauno/Repos/abslibre-pre-mips64el/ && + git add . && + git commit -m \"$(date)\" -a + git push origin master + git gc + " +} + +# Create .abs.tar.gz tarballs +create_tarballs() { + for repo in ${ABSLIBRE}/{i686,x86_64}/*; do + baserepo=${repo##*/} + arch=$(basename $(dirname $repo)) + + # Remove the old one + mkdir -p $FTP_BASE/$baserepo/os/$arch/ + rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz + # Create a new one joining arch and any + # Remove the first part of the path (it could be $repo but any isn't hit) + bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ + $repo/* ${ABSLIBRE}/any/${baserepo}/* + + done +} + +sync_abs +get_blacklist +sync_abs_libre +#sync_pre_mips64el +create_tarballs diff --git a/db-init b/db-init new file mode 100755 index 0000000..8da2455 --- /dev/null +++ b/db-init @@ -0,0 +1,8 @@ +#!/bin/bash +# Creates the repo structure defined in config + +source "$(dirname "$(readlink -e "$0")")/config" + +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" + +"$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/db-libremessages b/db-libremessages old mode 100755 new mode 100644 diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages deleted file mode 100755 index 095e1e6..0000000 --- a/db-list-unsigned-packages +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -set -e - -# Output a list of repo/package-name-and-version pairs representing -# unsigned packages available for architecture $1 and specified for -# architecture $2 (usually $1 or any, default is to list all). - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -lt 1 ]; then - msg "usage: %s " "${0##*/}" - exit 1 -fi - -arch=$1 -shift - -for repo in "${PKGREPOS[@]}" -do - db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" -done diff --git a/db-list-unsigned-packages.py b/db-list-unsigned-packages.py deleted file mode 100755 index 80cff51..0000000 --- a/db-list-unsigned-packages.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Output a list of repo/package-name-and-version pairs representing -unsigned packages in the database at standard input of repo named in -the first argument and specified for architectures listed in the -following arguments (usually the one of the database or any, default -is to list all). - -If the --keyset argument is passed, print the key fingerprint of every -signed package. -""" - - -import base64 -import subprocess -import sys -import tarfile - - -def main(): - """Do the job.""" - check_keys = False - if "--keyset" in sys.argv: - sys.argv.remove("--keyset") - check_keys = True - repo = sys.argv[1] - pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) - packages = [] - keys = [] - with tarfile.open(fileobj=sys.stdin.buffer) as archive: - for entry in archive: - if entry.name.endswith("/desc"): - content = archive.extractfile(entry) - skip = False - is_arch = False - key = None - for line in content: - if is_arch: - is_arch = False - if pkgarches and line.strip() not in pkgarches: - skip = True # different architecture - break - if line == b"%PGPSIG%\n": - skip = True # signed - key = b"" - if check_keys: - continue - else: - break - if line == b"%ARCH%\n": - is_arch = True - continue - if key is not None: - if line.strip(): - key += line.strip() - else: - break - if check_keys and key: - key_binary = base64.b64decode(key) - keys.append(key_binary) - packages.append(repo + "/" + entry.name[:-5]) - if skip: - continue - print(repo + "/" + entry.name[:-5]) - if check_keys and keys: - # We have collected all signed package names in packages and - # all keys in keys. Let's now ask gpg to list all signatures - # and find which keys made them. - packets = subprocess.check_output(("gpg", "--list-packets"), - input=b"".join(keys)) - i = 0 - for line in packets.decode("latin1").split("\n"): - if line.startswith(":signature packet:"): - keyid = line[line.index("keyid ") + len("keyid "):] - print(packages[i], keyid) - i += 1 - - -if __name__ == "__main__": - main() diff --git a/db-sync b/db-sync deleted file mode 100755 index facfae9..0000000 --- a/db-sync +++ /dev/null @@ -1,210 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics - -set -e - -# Run as `V=true db-sync` to get verbose output -VERBOSE=${V} -extra=() -${VERBOSE} && extra+=(-v) - -WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT - -# Returns contents of a repo -get_repos() { - # Exclude everything but db files - rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" -} - -get_repo_content() { - # Return all contents - bsdtar tf "${1}" | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${WORKDIR}/${1}/os/${2}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "%d packages in blacklist" ${#blacklist[@]} - - test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in "${ARCHREPOS[@]}"; do - for _arch in "${ARCHARCHES[@]}"; do - msg "Processing %s-%s" "${_repo}" "${_arch}" - - db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} - files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - # Get db contents - db=($(get_repo_content "${db_file}")) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - # IMPORTANT: the . in the sed command is needed because an empty - # whitelist would consist of a single * allowing any package to - # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" - - msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync "${extra[@]}" --no-motd -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from="/tmp/${_repo}-${_arch}.whitelist" \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - "${WORKDIR}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists, check for single *s just in case - cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist - - msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local pkgpool - for pkgpool in "${ARCHPKGPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ - "${FTP_BASE}/${pkgpool}/" - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local srcpool - for srcpool in "${ARCHSRCPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ - "${FTP_BASE}/${srcpool}/" - done - - date -u +%s > "${FTP_BASE}/lastsync" - - # Cleanup - unset blacklist whitelists _arch _repo repo_file -} - -trap_exit() { - local signal=$1; shift - echo - error "$@" - trap -- "$signal" - kill "-$signal" "$$" -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-sync.conf" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do - test -z "${!var}" && fatal_error "Empty %s" "${var}" -done - -# From makepkg -set -E -for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" -done -trap 'trap_exit INT "Aborted by user! Exiting..."' INT -trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR - -init diff --git a/db-sync-arm b/db-sync-arm deleted file mode 100755 index 952b546..0000000 --- a/db-sync-arm +++ /dev/null @@ -1,199 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics - -set -e - -# Run as `V=true db-sync` to get verbose output -VERBOSE=${V} -extra=() -${VERBOSE} && extra+=(-v) - -WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT - -# Returns contents of a repo -get_repos() { - # Exclude everything but db files - rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" -} - -get_repo_content() { - # Return all contents - bsdtar tf "${1}" | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${WORKDIR}/${2}/${1}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "%d packages in blacklist" ${#blacklist[@]} - - test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _arch in "${OURARCHES[@]}"; do - for _repo in "${ARMREPOS[@]}"; do - msg "Processing %s-%s" "${_repo}" "${_arch}" - - db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} - files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this arch-repo" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this arch-repo" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - # Get db contents - db=($(get_repo_content "${db_file}")) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - # IMPORTANT: the . in the sed command is needed because an empty - # whitelist would consist of a single * allowing any package to - # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" - - msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - # Sync excluding everything but whitelist - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from="/tmp/${_repo}-${_arch}.whitelist" \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ - "${FTP_BASE}/${PKGPOOLARM}/" - - msg "Putting databases back in place" - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - "${WORKDIR}/${_arch}/${_repo}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Cleanup - unset db - done - done - - - msg "Generating symbolic links to pool" - - for _arch in "${OURARCHES[@]}"; do - for _repo in "${ARMREPOS[@]}"; do - # Modify whitelist to search packages and create symlinks - sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist" - - msg "Putting symlinks in ${_repo}/os/${_arch}" - - while read _pkgfile; do - # Symlink to package - if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" - elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}" - fi - - # Symlink to signature - if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" - elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig" - fi - done < "/tmp/${_repo}-${_arch}.whitelist" - done - done - - date -u +%s > "${FTP_BASE}/lastsync" - - # Cleanup - unset blacklist whitelists _arch _repo repo_file _pkgfile -} - -trap_exit() { - local signal=$1; shift - echo - error "$@" - trap -- "$signal" - kill "-$signal" "$$" -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-sync-arm.conf" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do - test -z "${!var}" && fatal_error "Empty %s" "${var}" -done - -# From makepkg -set -E -for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" -done -trap 'trap_exit INT "Aborted by user! Exiting..."' INT -trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR - -init diff --git a/db-sync-arm.conf b/db-sync-arm.conf deleted file mode 100644 index eaa170f..0000000 --- a/db-sync-arm.conf +++ /dev/null @@ -1,7 +0,0 @@ -#mirror="mirror.yandex.ru" -mirror="ftp.halifax.rwth-aachen.de" - -## mirrors without sources folder -## use "archlinuxarm" instead "archlinux-arm" to mirror.yandex.ru - -mirrorpath="archlinux-arm" diff --git a/db-sync.conf b/db-sync.conf deleted file mode 100644 index 24fc44d..0000000 --- a/db-sync.conf +++ /dev/null @@ -1,11 +0,0 @@ -mirror="mirrors.kernel.org" - -## mirrors without sources folder -#mirror="mirrors.niyawe.de" -#mirror="mirror.nl.leaseweb.net" -#mirror="mirror.one.com" -#mirror="mirror.us.leaseweb.net" -#mirror="mirror.bytemark.co.uk" -#mirror="mirror.de.leaseweb.net" - -mirrorpath="archlinux" diff --git a/list_nonfree_in_db.py b/list_nonfree_in_db.py deleted file mode 100755 index a486fa5..0000000 --- a/list_nonfree_in_db.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python2 -#-*- encoding: utf-8 -*- -from filter import * -import argparse - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="nonfree_in_db", - description="Cleans nonfree files on repo",) - - parser.add_argument("-k", "--blacklist-file", type=str, - help="File containing blacklisted names", - required=True,) - - parser.add_argument("-b", "--database", type=str, - help="dabatase to clean", - required=True,) - - args=parser.parse_args() - - if not (args.blacklist_file and args.database): - parser.print_help() - exit(1) - - blacklist=listado(args.blacklist_file) - pkgs=get_pkginfo_from_db(args.database) - - print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) diff --git a/mkrepo b/mkrepo deleted file mode 100755 index b11dc0b..0000000 --- a/mkrepo +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Author: Nicolás Reynolds -# License: GPLv3+ -# Description: A script to quickly create new [repos] - -source "$(dirname "$(readlink -e "$0")")/config" - -for repo in "$@"; do - echo ":: Creating [$repo]" - for arch in "${ARCHES[@]}"; do - mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" - done -done - -echo ":: All done. Add the repo to the ParabolaWeb admin page." -- cgit v1.2.3 From fdc37e5cf06d845a88971881a99de034a723c94a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 01:59:40 -0400 Subject: Grab an updated test/ directory from lukeshu/archlinux+cleanup+librelib It's not like we were running the tests anyway, and this makes merging easier. --- test/__init__.py | 0 test/blacklist_sample | 2 - test/core.db.tar.gz | Bin 1345 -> 0 bytes test/depends | 4 - test/desc | 39 ------- test/lib/common.inc | 168 +++++++++++++++------------ test/packages/pkg-any-a/PKGBUILD | 4 +- test/packages/pkg-any-b/PKGBUILD | 4 +- test/packages/pkg-simple-a/PKGBUILD | 6 +- test/packages/pkg-simple-b/PKGBUILD | 6 +- test/packages/pkg-simple-epoch/PKGBUILD | 6 +- test/packages/pkg-split-a/PKGBUILD | 10 +- test/packages/pkg-split-b/PKGBUILD | 10 +- test/rsync_output_sample | 14 --- test/runTest | 10 +- test/test.d/create-filelists.sh | 48 ++++---- test/test.d/db-move.sh | 68 ++++++----- test/test.d/db-remove.sh | 50 ++++---- test/test.d/db-repo-add.sh | 30 +++-- test/test.d/db-repo-remove.sh | 38 +++---- test/test.d/db-update.sh | 69 ++++++----- test/test.d/ftpdir-cleanup.sh | 61 +++++----- test/test.d/packages.sh | 2 +- test/test.d/pool-transition.sh | 152 ------------------------- test/test.d/signed-packages.sh | 27 ++++- test/test.d/sourceballs.sh | 47 ++++---- test/test.d/testing2x.sh | 6 +- test/test_filter.py | 196 -------------------------------- 28 files changed, 353 insertions(+), 724 deletions(-) delete mode 100644 test/__init__.py delete mode 100644 test/blacklist_sample delete mode 100644 test/core.db.tar.gz delete mode 100644 test/depends delete mode 100644 test/desc delete mode 100644 test/rsync_output_sample delete mode 100755 test/test.d/pool-transition.sh delete mode 100644 test/test_filter.py diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/blacklist_sample b/test/blacklist_sample deleted file mode 100644 index 2a02af6..0000000 --- a/test/blacklist_sample +++ /dev/null @@ -1,2 +0,0 @@ -alex:alex-libre: Aquí va un comentario -gmime22 ::Non free dependencies \ No newline at end of file diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz deleted file mode 100644 index 5eb2081..0000000 Binary files a/test/core.db.tar.gz and /dev/null differ diff --git a/test/depends b/test/depends deleted file mode 100644 index 7ff3ad4..0000000 --- a/test/depends +++ /dev/null @@ -1,4 +0,0 @@ -%DEPENDS% -glibc>=2.13 -zlib - diff --git a/test/desc b/test/desc deleted file mode 100644 index abba644..0000000 --- a/test/desc +++ /dev/null @@ -1,39 +0,0 @@ -%FILENAME% -binutils-2.21-4-x86_64.pkg.tar.xz - -%NAME% -binutils - -%VERSION% -2.21-4 - -%DESC% -A set of programs to assemble and manipulate binary and object files - -%GROUPS% -base - -%CSIZE% -3412892 - -%ISIZE% -17571840 - -%MD5SUM% -4e666f87c78998f4839f33dc06d2043a - -%URL% -http://www.gnu.org/software/binutils/ - -%LICENSE% -GPL - -%ARCH% -x86_64 - -%BUILDDATE% -1297240369 - -%PACKAGER% -Allan McRae - diff --git a/test/lib/common.inc b/test/lib/common.inc index a2dee10..954868b 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -1,7 +1,24 @@ +#!/hint/bash set -E -. "$(dirname ${BASH_SOURCE[0]})/../../config" -. "$(dirname ${BASH_SOURCE[0]})/../../db-functions" +. "$(dirname "${BASH_SOURCE[0]}")/../../config" +. "$(dirname "${BASH_SOURCE[0]}")/../../config.testing" +. "$(dirname "${BASH_SOURCE[0]}")/../../db-functions" + +signpkg() { + if [[ -r '/etc/makepkg.conf' ]]; then + source '/etc/makepkg.conf' + else + die '/etc/makepkg.conf not found!' + fi + if [[ -r ~/.makepkg.conf ]]; then + . ~/.makepkg.conf + fi + if [[ -n $GPGKEY ]]; then + SIGNWITHKEY=(-u "${GPGKEY}") + fi + gpg --detach-sign --use-agent "${SIGNWITHKEY[@]}" "${@}" || die +} oneTimeSetUp() { local p @@ -11,34 +28,38 @@ oneTimeSetUp() { local pkgarch local pkgversion local build - pkgdir="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)" - cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" + pkgdir="$(mktemp -dt "${0##*/}.XXXXXXXXXX")" + cp -Lr "$(dirname "${BASH_SOURCE[0]}")"/../packages/* "${pkgdir}" msg 'Building packages...' for d in "${pkgdir}"/*; do - pushd $d >/dev/null - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - pkgarch=($(. PKGBUILD; echo ${arch[@]})) - pkgversion=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pushd "$d" >/dev/null + pkgname=($(. PKGBUILD; echo "${pkgname[@]}")) + pkgarch=($(. PKGBUILD; echo "${arch[@]}")) + pkgversion=$(. PKGBUILD; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") build=true - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - [ ! -f ${p}-${pkgversion}-${a}${PKGEXT} ] && build=false + for a in "${pkgarch[@]}"; do + for p in "${pkgname[@]}"; do + [ ! -f "${p}-${pkgversion}-${a}"${PKGEXT} ] && build=false done done - if ! ${build}; then + if ! "${build}"; then if [ "${pkgarch[0]}" == 'any' ]; then sudo extra-x86_64-build || die 'extra-x86_64-build failed' else - for a in ${pkgarch[@]}; do - sudo extra-${a}-build || die "extra-${a}-build failed" + for a in "${pkgarch[@]}"; do + if in_array "$a" "${ARCH_BUILD[@]}"; then + sudo "extra-${a}-build" || die "extra-${a}-build failed" + fi done fi - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/$(basename ${d}) - done + for a in "${pkgarch[@]}"; do + if in_array "$a" "${ARCH_BUILD[@]}"; then + for p in "${pkgname[@]}"; do + cp "${p}-${pkgversion}-${a}"${PKGEXT} "$(dirname "${BASH_SOURCE[0]}")/../packages/${d##*/}" + done + fi done fi popd >/dev/null @@ -55,17 +76,17 @@ setUp() { local r local a - [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" - TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" + [ -f "$(dirname "${BASH_SOURCE[0]}")/../../config.local" ] && die "$(dirname "${BASH_SOURCE[0]}")/../../config.local exists" + TMP="$(mktemp -dt "${0##*/}.XXXXXXXXXX")" #msg "Using ${TMP}" PKGREPOS=('core' 'extra' 'testing') PKGPOOL='pool/packages' mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} - for r in ${PKGREPOS[@]}; do + for r in "${PKGREPOS[@]}"; do mkdir -p "${TMP}/staging/${r}" - for a in ${ARCHES[@]} any; do + for a in "${ARCHES[@]}"; do mkdir -p "${TMP}/ftp/${r}/os/${a}" done done @@ -74,21 +95,23 @@ setUp() { msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" - svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + arch_svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" for p in "${pkgdir}"/*; do - pkg=$(basename $p) + pkg=${p##*/} mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} - cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ - svn add -q "${TMP}/svn-packages-copy"/${pkg} - svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" + cp "${p}"/* "${TMP}/svn-packages-copy/${pkg}/trunk/" + arch_svn add -q "${TMP}/svn-packages-copy/${pkg}" + arch_svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done - cat < "$(dirname ${BASH_SOURCE[0]})/../../config.local" + cat < "$(dirname "${BASH_SOURCE[0]}")/../../config.local" FTP_BASE="${TMP}/ftp" SVNREPO="file://${TMP}/svn-packages-repo" - PKGREPOS=(${PKGREPOS[@]}) + PKGREPOS=("${PKGREPOS[@]}") PKGPOOL="${PKGPOOL}" + TESTING_REPO='testing' + STABLE_REPOS=('core' 'extra') CLEANUP_DESTDIR="${TMP}/package-cleanup" SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" @@ -97,12 +120,12 @@ setUp() { SOURCE_CLEANUP_DRYRUN=false REQUIRE_SIGNATURE=true eot - . "$(dirname ${BASH_SOURCE[0]})/../../config" + . "$(dirname "${BASH_SOURCE[0]}")/../../config" } tearDown() { rm -rf "${TMP}" - rm -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" + rm -f "$(dirname "${BASH_SOURCE[0]}")/../../config.local" echo } @@ -110,18 +133,24 @@ releasePackage() { local repo=$1 local pkgbase=$2 local arch=$3 + local a + local p + local pkgver + local pkgname - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null - archrelease ${repo}-${arch} >/dev/null 2&>1 - pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" >/dev/null + archrelease "${repo}-${arch}" >/dev/null 2>&1 + pkgver=$(. PKGBUILD; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") + pkgname=($(. PKGBUILD; echo "${pkgname[@]}")) popd >/dev/null - cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ + cp "${pkgdir}/${pkgbase}"/*-"${pkgver}-${arch}"${PKGEXT} "${STAGING}/${repo}/" - if ${REQUIRE_SIGNATURE}; then - # TODO: really sign the packages with a valid key - find "${STAGING}"/${repo}/ -type f \ - -name "*-${pkgver}-${arch}${PKGEXT}" \ - -exec touch {}.sig \; + if "${REQUIRE_SIGNATURE}"; then + for a in "${arch[@]}"; do + for p in "${pkgname[@]}"; do + signpkg "${STAGING}/${repo}/${p}-${pkgver}-${a}"${PKGEXT} + done + done fi } @@ -132,42 +161,39 @@ checkAnyPackageDB() { local db [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - if ${REQUIRE_SIGNATURE}; then + if "${REQUIRE_SIGNATURE}"; then [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" fi - for arch in i686 x86_64; do + for arch in "${ARCH_BUILD[@]}"; do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - if ${REQUIRE_SIGNATURE}; then + if "${REQUIRE_SIGNATURE}"; then [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" fi - done - [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + for db in "${DBEXT}" "${FILESEXT}"; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep "${pkg}" &>/dev/null) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done done - - [ -r "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" - [ -r "${FTP_BASE}/${repo}/os/any/${pkg}.sig" ] && fail "${repo}/os/any/${pkg}.sig should not exist" + [ -r "${STAGING}/${repo}/${pkg}" ] && fail "${repo}/${pkg} found in staging dir" + [ -r "${STAGING}/${repo}/${pkg}".sig ] && fail "${repo}/${pkg}.sig found in staging dir" } checkAnyPackage() { local repo=$1 local pkg=$2 - checkAnyPackageDB $repo $pkg + checkAnyPackageDB "$repo" "$pkg" local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" } @@ -180,23 +206,23 @@ checkPackageDB() { [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" - [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" + [ -r "${STAGING}/${repo}/${pkg}" ] && fail "${repo}/${pkg} found in staging dir" [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - if ${REQUIRE_SIGNATURE}; then + if "${REQUIRE_SIGNATURE}"; then [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" + [ -r "${STAGING}/${repo}/${pkg}.sig" ] && fail "${repo}/${pkg}.sig found in staging dir" [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" fi - for db in ${DBEXT} ${FILESEXT}; do + for db in "${DBEXT}" "${FILESEXT}"; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep "${pkg}" &>/dev/null) \ || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" done } @@ -206,10 +232,10 @@ checkPackage() { local pkg=$2 local arch=$3 - checkPackageDB $repo $pkg $arch + checkPackageDB "$repo" "$pkg" "$arch" local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" } @@ -220,9 +246,9 @@ checkRemovedPackageDB() { local arch=$3 local db - for db in ${DBEXT} ${FILESEXT}; do + for db in "${DBEXT}" "${FILESEXT}"; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep "${pkgbase}" &>/dev/null) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" done } @@ -232,9 +258,9 @@ checkRemovedPackage() { local pkgbase=$2 local arch=$3 - checkRemovedPackageDB $repo $pkgbase $arch + checkRemovedPackageDB "$repo" "$pkgbase" "$arch" - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" } @@ -245,10 +271,10 @@ checkRemovedAnyPackageDB() { local arch local db - for db in ${DBEXT} ${FILESEXT}; do - for arch in i686 x86_64; do + for db in "${DBEXT}" "${FILESEXT}"; do + for arch in "${ARCH_BUILD[@]}"; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep "${pkgbase}" &>/dev/null) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" done done @@ -258,9 +284,9 @@ checkRemovedAnyPackage() { local repo=$1 local pkgbase=$2 - checkRemovedAnyPackageDB $repo $pkgbase + checkRemovedAnyPackageDB "$repo" "$pkgbase" - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" } diff --git a/test/packages/pkg-any-a/PKGBUILD b/test/packages/pkg-any-a/PKGBUILD index 8749a35..cd5d66f 100644 --- a/test/packages/pkg-any-a/PKGBUILD +++ b/test/packages/pkg-any-a/PKGBUILD @@ -7,6 +7,6 @@ url='http://www.archlinux.org/' license=('GPL') package() { - install -d -m755 ${pkgdir}/usr/share/${pkgname} - echo 'test' > ${pkgdir}/usr/share/${pkgname}/test + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test } diff --git a/test/packages/pkg-any-b/PKGBUILD b/test/packages/pkg-any-b/PKGBUILD index e6a0498..90794fc 100644 --- a/test/packages/pkg-any-b/PKGBUILD +++ b/test/packages/pkg-any-b/PKGBUILD @@ -7,6 +7,6 @@ url='http://www.archlinux.org/' license=('GPL') package() { - install -d -m755 ${pkgdir}/usr/share/${pkgname} - echo 'test' > ${pkgdir}/usr/share/${pkgname}/test + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test } diff --git a/test/packages/pkg-simple-a/PKGBUILD b/test/packages/pkg-simple-a/PKGBUILD index 953ecfa..9b4478e 100644 --- a/test/packages/pkg-simple-a/PKGBUILD +++ b/test/packages/pkg-simple-a/PKGBUILD @@ -12,11 +12,11 @@ md5sums=('c6cb8dcc86253355fed559416d0c8dcf' '3c1e4279feb678fd9cabaccdb28e40d0') build() { - cd ${srcdir} + cd "${srcdir}" make } package() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname} + cd "${srcdir}" + make install DESTDIR="${pkgdir}" DESTBIN="${pkgname}" } diff --git a/test/packages/pkg-simple-b/PKGBUILD b/test/packages/pkg-simple-b/PKGBUILD index 95ffd09..4a9e58d 100644 --- a/test/packages/pkg-simple-b/PKGBUILD +++ b/test/packages/pkg-simple-b/PKGBUILD @@ -12,11 +12,11 @@ md5sums=('c6cb8dcc86253355fed559416d0c8dcf' '3c1e4279feb678fd9cabaccdb28e40d0') build() { - cd ${srcdir} + cd "${srcdir}" make } package() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname} + cd "${srcdir}" + make install DESTDIR="${pkgdir}" DESTBIN="${pkgname}" } diff --git a/test/packages/pkg-simple-epoch/PKGBUILD b/test/packages/pkg-simple-epoch/PKGBUILD index eebe2bd..0761b32 100644 --- a/test/packages/pkg-simple-epoch/PKGBUILD +++ b/test/packages/pkg-simple-epoch/PKGBUILD @@ -13,11 +13,11 @@ md5sums=('c6cb8dcc86253355fed559416d0c8dcf' '3c1e4279feb678fd9cabaccdb28e40d0') build() { - cd ${srcdir} + cd "${srcdir}" make } package() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname} + cd "${srcdir}" + make install DESTDIR="${pkgdir}" DESTBIN="${pkgname}" } diff --git a/test/packages/pkg-split-a/PKGBUILD b/test/packages/pkg-split-a/PKGBUILD index e941976..f7a0576 100644 --- a/test/packages/pkg-split-a/PKGBUILD +++ b/test/packages/pkg-split-a/PKGBUILD @@ -13,16 +13,16 @@ md5sums=('c6cb8dcc86253355fed559416d0c8dcf' '3c1e4279feb678fd9cabaccdb28e40d0') build() { - cd ${srcdir} + cd "${srcdir}" make } package_pkg-split-a1() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} + cd "${srcdir}" + make install DESTDIR="${pkgdir}" DESTBIN="${pkgname[0]}" } package_pkg-split-a2() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} + cd "${srcdir}" + make install DESTDIR="${pkgdir}" DESTBIN="${pkgname[1]}" } diff --git a/test/packages/pkg-split-b/PKGBUILD b/test/packages/pkg-split-b/PKGBUILD index 6ddbc45..3bd635c 100644 --- a/test/packages/pkg-split-b/PKGBUILD +++ b/test/packages/pkg-split-b/PKGBUILD @@ -14,16 +14,16 @@ md5sums=('c6cb8dcc86253355fed559416d0c8dcf' '3c1e4279feb678fd9cabaccdb28e40d0') build() { - cd ${srcdir} + cd "${srcdir}" make } package_pkg-split-b1() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} + cd "${srcdir}" + make install DESTDIR="${pkgdir}" DESTBIN="${pkgname[0]}" } package_pkg-split-b2() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} + cd "${srcdir}" + make install DESTDIR="${pkgdir}" DESTBIN="${pkgname[1]}" } diff --git a/test/rsync_output_sample b/test/rsync_output_sample deleted file mode 100644 index 72d9cd0..0000000 --- a/test/rsync_output_sample +++ /dev/null @@ -1,14 +0,0 @@ -dr-xr-sr-x 4096 2010/09/11 11:37:10 . --rw-r--r-- 11 2011/02/08 00:00:01 lastsync -drwxrwxr-x 15 2010/09/11 11:28:50 community-staging -drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os -drwxrwxr-x 8192 2011/02/07 17:00:01 community-staging/os/i686 -lrwxrwxrwx 52 2010/12/23 16:51:01 community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz -> ../../../pool/community/alex-2.3.4-1-i686.pkg.tar.xz -lrwxrwxrwx 27 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db -> community-staging.db.tar.gz --rw-rw-r-- 2237 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db.tar.gz --rw-rw-r-- 3209 2011/02/07 14:00:13 community-staging/os/i686/community-staging.db.tar.gz.old -drwxrwxr-x 15 2009/07/22 15:07:56 community -drwxrwxr-x 40 2009/08/04 15:57:42 community/os -drwxrwsr-x 36864 2011/02/03 05:00:01 community/os/any --rw-rw-r-- 303336 2010/07/16 10:06:28 community/os/any/any2dvd-0.34-4-any.pkg.tar.xz --rw-rw-r-- 221664 2010/03/28 15:55:48 community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz diff --git a/test/runTest b/test/runTest index b8713d8..7163e30 100755 --- a/test/runTest +++ b/test/runTest @@ -1,12 +1,12 @@ #!/bin/bash -. "$(dirname ${BASH_SOURCE[0]})/lib/common.inc" +. "$(dirname "${BASH_SOURCE[0]}")/lib/common.inc" -for t in "$(dirname ${BASH_SOURCE[0]})/test.d/"*.sh; do - l=$(basename ${t} .sh) - if [ -x ${t} ]; then +for t in "$(dirname "${BASH_SOURCE[0]}")/test.d/"*.sh; do + l=$(basename "${t}" .sh) + if [ -x "${t}" ]; then msg "Running test '${l}'" - ${t} + "${t}" [ $? -ne 0 ] && die "Test '${l}' failed" echo -e "\n\n\n" else diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 49734c4..21bf292 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -1,23 +1,22 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testCreateSimpleFileLists() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}" fi @@ -26,18 +25,17 @@ testCreateSimpleFileLists() { } testCreateAnyFileLists() { - local arches=('i686' 'x86_64') local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" any done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}" fi @@ -46,7 +44,6 @@ testCreateAnyFileLists() { } testCreateSplitFileLists() { - local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -54,17 +51,17 @@ testCreateSplitFileLists() { local pkgnames local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do + for pkgbase in "${pkgs[@]}"; do pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) - for pkgname in ${pkgnames[@]}; do - for arch in ${arches[@]}; do + for pkgname in "${pkgnames[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}" fi @@ -75,23 +72,22 @@ testCreateSplitFileLists() { testCleanupFileLists() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-a + for arch in "${ARCH_BUILD[@]}"; do + ../db-remove extra "${arch}" pkg-simple-a done - for arch in ${arches[@]}; do + for arch in "${ARCH_BUILD[@]}"; do if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}" fi diff --git a/test/test.d/db-move.sh b/test/test.d/db-move.sh index 9d7c1f6..4b4120e 100755 --- a/test/test.d/db-move.sh +++ b/test/test.d/db-move.sh @@ -1,17 +1,16 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testMoveSimplePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage testing "${pkgbase}" "${arch}" done done @@ -19,23 +18,22 @@ testMoveSimplePackages() { ../db-move testing extra pkg-simple-a - for arch in ${arches[@]}; do - checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + for arch in "${ARCH_BUILD[@]}"; do + checkPackage extra "pkg-simple-a-1-1-${arch}.pkg.tar.xz" "${arch}" + checkRemovedPackage testing "pkg-simple-a-1-1-${arch}.pkg.tar.xz" "${arch}" - checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} + checkPackage testing "pkg-simple-b-1-1-${arch}.pkg.tar.xz" "${arch}" done } testMoveMultiplePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage testing "${pkgbase}" "${arch}" done done @@ -43,23 +41,22 @@ testMoveMultiplePackages() { ../db-move testing extra pkg-simple-a pkg-simple-b - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + checkPackage extra "${pkgbase}-1-1-${arch}.pkg.tar.xz" "${arch}" + checkRemovedPackage testing "${pkgbase}-1-1-${arch}.pkg.tar.xz" "${arch}" done done } testMoveEpochPackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-epoch') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage testing "${pkgbase}" "${arch}" done done @@ -67,9 +64,9 @@ testMoveEpochPackages() { ../db-move testing extra pkg-simple-epoch - for arch in ${arches[@]}; do - checkPackage extra pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} + for arch in "${ARCH_BUILD[@]}"; do + checkPackage extra "pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz" "${arch}" + checkRemovedPackage testing "pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz" "${arch}" done } @@ -77,8 +74,8 @@ testMoveAnyPackages() { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any + for pkgbase in "${pkgs[@]}"; do + releasePackage testing "${pkgbase}" any done ../db-update @@ -90,29 +87,28 @@ testMoveAnyPackages() { } testMoveSplitPackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage testing "${pkgbase}" "${arch}" done done ../db-update ../db-move testing extra pkg-split-a - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/pkg-split-a"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + for arch in "${ARCH_BUILD[@]}"; do + for pkg in "${pkgdir}/pkg-split-a"/*-"${arch}"${PKGEXT}; do + checkPackage extra "${pkg##*/}" "${arch}" done done - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/pkg-split-b"/*-${arch}${PKGEXT}; do - checkPackage testing $(basename ${pkg}) ${arch} + for arch in "${ARCH_BUILD[@]}"; do + for pkg in "${pkgdir}/pkg-split-b"/*-"${arch}"${PKGEXT}; do + checkPackage testing "${pkg##*/}" "${arch}" done done diff --git a/test/test.d/db-remove.sh b/test/test.d/db-remove.sh index 416e693..a391197 100755 --- a/test/test.d/db-remove.sh +++ b/test/test.d/db-remove.sh @@ -1,56 +1,54 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testRemovePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + ../db-remove extra "${arch}" "${pkgbase}" done done - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + checkRemovedPackage extra "${pkgbase}" "${arch}" done done } testRemoveMultiplePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgs[@]} + for arch in "${ARCH_BUILD[@]}"; do + ../db-remove extra "${arch}" "${pkgs[@]}" done - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + checkRemovedPackage extra "${pkgbase}" "${arch}" done done } @@ -59,18 +57,18 @@ testRemoveAnyPackages() { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" any done ../db-update - for pkgbase in ${pkgs[@]}; do - ../db-remove extra any ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + ../db-remove extra any "${pkgbase}" done - for pkgbase in ${pkgs[@]}; do - checkRemovedAnyPackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkRemovedAnyPackage extra "${pkgbase}" done } diff --git a/test/test.d/db-repo-add.sh b/test/test.d/db-repo-add.sh index 8603104..266a696 100755 --- a/test/test.d/db-repo-add.sh +++ b/test/test.d/db-repo-add.sh @@ -1,52 +1,50 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testAddSimplePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" - ../db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz + ../db-repo-add extra "${arch}" "${pkgbase}-1-1-${arch}.pkg.tar.xz" done done - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + checkPackageDB extra "${pkgbase}-1-1-${arch}.pkg.tar.xz" "${arch}" done done } testAddMultiplePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch - for arch in ${arches[@]}; do + for arch in "${ARCH_BUILD[@]}"; do add_pkgs=() - for pkgbase in ${pkgs[@]}; do + for pkgbase in "${pkgs[@]}"; do cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" - add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz + add_pkgs+=("${pkgbase}-1-1-${arch}.pkg.tar.xz") done - ../db-repo-add extra ${arch} ${add_pkgs[@]} + ../db-repo-add extra "${arch}" "${add_pkgs[@]}" done - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + checkPackageDB extra "${pkgbase}-1-1-${arch}.pkg.tar.xz" "${arch}" done done } diff --git a/test/test.d/db-repo-remove.sh b/test/test.d/db-repo-remove.sh index 315d63d..727188d 100755 --- a/test/test.d/db-repo-remove.sh +++ b/test/test.d/db-repo-remove.sh @@ -1,56 +1,54 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testRemovePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - ../db-repo-remove extra ${arch} ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + ../db-repo-remove extra "${arch}" "${pkgbase}" done done - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackageDB extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + checkRemovedPackageDB extra "${pkgbase}" "${arch}" done done } testRemoveMultiplePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for arch in ${arches[@]}; do - ../db-repo-remove extra ${arch} ${pkgs[@]} + for arch in "${ARCH_BUILD[@]}"; do + ../db-repo-remove extra "${arch}" "${pkgs[@]}" done - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackageDB extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + checkRemovedPackageDB extra "${pkgbase}" "${arch}" done done } diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index e38c328..ca4efe7 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -1,25 +1,24 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testAddSimplePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + checkPackage extra "${pkgbase}-1-1-${arch}.pkg.tar.xz" "${arch}" done done } @@ -40,36 +39,35 @@ testAddAnyPackages() { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" any done ../db-update - for pkgbase in ${pkgs[@]}; do - checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz + for pkgbase in "${pkgs[@]}"; do + checkAnyPackage extra "${pkgbase}-1-1-any.pkg.tar.xz" done } testAddSplitPackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + for pkg in "${pkgdir}/${pkgbase}"/*-"${arch}"${PKGEXT}; do + checkPackage extra "${pkg##*/}" "${arch}" done done done @@ -81,8 +79,8 @@ testUpdateAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 + arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo extra-i686-build mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -99,8 +97,8 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 + arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo extra-i686-build mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -134,32 +132,41 @@ testUpdateSameAnyPackageToDifferentRepositories() { local arch for arch in i686 x86_64; do ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q "${pkgbase}") \ && fail "${pkgbase} should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" done } testAddIncompleteSplitPackage() { - local arches=('i686' 'x86_64') local repo='extra' local pkgbase='pkg-split-a' local arch - for arch in ${arches[@]}; do - releasePackage ${repo} ${pkgbase} ${arch} + for arch in "${ARCH_BUILD[@]}"; do + releasePackage "${repo}" "${pkgbase}" "${arch}" done # remove a split package to make db-update fail - rm "${STAGING}"/extra/${pkgbase}1-* + rm "${STAGING}/extra/${pkgbase}1-"* ../db-update >/dev/null 2>&1 && fail "db-update should fail when a split package is missing!" - for arch in ${arches[@]}; do + for arch in "${ARCH_BUILD[@]}"; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q "${pkgbase}") \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" done } +testUnknownRepo() { + mkdir "${STAGING}/unknown/" + releasePackage extra 'pkg-simple-a' 'i686' + releasePackage unknown 'pkg-simple-b' 'i686' + ../db-update + checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + [ -e "${FTP_BASE}/unknown" ] && fail "db-update pushed a package into an unknown repository" + rm -rf "${STAGING}/unknown/" +} + . "${curdir}/../lib/shunit2" diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh index 20026b4..5a7afea 100755 --- a/test/test.d/ftpdir-cleanup.sh +++ b/test/test.d/ftpdir-cleanup.sh @@ -1,62 +1,60 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testCleanupSimplePackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-a + for arch in "${ARCH_BUILD[@]}"; do + ../db-remove extra "${arch}" pkg-simple-a done ../cron-jobs/ftpdir-cleanup >/dev/null - for arch in ${arches[@]}; do + for arch in "${ARCH_BUILD[@]}"; do local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" - checkRemovedPackage extra 'pkg-simple-a' ${arch} + checkRemovedPackage extra 'pkg-simple-a' "${arch}" [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" - checkPackage extra ${pkg2} ${arch} + checkPackage extra "${pkg2}" "${arch}" done } testCleanupEpochPackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-epoch') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-epoch + for arch in "${ARCH_BUILD[@]}"; do + ../db-remove extra "${arch}" pkg-simple-epoch done ../cron-jobs/ftpdir-cleanup >/dev/null - for arch in ${arches[@]}; do + for arch in "${ARCH_BUILD[@]}"; do local pkg1="pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz" - checkRemovedPackage extra 'pkg-simple-epoch' ${arch} + checkRemovedPackage extra 'pkg-simple-epoch' "${arch}" [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" done @@ -67,8 +65,8 @@ testCleanupAnyPackages() { local pkgbase local arch='any' - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" any done ../db-update @@ -81,39 +79,38 @@ testCleanupAnyPackages() { [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" - checkAnyPackage extra ${pkg2} + checkAnyPackage extra "${pkg2}" } testCleanupSplitPackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgs[0]} + for arch in "${ARCH_BUILD[@]}"; do + ../db-remove extra "${arch}" "${pkgs[0]}" done ../cron-jobs/ftpdir-cleanup >/dev/null - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}${PKGEXT}; do - checkRemovedPackage extra ${pkgs[0]} ${arch} + for arch in "${ARCH_BUILD[@]}"; do + for pkg in "${pkgdir}/${pkgs[0]}"/*-"${arch}"${PKGEXT}; do + checkRemovedPackage extra "${pkgs[0]}" "${arch}" [ -f "${FTP_BASE}/${PKGPOOL}/${pkg}" ] && fail "${PKGPOOL}/${pkg} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" done - for pkg in "${pkgdir}/${pkgs[1]}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + for pkg in "${pkgdir}/${pkgs[1]}"/*-"${arch}"${PKGEXT}; do + checkPackage extra "${pkg##*/}" "${arch}" done done } diff --git a/test/test.d/packages.sh b/test/test.d/packages.sh index 488cb15..18266eb 100755 --- a/test/test.d/packages.sh +++ b/test/test.d/packages.sh @@ -1,6 +1,6 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testPackages() { diff --git a/test/test.d/pool-transition.sh b/test/test.d/pool-transition.sh deleted file mode 100755 index 5873f00..0000000 --- a/test/test.d/pool-transition.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testMovePackagesWithoutPool() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') - local pkgbase - local arch - local pkg - local old - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - - # transform two packages to old style layout - for arch in ${arches[@]}; do - for old in 0 2; do - for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}${PKGEXT}; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} - done - checkRemovedPackage testing ${pkgbase} ${arch} - done - done -} - -testUpdateAnyPackageWithoutPool() { - local pkgname='pkg-any-a' - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - local pkg2='pkg-any-a-1-2-any.pkg.tar.xz' - local arch - - - releasePackage extra pkg-any-a any - ../db-update - # transform two packages to old style layout - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" "${FTP_BASE}/extra/os/any" - for arch in i686 x86_64; do - ln -sf "../any/${pkg1}" "${FTP_BASE}/extra/os/${arch}" - done - - pushd "${TMP}/svn-packages-copy/${pkgname}/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 - mv "${pkg2}" "${pkgdir}/${pkgname}/" - popd >/dev/null - - releasePackage extra ${pkgname} any - ../db-update - rm -f "${pkgdir}/${pkgname}/${pkg2}" - - ../cron-jobs/ftpdir-cleanup >/dev/null - - checkAnyPackage extra "${pkg2}" - - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/extra/os/${arch}/${pkg1}" ] && fail "extra/os/${arch}/${pkg1} found" - done -} - -testMoveAnyPackagesWithoutPool() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch - local pkg - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any - done - - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-any${PKGEXT}; do - checkAnyPackage extra $(basename ${pkg}) - done - checkRemovedAnyPackage testing ${pkgbase} - done - - for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=$(basename $pkg) - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/testing/os/${arch}/${pkg}" ] && fail "testing/os/${arch}/${pkg} found" - done - done -} - -testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { - local pkg - local arch - - releasePackage extra pkg-any-a any - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/pkg-any-a"/*-any${PKGEXT}; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" - done - done - - releasePackage testing pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) - - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q pkg-any-a) \ - && fail "pkg-any-a should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" - done -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/signed-packages.sh b/test/test.d/signed-packages.sh index 5d6f4ff..3ffe146 100755 --- a/test/test.d/signed-packages.sh +++ b/test/test.d/signed-packages.sh @@ -1,13 +1,36 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" +testAddSignedPackage() { + releasePackage extra 'pkg-simple-a' 'i686' + ../db-update || fail "db-update failed!" +} + testAddUnsignedPackage() { releasePackage extra 'pkg-simple-a' 'i686' - # remove any signature rm "${STAGING}"/extra/*.sig ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" } +testAddInvalidSignedPackage() { + local p + releasePackage extra 'pkg-simple-a' 'i686' + for p in "${STAGING}"/extra/*${PKGEXT}; do + unxz "$p" + xz -0 "${p%%.xz}" + done + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is invalid!" +} + +testAddBrokenSignature() { + local s + releasePackage extra 'pkg-simple-a' 'i686' + for s in "${STAGING}"/extra/*.sig; do + echo 0 > "$s" + done + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" +} + . "${curdir}/../lib/shunit2" diff --git a/test/test.d/sourceballs.sh b/test/test.d/sourceballs.sh index fdcf08c..81c9265 100755 --- a/test/test.d/sourceballs.sh +++ b/test/test.d/sourceballs.sh @@ -1,24 +1,23 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testSourceballs() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update ../cron-jobs/sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + for pkgbase in "${pkgs[@]}"; do + [ ! -r "${FTP_BASE}/${SRCPOOL}/${pkgbase}"-*"${SRCEXT}" ] && fail "source package not found!" done } @@ -26,59 +25,57 @@ testAnySourceballs() { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" any done ../db-update ../cron-jobs/sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + for pkgbase in "${pkgs[@]}"; do + [ ! -r "${FTP_BASE}/${SRCPOOL}/${pkgbase}"-*"${SRCEXT}" ] && fail "source package not found!" done } testSplitSourceballs() { - local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update ../cron-jobs/sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + for pkgbase in "${pkgs[@]}"; do + [ ! -r "${FTP_BASE}/${SRCPOOL}/${pkgbase}"-*"${SRCEXT}" ] && fail "source package not found!" done } testSourceballsCleanup() { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${ARCH_BUILD[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update ../cron-jobs/sourceballs - for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-a + for arch in "${ARCH_BUILD[@]}"; do + ../db-remove extra "${arch}" pkg-simple-a done ../cron-jobs/sourceballs - [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" - [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" + [ -r "${FTP_BASE}/${SRCPOOL}/pkg-simple-a"-*"${SRCEXT}" ] && fail "source package was not removed!" + [ ! -r "${FTP_BASE}/${SRCPOOL}/pkg-simple-b"-*"${SRCEXT}" ] && fail "source package not found!" } . "${curdir}/../lib/shunit2" diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index eda6cd6..c611ce4 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -1,6 +1,6 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(dirname "$(readlink -e "$0")")" . "${curdir}/../lib/common.inc" testTesting2xAnyPackage() { @@ -9,8 +9,8 @@ testTesting2xAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 + arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo extra-i686-build mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null diff --git a/test/test_filter.py b/test/test_filter.py deleted file mode 100644 index d8006f9..0000000 --- a/test/test_filter.py +++ /dev/null @@ -1,196 +0,0 @@ -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -class pkginfo_from_file_KnownValues(unittest.TestCase): - # (filename, name, version, release, arch) - # filename is location - known=( - ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), - ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), - ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), - ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), - ) - - def generate_results(self, example_tuple, attr): - location, name, version, release, arch = example_tuple - return pkginfo_from_filename(location)[attr], locals()[attr] - - def testReturnPackageObject(self): - for i in self.known: - location, name, version, release, arch = i - self.assertIsInstance(pkginfo_from_filename(location),Package) - - def testNames(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) - -class pkginfo_from_file_BadInput(unittest.TestCase): - bad=("community-testing/os/i686/community-testing.db", - "community-testing/os/i686/community-testing.db.tar.gz", - "community-testing/os/i686/community-testing.db.tar.gz.old", - "community-testing/os/i686/community-testing.files", - "community-testing/os/i686/community-testing.files.tar.gz", - "community-testing/os/x86_64") - - def testBadInput(self): - for i in self.bad: - self.assertRaises(NonValidFile,pkginfo_from_filename,i) - -class pkginfoFromRsyncOutput(unittest.TestCase): - example_package_list=(Package(),Package(),Package()) - example_package_list[0].package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", - "depends" : False,} - example_package_list[1].package_info={ "name" : "any2dvd", - "version" : "0.34", - "release" : "4", - "arch" : "any", - "license" : False, - "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", - "depends" : False,} - example_package_list[2].package_info={ "name" : "gmime22", - "version" : "2.2.26", - "release" : "1", - "arch" : "x86_64", - "license" : False, - "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", - "depends" : False,} - - try: - output_file = open("rsync_output_sample") - rsync_out= output_file.read() - output_file.close() - except IOError: print("There is no rsync_output_sample file") - - pkglist = pkginfo_from_rsync_output(rsync_out) - - def testOutputArePackages(self): - if not self.pkglist: - self.fail("not pkglist:" + str(self.pkglist)) - for pkg in self.pkglist: - self.assertIsInstance(pkg,Package) - - def testPackageInfo(self): - if not self.pkglist: - self.fail("Pkglist doesn't exist: " + str(self.pkglist)) - self.assertEqual(self.pkglist,self.example_package_list) - -class generateRsyncBlacklist(unittest.TestCase): - example_package_list=(Package(),Package(),Package()) - example_package_list[0].package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", - "depends" : False,} - example_package_list[1].package_info={ "name" : "any2dvd", - "version" : "0.34", - "release" : "4", - "arch" : "any", - "license" : False, - "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", - "depends" : False,} - example_package_list[2].package_info={ "name" : "gmime22", - "version" : "2.2.26", - "release" : "1", - "arch" : "x86_64", - "license" : False, - "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", - "depends" : False,} - - def testListado(self): - self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) - - def testExcludeFiles(self): - a=rsyncBlacklist_from_blacklist(self.example_package_list, - listado("blacklist_sample"), - False) - b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] - self.assertEqual(a,b) - -class pkginfo_from_descKnownValues(unittest.TestCase): - pkgsample=Package() - pkgsample.package_info={"name" : "binutils", - "version" : "2.21", - "release" : "4", - "arch" : "x86_64", - "license" : "GPL", - "location": "binutils-2.21-4-x86_64.pkg.tar.xz", - "depends" : False,} - fsock=open("desc") - pkggen=pkginfo_from_desc(fsock.read()) - fsock.close() - def testPkginfoFromDesc(self): - if self.pkggen is None: - self.fail("return value is None") - self.assertEqual(self.pkgsample,self.pkggen) - -class pkginfo_from_db(unittest.TestCase): - archdb = os.path.join("./workdir") - example_package_list=(Package(),Package(),Package()) - example_package_list[0].package_info={ "name" : "acl", - "version" : "2.2.49", - "release" : "2", - "arch" : "x86_64", - "license" : ("LGPL",), - "location": "acl-2.2.49-2-x86_64.pkg.tar.xz", - "depends" : ("attr>=2.4.41"),} - example_package_list[1].package_info={ "name" : "glibc", - "version" : "2.13", - "release" : "4", - "arch" : "x86_64", - "license" : ("GPL","LGPL"), - "location": "glibc-2.13-4-x86_64.pkg.tar.xz", - "depends" : ("linux-api-headers>=2.6.37","tzdata",),} - example_package_list[2].package_info={ "name" : "", - "version" : "2.2.26", - "release" : "1", - "arch" : "x86_64", - "license" : False, - "location": "", - "depends" : False,} - - -if __name__ == "__main__": - unittest.main() - -- cgit v1.2.3 From 1f7bbb22ced3495140810b9ec21d93ebb4c161f3 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 02:17:15 -0400 Subject: begone with db-libremessages --- cron-jobs/db-cleanup | 4 +-- db-import-any-to-ours | 2 +- db-import-pkg-archlinux | 7 +++- db-import-pkg-archlinuxarm | 7 +++- db-libremessages | 83 ---------------------------------------------- 5 files changed, 15 insertions(+), 88 deletions(-) delete mode 100644 db-libremessages diff --git a/cron-jobs/db-cleanup b/cron-jobs/db-cleanup index ffa2601..ce0e5f7 100755 --- a/cron-jobs/db-cleanup +++ b/cron-jobs/db-cleanup @@ -15,8 +15,8 @@ trap_exit() { exit 1 } -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" +source "$(dirname $(dirname "$(readlink -e "$0")"))/config" +source "$(librelib messages)" # From makepkg set -E diff --git a/db-import-any-to-ours b/db-import-any-to-ours index a901d54..8a4e874 100755 --- a/db-import-any-to-ours +++ b/db-import-any-to-ours @@ -8,7 +8,7 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" +source "$(librelib messages)" # From makepkg set -E diff --git a/db-import-pkg-archlinux b/db-import-pkg-archlinux index 81221fd..d20d095 100755 --- a/db-import-pkg-archlinux +++ b/db-import-pkg-archlinux @@ -190,9 +190,14 @@ trap_exit() { kill "-$signal" "$$" } +fatal_error() { + error "$@" + exit 1 +} + source "$(dirname "$(readlink -e "$0")")/config" source "$(dirname "$(readlink -e "$0")")/db-import-pkg-archlinux.conf" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" +source "$(librelib messages)" # Check variables presence for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do diff --git a/db-import-pkg-archlinuxarm b/db-import-pkg-archlinuxarm index c707e5b..a512278 100755 --- a/db-import-pkg-archlinuxarm +++ b/db-import-pkg-archlinuxarm @@ -179,9 +179,14 @@ trap_exit() { kill "-$signal" "$$" } +fatal_error() { + error "$@" + exit 1 +} + source "$(dirname "$(readlink -e "$0")")/config" source "$(dirname "$(readlink -e "$0")")/db-import-pkg-archlinuxarm.conf" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" +source "$(librelib messages)" # Check variables presence for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do diff --git a/db-libremessages b/db-libremessages deleted file mode 100644 index 37df149..0000000 --- a/db-libremessages +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2006-2010 Pacman Development Team -# Copyright (c) 2002-2006 by Judd Vinet -# Copyright (c) 2005 by Aurelien Foret -# Copyright (c) 2006 by Miklos Vajna -# Copyright (c) 2005 by Christian Hamar -# Copyright (c) 2006 by Alex Smith -# Copyright (c) 2006 by Andras Voroskoi -# Copyright (c) 2011 by Joshua Haase -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# gettext initialization -export TEXTDOMAIN='libretools' -export TEXTDOMAINDIR='/usr/share/locale' - -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW - -if tput setaf 0 &>/dev/null; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" - PURPLE="${ALL_OFF}$(tput setaf 5)" -else - ALL_OFF="\033[1;0m" - BOLD="\033[1;1m" - BLUE="${BOLD}\033[1;34m" - GREEN="${BOLD}\033[1;32m" - RED="${BOLD}\033[1;31m" - YELLOW="${BOLD}\033[1;33m" - PURPLE="${BOLD}\033[1;30;40m" -fi - -stdnull() { - local action=$1; - eval "${action} >/dev/null 2>&1" -} - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -fatal_error() { - local mesg=$1; shift - error "$mesg" "$@" - - exit 1 -} -- cgit v1.2.3 From 8fc0116dc5a2205d910b74e4d1fe8b6b79c50757 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 10:50:09 -0400 Subject: bring back xihh's tests into tests-xihh/ --- tests-xihh/__init__.py | 0 tests-xihh/core.db.tar.gz | Bin 0 -> 1345 bytes tests-xihh/depends | 4 + tests-xihh/desc | 39 ++++++++ tests-xihh/rsync_output_sample | 14 +++ tests-xihh/test_filter.py | 196 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 253 insertions(+) create mode 100644 tests-xihh/__init__.py create mode 100644 tests-xihh/core.db.tar.gz create mode 100644 tests-xihh/depends create mode 100644 tests-xihh/desc create mode 100644 tests-xihh/rsync_output_sample create mode 100644 tests-xihh/test_filter.py diff --git a/tests-xihh/__init__.py b/tests-xihh/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests-xihh/core.db.tar.gz b/tests-xihh/core.db.tar.gz new file mode 100644 index 0000000..5eb2081 Binary files /dev/null and b/tests-xihh/core.db.tar.gz differ diff --git a/tests-xihh/depends b/tests-xihh/depends new file mode 100644 index 0000000..7ff3ad4 --- /dev/null +++ b/tests-xihh/depends @@ -0,0 +1,4 @@ +%DEPENDS% +glibc>=2.13 +zlib + diff --git a/tests-xihh/desc b/tests-xihh/desc new file mode 100644 index 0000000..abba644 --- /dev/null +++ b/tests-xihh/desc @@ -0,0 +1,39 @@ +%FILENAME% +binutils-2.21-4-x86_64.pkg.tar.xz + +%NAME% +binutils + +%VERSION% +2.21-4 + +%DESC% +A set of programs to assemble and manipulate binary and object files + +%GROUPS% +base + +%CSIZE% +3412892 + +%ISIZE% +17571840 + +%MD5SUM% +4e666f87c78998f4839f33dc06d2043a + +%URL% +http://www.gnu.org/software/binutils/ + +%LICENSE% +GPL + +%ARCH% +x86_64 + +%BUILDDATE% +1297240369 + +%PACKAGER% +Allan McRae + diff --git a/tests-xihh/rsync_output_sample b/tests-xihh/rsync_output_sample new file mode 100644 index 0000000..72d9cd0 --- /dev/null +++ b/tests-xihh/rsync_output_sample @@ -0,0 +1,14 @@ +dr-xr-sr-x 4096 2010/09/11 11:37:10 . +-rw-r--r-- 11 2011/02/08 00:00:01 lastsync +drwxrwxr-x 15 2010/09/11 11:28:50 community-staging +drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os +drwxrwxr-x 8192 2011/02/07 17:00:01 community-staging/os/i686 +lrwxrwxrwx 52 2010/12/23 16:51:01 community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz -> ../../../pool/community/alex-2.3.4-1-i686.pkg.tar.xz +lrwxrwxrwx 27 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db -> community-staging.db.tar.gz +-rw-rw-r-- 2237 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db.tar.gz +-rw-rw-r-- 3209 2011/02/07 14:00:13 community-staging/os/i686/community-staging.db.tar.gz.old +drwxrwxr-x 15 2009/07/22 15:07:56 community +drwxrwxr-x 40 2009/08/04 15:57:42 community/os +drwxrwsr-x 36864 2011/02/03 05:00:01 community/os/any +-rw-rw-r-- 303336 2010/07/16 10:06:28 community/os/any/any2dvd-0.34-4-any.pkg.tar.xz +-rw-rw-r-- 221664 2010/03/28 15:55:48 community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz diff --git a/tests-xihh/test_filter.py b/tests-xihh/test_filter.py new file mode 100644 index 0000000..d8006f9 --- /dev/null +++ b/tests-xihh/test_filter.py @@ -0,0 +1,196 @@ +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class pkginfo_from_file_KnownValues(unittest.TestCase): + # (filename, name, version, release, arch) + # filename is location + known=( + ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), + ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), + ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), + ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), + ) + + def generate_results(self, example_tuple, attr): + location, name, version, release, arch = example_tuple + return pkginfo_from_filename(location)[attr], locals()[attr] + + def testReturnPackageObject(self): + for i in self.known: + location, name, version, release, arch = i + self.assertIsInstance(pkginfo_from_filename(location),Package) + + def testNames(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +class pkginfo_from_file_BadInput(unittest.TestCase): + bad=("community-testing/os/i686/community-testing.db", + "community-testing/os/i686/community-testing.db.tar.gz", + "community-testing/os/i686/community-testing.db.tar.gz.old", + "community-testing/os/i686/community-testing.files", + "community-testing/os/i686/community-testing.files.tar.gz", + "community-testing/os/x86_64") + + def testBadInput(self): + for i in self.bad: + self.assertRaises(NonValidFile,pkginfo_from_filename,i) + +class pkginfoFromRsyncOutput(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + try: + output_file = open("rsync_output_sample") + rsync_out= output_file.read() + output_file.close() + except IOError: print("There is no rsync_output_sample file") + + pkglist = pkginfo_from_rsync_output(rsync_out) + + def testOutputArePackages(self): + if not self.pkglist: + self.fail("not pkglist:" + str(self.pkglist)) + for pkg in self.pkglist: + self.assertIsInstance(pkg,Package) + + def testPackageInfo(self): + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,self.example_package_list) + +class generateRsyncBlacklist(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=rsyncBlacklist_from_blacklist(self.example_package_list, + listado("blacklist_sample"), + False) + b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] + self.assertEqual(a,b) + +class pkginfo_from_descKnownValues(unittest.TestCase): + pkgsample=Package() + pkgsample.package_info={"name" : "binutils", + "version" : "2.21", + "release" : "4", + "arch" : "x86_64", + "license" : "GPL", + "location": "binutils-2.21-4-x86_64.pkg.tar.xz", + "depends" : False,} + fsock=open("desc") + pkggen=pkginfo_from_desc(fsock.read()) + fsock.close() + def testPkginfoFromDesc(self): + if self.pkggen is None: + self.fail("return value is None") + self.assertEqual(self.pkgsample,self.pkggen) + +class pkginfo_from_db(unittest.TestCase): + archdb = os.path.join("./workdir") + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "acl", + "version" : "2.2.49", + "release" : "2", + "arch" : "x86_64", + "license" : ("LGPL",), + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz", + "depends" : ("attr>=2.4.41"),} + example_package_list[1].package_info={ "name" : "glibc", + "version" : "2.13", + "release" : "4", + "arch" : "x86_64", + "license" : ("GPL","LGPL"), + "location": "glibc-2.13-4-x86_64.pkg.tar.xz", + "depends" : ("linux-api-headers>=2.6.37","tzdata",),} + example_package_list[2].package_info={ "name" : "", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "", + "depends" : False,} + + +if __name__ == "__main__": + unittest.main() + -- cgit v1.2.3 From df1095483a7f2b1c79dcc2f531dde65a4634c4cf Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 10:59:23 -0400 Subject: begone with create-repo --- create-repo | 21 --------------------- db-init | 2 -- 2 files changed, 23 deletions(-) delete mode 100755 create-repo diff --git a/create-repo b/create-repo deleted file mode 100755 index 3feb098..0000000 --- a/create-repo +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Creates repository structure - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -eq 0 ]; then - msg "Usage: %s repo1 [repo2 ... repoX]" "${0##*/}" - exit 1 -fi - -msg "Creating repos..." -for _repo in "$@"; do - msg2 "Creating [%s]" "${_repo}" - for _arch in "${ARCHES[@]}"; do - mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ - error "Failed creating %s dir" "${_arch}" - done -done - -msg "Don't forget to add them to the PKGREPOS array on %s" "$(dirname "$(readlink -e "$0")")/config" diff --git a/db-init b/db-init index 8da2455..e25dbff 100755 --- a/db-init +++ b/db-init @@ -4,5 +4,3 @@ source "$(dirname "$(readlink -e "$0")")/config" mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" - -"$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" -- cgit v1.2.3 From 1a24330d5e2bb378b87d9708cd039363e212fd29 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 11:09:59 -0400 Subject: tidy --- db-check-nonfree-in-db | 2 +- tests-xihh/blacklist_sample | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 tests-xihh/blacklist_sample diff --git a/db-check-nonfree-in-db b/db-check-nonfree-in-db index a486fa5..3e6b273 100755 --- a/db-check-nonfree-in-db +++ b/db-check-nonfree-in-db @@ -5,7 +5,7 @@ import argparse if __name__ == "__main__": parser = argparse.ArgumentParser( - prog="nonfree_in_db", + prog="db-check-nonfree-in-db", description="Cleans nonfree files on repo",) parser.add_argument("-k", "--blacklist-file", type=str, diff --git a/tests-xihh/blacklist_sample b/tests-xihh/blacklist_sample new file mode 100644 index 0000000..2a02af6 --- /dev/null +++ b/tests-xihh/blacklist_sample @@ -0,0 +1,2 @@ +alex:alex-libre: Aquí va un comentario +gmime22 ::Non free dependencies \ No newline at end of file -- cgit v1.2.3 From 14c69e309503662b709e4a066ca6b7deb8b27195 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 12:25:50 -0400 Subject: Sync test/ with lukeshu/archlinux+cleanup+librelib again. --- test/lib/common.inc | 4 ++-- test/test.d/create-filelists.sh | 10 +++++----- test/test.d/db-update.sh | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/lib/common.inc b/test/lib/common.inc index 954868b..bef8749 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -35,7 +35,7 @@ oneTimeSetUp() { pushd "$d" >/dev/null pkgname=($(. PKGBUILD; echo "${pkgname[@]}")) pkgarch=($(. PKGBUILD; echo "${arch[@]}")) - pkgversion=$(. PKGBUILD; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") + pkgversion=$(. PKGBUILD; get_full_version) build=true for a in "${pkgarch[@]}"; do @@ -140,7 +140,7 @@ releasePackage() { pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" >/dev/null archrelease "${repo}-${arch}" >/dev/null 2>&1 - pkgver=$(. PKGBUILD; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") + pkgver=$(. PKGBUILD; get_full_version) pkgname=($(. PKGBUILD; echo "${pkgname[@]}")) popd >/dev/null cp "${pkgdir}/${pkgbase}"/*-"${pkgver}-${arch}"${PKGEXT} "${STAGING}/${repo}/" diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 21bf292..20dafc6 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -17,7 +17,7 @@ testCreateSimpleFileLists() { for pkgbase in "${pkgs[@]}"; do for arch in "${ARCH_BUILD[@]}"; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/${pkgbase}" &>/dev/null; then fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}" fi done @@ -36,7 +36,7 @@ testCreateAnyFileLists() { for pkgbase in "${pkgs[@]}"; do for arch in "${ARCH_BUILD[@]}"; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/share/${pkgbase}/test" &>/dev/null; then fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}" fi done @@ -62,7 +62,7 @@ testCreateSplitFileLists() { pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) for pkgname in "${pkgnames[@]}"; do for arch in "${ARCH_BUILD[@]}"; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/${pkgname}" &>/dev/null; then fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}" fi done @@ -88,10 +88,10 @@ testCleanupFileLists() { done for arch in "${ARCH_BUILD[@]}"; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/pkg-simple-b" &>/dev/null; then fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}" fi - if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-a"; then + if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/pkg-simple-a" &>/dev/null; then fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}" fi done diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index ca4efe7..540eccf 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -132,7 +132,7 @@ testUpdateSameAnyPackageToDifferentRepositories() { local arch for arch in i686 x86_64; do ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q "${pkgbase}") \ + && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep "${pkgbase}" &>/dev/null) \ && fail "${pkgbase} should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" done } @@ -154,7 +154,7 @@ testAddIncompleteSplitPackage() { for arch in "${ARCH_BUILD[@]}"; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q "${pkgbase}") \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep "${pkgbase}" &>/dev/null) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" done } -- cgit v1.2.3 From f0e4cdbb1ae4d36b5555baeaf420d83258b3003d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 13:45:30 -0400 Subject: Clean up the config files. --- config | 48 ++++++++++++++++------------------------- db-check-nonfree | 1 + db-check-nonfree.conf | 1 + db-import-any-to-ours | 1 + db-import-pkg-archlinux.conf | 8 +++++++ db-import-pkg-archlinuxarm | 2 +- db-import-pkg-archlinuxarm.conf | 6 ++++++ 7 files changed, 36 insertions(+), 31 deletions(-) create mode 100644 db-check-nonfree.conf diff --git a/config b/config index f41e6f6..38cdae5 100644 --- a/config +++ b/config @@ -1,33 +1,30 @@ #!/hint/bash +# Please try to refrain from adding new variables to this file. +# Instead, create separate ${toolname}.conf files. Only add a +# variable here if multiple tools start needing the option. FTP_BASE="/srv/repo/main" -# Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') -ARMREPOS=('core' 'extra' 'community') -# Official Parabola repos -OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') -# User repos -USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') -# Community project repos -PROJREPOS=('nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') -# Remote repos -PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") +PKGREPOS=( + 'core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing' + 'libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing' + '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' + 'nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') PKGPOOL='pool/parabola_gnu+linux' -PKGPOOLARM='pool/arch_gnu+linux_arm' SRCPOOL='src/parabola_gnu+linux' # Directories where packages are shared between repos # *relative to FTP_BASE* -ARCHPKGPOOLS=(pool/{packages,community}) -ARMPKGPOOLS=(pool/arch_gnu+linux_arm) -OURPKGPOOLS=(pool/parabola_gnu+linux) -PKGPOOLS=(${OURPKGPOOLS[@]} ${ARMPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) +PKGPOOLS=(pool/parabola_gnu+linux # Parabola GNU/Linux-libre + pool/arch_gnu+linux_arm # Arch Linux ARM + pool/{packages,community} # Arch Linux + ) # Directories where sources are stored -ARCHSRCPOOLS=(sources/{packages,community}) -ARMSRCPOOLS=(src/arch_gnu+linux_arm) -OURPKGPOOLS=(src/parabola_gnu+linux) -SRCPOOLS=(${OURSRCPOOLS[@]} ${ARMSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) +OURPKGPOOLS=( +SRCPOOLS=(src/parabola_gnu+linux # Parabola GNU/Linux-libre + src/arch_gnu+linux_arm # Arch Linux ARM + sources/{packages,community} # Arch Linux + ) CLEANUP_DESTDIR="$FTP_BASE/old/pkg" CLEANUP_DRYRUN=false @@ -42,20 +39,11 @@ SOURCE_CLEANUP_KEEP=30 REQUIRE_SIGNATURE=true LOCK_DELAY=10 -LOCK_TIMEOUT=300 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) -OURARCHES=(armv7h) -ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) +ARCHES=(i686 x86_64 armv7h) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" - -MAKEPKGCONF="~/.makepkg.conf" -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" - -# parabolaweb root -WEB_DIR=/srv/http/parabola.nu/web diff --git a/db-check-nonfree b/db-check-nonfree index 37b7cf6..ea123fa 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -1,6 +1,7 @@ #!/bin/bash . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-check-nonfree.conf" . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then diff --git a/db-check-nonfree.conf b/db-check-nonfree.conf new file mode 100644 index 0000000..ea3330c --- /dev/null +++ b/db-check-nonfree.conf @@ -0,0 +1 @@ +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" diff --git a/db-import-any-to-ours b/db-import-any-to-ours index 8a4e874..10df57d 100755 --- a/db-import-any-to-ours +++ b/db-import-any-to-ours @@ -8,6 +8,7 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-pkg-archlinux.conf" source "$(librelib messages)" # From makepkg diff --git a/db-import-pkg-archlinux.conf b/db-import-pkg-archlinux.conf index 24fc44d..525fc58 100644 --- a/db-import-pkg-archlinux.conf +++ b/db-import-pkg-archlinux.conf @@ -1,3 +1,11 @@ +ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') +ARCHPKGPOOLS=(pool/{packages,community}) +ARCHSRCPOOLS=(sources/{packages,community}) +ARCHARCHES=(i686 x86_64) +OURARCHES=(armv7h) + +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" + mirror="mirrors.kernel.org" ## mirrors without sources folder diff --git a/db-import-pkg-archlinuxarm b/db-import-pkg-archlinuxarm index a512278..536b603 100755 --- a/db-import-pkg-archlinuxarm +++ b/db-import-pkg-archlinuxarm @@ -189,7 +189,7 @@ source "$(dirname "$(readlink -e "$0")")/db-import-pkg-archlinuxarm.conf" source "$(librelib messages)" # Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE; do test -z "${!var}" && fatal_error "Empty %s" "${var}" done diff --git a/db-import-pkg-archlinuxarm.conf b/db-import-pkg-archlinuxarm.conf index eaa170f..4a12a25 100644 --- a/db-import-pkg-archlinuxarm.conf +++ b/db-import-pkg-archlinuxarm.conf @@ -1,3 +1,9 @@ +ARMREPOS=('core' 'extra' 'community') +PKGPOOLARM='pool/arch_gnu+linux_arm' +OURARCHES=(armv7h) + +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" + #mirror="mirror.yandex.ru" mirror="ftp.halifax.rwth-aachen.de" -- cgit v1.2.3 From a4fb5765c0fe8f6d03985d5375b09c0cd68903f2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 13:49:03 -0400 Subject: switch up the naming schem for the import scripts --- db-import-any-to-ours | 72 -------------- db-import-archlinux-any-to-ours | 72 ++++++++++++++ db-import-archlinux-pkg | 215 ++++++++++++++++++++++++++++++++++++++++ db-import-archlinux-src | 124 +++++++++++++++++++++++ db-import-archlinux.conf | 19 ++++ db-import-archlinuxarm-pkg | 204 ++++++++++++++++++++++++++++++++++++++ db-import-archlinuxarm.conf | 13 +++ db-import-pkg-archlinux | 215 ---------------------------------------- db-import-pkg-archlinux.conf | 19 ---- db-import-pkg-archlinuxarm | 204 -------------------------------------- db-import-pkg-archlinuxarm.conf | 13 --- db-import-src-archlinux | 124 ----------------------- 12 files changed, 647 insertions(+), 647 deletions(-) delete mode 100755 db-import-any-to-ours create mode 100755 db-import-archlinux-any-to-ours create mode 100755 db-import-archlinux-pkg create mode 100755 db-import-archlinux-src create mode 100644 db-import-archlinux.conf create mode 100755 db-import-archlinuxarm-pkg create mode 100644 db-import-archlinuxarm.conf delete mode 100755 db-import-pkg-archlinux delete mode 100644 db-import-pkg-archlinux.conf delete mode 100755 db-import-pkg-archlinuxarm delete mode 100644 db-import-pkg-archlinuxarm.conf delete mode 100755 db-import-src-archlinux diff --git a/db-import-any-to-ours b/db-import-any-to-ours deleted file mode 100755 index 10df57d..0000000 --- a/db-import-any-to-ours +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -# Releases 'any' packages from Arch arches to ours - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-import-pkg-archlinux.conf" -source "$(librelib messages)" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -# The architecture to compare with -BASEARCH='x86_64' - -# Traverse all Arch repos -for _repo in "${ARCHREPOS[@]}"; do - msg "Processing %s..." "${_repo}" - - # Find 'any' packages - # This is hardcoded but it could release other arches... - PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - -iname '*-any.pkg.tar.?z' \ - -printf "%f ")) - - if [ ${#PKGS[@]} -eq 0 ]; then - msg2 "No '%s' packages here" any - continue - fi - - for _arch in "${OURARCHES[@]}"; do - msg2 "Syncing %s..." "${_arch}" - - # Sync 'any' only and extract the synced packages - SYNCED=($( - rsync -av \ - --include='*-any.pkg.tar.?z' \ - --include='*-any.pkg.tar.?z.sig' \ - --exclude='*' \ - "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \ - grep 'any\.pkg\.tar\..z$' | \ - cut -d ' ' -f 1 )) - - if [ ${#SYNCED[@]} -eq 0 ]; then - msg2 "Already synced (or error happened)" - continue - fi - - msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" - - msg2 "Adding to db..." - - pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null - - # Add the packages to the db - repo-add "${_repo}${DBEXT}" "${SYNCED[@]}" - - popd >/dev/null - - # Avoid mixups - unset SYNCED PKGS - done -done diff --git a/db-import-archlinux-any-to-ours b/db-import-archlinux-any-to-ours new file mode 100755 index 0000000..ab9bb77 --- /dev/null +++ b/db-import-archlinux-any-to-ours @@ -0,0 +1,72 @@ +#!/bin/bash +# Releases 'any' packages from Arch arches to ours + +trap_exit() { + echo + error "$@" + exit 1 +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-archlinux.conf" +source "$(librelib messages)" + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +# The architecture to compare with +BASEARCH='x86_64' + +# Traverse all Arch repos +for _repo in "${ARCHREPOS[@]}"; do + msg "Processing %s..." "${_repo}" + + # Find 'any' packages + # This is hardcoded but it could release other arches... + PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + -iname '*-any.pkg.tar.?z' \ + -printf "%f ")) + + if [ ${#PKGS[@]} -eq 0 ]; then + msg2 "No '%s' packages here" any + continue + fi + + for _arch in "${OURARCHES[@]}"; do + msg2 "Syncing %s..." "${_arch}" + + # Sync 'any' only and extract the synced packages + SYNCED=($( + rsync -av \ + --include='*-any.pkg.tar.?z' \ + --include='*-any.pkg.tar.?z.sig' \ + --exclude='*' \ + "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \ + grep 'any\.pkg\.tar\..z$' | \ + cut -d ' ' -f 1 )) + + if [ ${#SYNCED[@]} -eq 0 ]; then + msg2 "Already synced (or error happened)" + continue + fi + + msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" + + msg2 "Adding to db..." + + pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null + + # Add the packages to the db + repo-add "${_repo}${DBEXT}" "${SYNCED[@]}" + + popd >/dev/null + + # Avoid mixups + unset SYNCED PKGS + done +done diff --git a/db-import-archlinux-pkg b/db-import-archlinux-pkg new file mode 100755 index 0000000..aae7862 --- /dev/null +++ b/db-import-archlinux-pkg @@ -0,0 +1,215 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * make a tarball of files used for forensics + +set -e + +# Run as `V=true db-import-pkg-archlinux` to get verbose output +VERBOSE=${V} +extra=() +${VERBOSE} && extra+=(-v) + +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + +# Returns contents of a repo +get_repos() { + # Exclude everything but db files + rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" +} + +get_repo_content() { + # Return all contents + bsdtar tf "${1}" | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + echo "${WORKDIR}/${1}/os/${2}/${1}" +} + +# Process the databases and get the libre packages +init() { + + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in "${ARCHREPOS[@]}"; do + for _arch in "${ARCHARCHES[@]}"; do + msg "Processing %s-%s" "${_repo}" "${_arch}" + + db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} + files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content "${db_file}")) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" + + msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync "${extra[@]}" --no-motd -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + "${WORKDIR}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists, check for single *s just in case + cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist + + msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + local pkgpool + for pkgpool in "${ARCHPKGPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ + "${FTP_BASE}/${pkgpool}/" + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + local srcpool + for srcpool in "${ARCHSRCPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ + "${FTP_BASE}/${srcpool}/" + done + + date -u +%s > "${FTP_BASE}/lastsync" + + # Cleanup + unset blacklist whitelists _arch _repo repo_file +} + +trap_exit() { + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" +} + +fatal_error() { + error "$@" + exit 1 +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-archlinux.conf" +source "$(librelib messages)" + +# Check variables presence +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do + test -z "${!var}" && fatal_error "Empty %s" "${var}" +done + +# From makepkg +set -E +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR + +init diff --git a/db-import-archlinux-src b/db-import-archlinux-src new file mode 100755 index 0000000..72171f1 --- /dev/null +++ b/db-import-archlinux-src @@ -0,0 +1,124 @@ +#!/bin/bash + +set -e + +FTP_BASE=/srv/repo/main +ABSLIBRE=/srv/abslibre +ABSGIT=/srv/git/abslibre/abslibre.git +# Remote +# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git +BLACKLIST=/home/repo/blacklist/blacklist.txt +SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' +BLFILE=/tmp/blacklist.txt + +# Variables from abs.conf +ABSROOT="/srv/abs/" +# DON'T CHANGE. WE NEED IT FOR ABSLIBRE +SYNCSERVER="rsync.archlinux.org" +ARCH="i686" +MIRRORLIST="/etc/pacman.d/mirrorlist" +REPOS=(core extra community testing community-testing !staging !community-staging) + +# Steps +# * Sync abs +# * Download blacklist.txt +# * Sync abslibre from abs excluding from blacklist +# * Create repo.abs.tar.gz tarballs + +function sync_abs() { + for ARCH in any i686 x86_64; do + rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? + done + + # fix some permissions + find "${ABSROOT}" -type d -print0 | xargs -0 chmod 755 + find "${ABSROOT}" -type f -print0 | xargs -0 chmod 644 +} + +function get_blacklist() { + printf ":: Updating blacklist...\t" + cat "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ + sed "s/^/**\//" > ${BLFILE} || { + printf "[FAILED]\n" + return 1 + } + + # Prevent using an empty blacklist + [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 + + printf "[OK]\n" +} + +function sync_abs_libre() { + + # Clone ABSLibre git repo + rm -rf /tmp/abslibre + git clone "$ABSGIT" /tmp/abslibre + + # Sync from ABS and then sync from ABSLibre + printf ":: Syncing ABSLibre...\t" + (rsync ${SYNCARGS} --delete-excluded \ + --exclude-from=${BLFILE} \ + ${ABSROOT} \ + ${ABSLIBRE} \ + && + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + printf "[FAILED]\n" + return 1 + } + + # fix some permissions + find "${ABSLIBRE}" -type d -print0 | xargs -0 chmod 755 + find "${ABSLIBRE}" -type f -print0 | xargs -0 chmod 644 + + printf "[OK]\n" +} + +# This part is very hacky and particular to the current setup :P +sync_pre_mips64el() { + pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null + + sudo -u fauno sh -c " + rsync ${SYNCARGS} \ + --exclude=.git* \ + --exclude=community-staging \ + --exclude=community-testing \ + --exclude=gnome-unstable \ + --exclude=kde-unstable \ + --exclude=multilib \ + --exclude=multilib-testing \ + --exclude=multilib-staging \ + --exclude=staging \ + --exclude=testing \ + ${ABSLIBRE}/x86_64/ \ + /home/fauno/Repos/abslibre-pre-mips64el/ && + git add . && + git commit -m \"$(date)\" -a + git push origin master + git gc + " +} + +# Create .abs.tar.gz tarballs +create_tarballs() { + for repo in ${ABSLIBRE}/{i686,x86_64}/*; do + baserepo=${repo##*/} + arch=$(basename $(dirname $repo)) + + # Remove the old one + mkdir -p $FTP_BASE/$baserepo/os/$arch/ + rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz + # Create a new one joining arch and any + # Remove the first part of the path (it could be $repo but any isn't hit) + bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ + $repo/* ${ABSLIBRE}/any/${baserepo}/* + + done +} + +sync_abs +get_blacklist +sync_abs_libre +#sync_pre_mips64el +create_tarballs diff --git a/db-import-archlinux.conf b/db-import-archlinux.conf new file mode 100644 index 0000000..525fc58 --- /dev/null +++ b/db-import-archlinux.conf @@ -0,0 +1,19 @@ +ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') +ARCHPKGPOOLS=(pool/{packages,community}) +ARCHSRCPOOLS=(sources/{packages,community}) +ARCHARCHES=(i686 x86_64) +OURARCHES=(armv7h) + +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" + +mirror="mirrors.kernel.org" + +## mirrors without sources folder +#mirror="mirrors.niyawe.de" +#mirror="mirror.nl.leaseweb.net" +#mirror="mirror.one.com" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.bytemark.co.uk" +#mirror="mirror.de.leaseweb.net" + +mirrorpath="archlinux" diff --git a/db-import-archlinuxarm-pkg b/db-import-archlinuxarm-pkg new file mode 100755 index 0000000..8d8cb3d --- /dev/null +++ b/db-import-archlinuxarm-pkg @@ -0,0 +1,204 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * make a tarball of files used for forensics + +set -e + +# Run as `V=true db-import-pkg-archlinux` to get verbose output +VERBOSE=${V} +extra=() +${VERBOSE} && extra+=(-v) + +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + +# Returns contents of a repo +get_repos() { + # Exclude everything but db files + rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" +} + +get_repo_content() { + # Return all contents + bsdtar tf "${1}" | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + echo "${WORKDIR}/${2}/${1}/${1}" +} + +# Process the databases and get the libre packages +init() { + + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _arch in "${OURARCHES[@]}"; do + for _repo in "${ARMREPOS[@]}"; do + msg "Processing %s-%s" "${_repo}" "${_arch}" + + db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} + files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this arch-repo" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this arch-repo" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content "${db_file}")) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" + + msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + # Sync excluding everything but whitelist + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ + "${FTP_BASE}/${PKGPOOLARM}/" + + msg "Putting databases back in place" + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + "${WORKDIR}/${_arch}/${_repo}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Cleanup + unset db + done + done + + + msg "Generating symbolic links to pool" + + for _arch in "${OURARCHES[@]}"; do + for _repo in "${ARMREPOS[@]}"; do + # Modify whitelist to search packages and create symlinks + sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist" + + msg "Putting symlinks in ${_repo}/os/${_arch}" + + while read _pkgfile; do + # Symlink to package + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" + elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}" + fi + + # Symlink to signature + if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" + elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" ]; then + ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" \ + "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig" + fi + done < "/tmp/${_repo}-${_arch}.whitelist" + done + done + + date -u +%s > "${FTP_BASE}/lastsync" + + # Cleanup + unset blacklist whitelists _arch _repo repo_file _pkgfile +} + +trap_exit() { + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" +} + +fatal_error() { + error "$@" + exit 1 +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-archlinuxarm.conf" +source "$(librelib messages)" + +# Check variables presence +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE; do + test -z "${!var}" && fatal_error "Empty %s" "${var}" +done + +# From makepkg +set -E +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR + +init diff --git a/db-import-archlinuxarm.conf b/db-import-archlinuxarm.conf new file mode 100644 index 0000000..4a12a25 --- /dev/null +++ b/db-import-archlinuxarm.conf @@ -0,0 +1,13 @@ +ARMREPOS=('core' 'extra' 'community') +PKGPOOLARM='pool/arch_gnu+linux_arm' +OURARCHES=(armv7h) + +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" + +#mirror="mirror.yandex.ru" +mirror="ftp.halifax.rwth-aachen.de" + +## mirrors without sources folder +## use "archlinuxarm" instead "archlinux-arm" to mirror.yandex.ru + +mirrorpath="archlinux-arm" diff --git a/db-import-pkg-archlinux b/db-import-pkg-archlinux deleted file mode 100755 index d20d095..0000000 --- a/db-import-pkg-archlinux +++ /dev/null @@ -1,215 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics - -set -e - -# Run as `V=true db-import-pkg-archlinux` to get verbose output -VERBOSE=${V} -extra=() -${VERBOSE} && extra+=(-v) - -WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT - -# Returns contents of a repo -get_repos() { - # Exclude everything but db files - rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" -} - -get_repo_content() { - # Return all contents - bsdtar tf "${1}" | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${WORKDIR}/${1}/os/${2}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "%d packages in blacklist" ${#blacklist[@]} - - test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in "${ARCHREPOS[@]}"; do - for _arch in "${ARCHARCHES[@]}"; do - msg "Processing %s-%s" "${_repo}" "${_arch}" - - db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} - files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - # Get db contents - db=($(get_repo_content "${db_file}")) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - # IMPORTANT: the . in the sed command is needed because an empty - # whitelist would consist of a single * allowing any package to - # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" - - msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync "${extra[@]}" --no-motd -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from="/tmp/${_repo}-${_arch}.whitelist" \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - "${WORKDIR}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists, check for single *s just in case - cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist - - msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local pkgpool - for pkgpool in "${ARCHPKGPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ - "${FTP_BASE}/${pkgpool}/" - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local srcpool - for srcpool in "${ARCHSRCPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ - "${FTP_BASE}/${srcpool}/" - done - - date -u +%s > "${FTP_BASE}/lastsync" - - # Cleanup - unset blacklist whitelists _arch _repo repo_file -} - -trap_exit() { - local signal=$1; shift - echo - error "$@" - trap -- "$signal" - kill "-$signal" "$$" -} - -fatal_error() { - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-import-pkg-archlinux.conf" -source "$(librelib messages)" - -# Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do - test -z "${!var}" && fatal_error "Empty %s" "${var}" -done - -# From makepkg -set -E -for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" -done -trap 'trap_exit INT "Aborted by user! Exiting..."' INT -trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR - -init diff --git a/db-import-pkg-archlinux.conf b/db-import-pkg-archlinux.conf deleted file mode 100644 index 525fc58..0000000 --- a/db-import-pkg-archlinux.conf +++ /dev/null @@ -1,19 +0,0 @@ -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') -ARCHPKGPOOLS=(pool/{packages,community}) -ARCHSRCPOOLS=(sources/{packages,community}) -ARCHARCHES=(i686 x86_64) -OURARCHES=(armv7h) - -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" - -mirror="mirrors.kernel.org" - -## mirrors without sources folder -#mirror="mirrors.niyawe.de" -#mirror="mirror.nl.leaseweb.net" -#mirror="mirror.one.com" -#mirror="mirror.us.leaseweb.net" -#mirror="mirror.bytemark.co.uk" -#mirror="mirror.de.leaseweb.net" - -mirrorpath="archlinux" diff --git a/db-import-pkg-archlinuxarm b/db-import-pkg-archlinuxarm deleted file mode 100755 index 536b603..0000000 --- a/db-import-pkg-archlinuxarm +++ /dev/null @@ -1,204 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics - -set -e - -# Run as `V=true db-import-pkg-archlinux` to get verbose output -VERBOSE=${V} -extra=() -${VERBOSE} && extra+=(-v) - -WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT - -# Returns contents of a repo -get_repos() { - # Exclude everything but db files - rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" -} - -get_repo_content() { - # Return all contents - bsdtar tf "${1}" | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${WORKDIR}/${2}/${1}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "%d packages in blacklist" ${#blacklist[@]} - - test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _arch in "${OURARCHES[@]}"; do - for _repo in "${ARMREPOS[@]}"; do - msg "Processing %s-%s" "${_repo}" "${_arch}" - - db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} - files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this arch-repo" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this arch-repo" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - # Get db contents - db=($(get_repo_content "${db_file}")) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - # IMPORTANT: the . in the sed command is needed because an empty - # whitelist would consist of a single * allowing any package to - # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" - - msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - # Sync excluding everything but whitelist - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from="/tmp/${_repo}-${_arch}.whitelist" \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ - "${FTP_BASE}/${PKGPOOLARM}/" - - msg "Putting databases back in place" - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - "${WORKDIR}/${_arch}/${_repo}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Cleanup - unset db - done - done - - - msg "Generating symbolic links to pool" - - for _arch in "${OURARCHES[@]}"; do - for _repo in "${ARMREPOS[@]}"; do - # Modify whitelist to search packages and create symlinks - sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist" - - msg "Putting symlinks in ${_repo}/os/${_arch}" - - while read _pkgfile; do - # Symlink to package - if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}" - elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}" - fi - - # Symlink to signature - if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig" - elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" ]; then - ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" \ - "${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig" - fi - done < "/tmp/${_repo}-${_arch}.whitelist" - done - done - - date -u +%s > "${FTP_BASE}/lastsync" - - # Cleanup - unset blacklist whitelists _arch _repo repo_file _pkgfile -} - -trap_exit() { - local signal=$1; shift - echo - error "$@" - trap -- "$signal" - kill "-$signal" "$$" -} - -fatal_error() { - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-import-pkg-archlinuxarm.conf" -source "$(librelib messages)" - -# Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE; do - test -z "${!var}" && fatal_error "Empty %s" "${var}" -done - -# From makepkg -set -E -for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" -done -trap 'trap_exit INT "Aborted by user! Exiting..."' INT -trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR - -init diff --git a/db-import-pkg-archlinuxarm.conf b/db-import-pkg-archlinuxarm.conf deleted file mode 100644 index 4a12a25..0000000 --- a/db-import-pkg-archlinuxarm.conf +++ /dev/null @@ -1,13 +0,0 @@ -ARMREPOS=('core' 'extra' 'community') -PKGPOOLARM='pool/arch_gnu+linux_arm' -OURARCHES=(armv7h) - -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" - -#mirror="mirror.yandex.ru" -mirror="ftp.halifax.rwth-aachen.de" - -## mirrors without sources folder -## use "archlinuxarm" instead "archlinux-arm" to mirror.yandex.ru - -mirrorpath="archlinux-arm" diff --git a/db-import-src-archlinux b/db-import-src-archlinux deleted file mode 100755 index 72171f1..0000000 --- a/db-import-src-archlinux +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -set -e - -FTP_BASE=/srv/repo/main -ABSLIBRE=/srv/abslibre -ABSGIT=/srv/git/abslibre/abslibre.git -# Remote -# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git -BLACKLIST=/home/repo/blacklist/blacklist.txt -SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' -BLFILE=/tmp/blacklist.txt - -# Variables from abs.conf -ABSROOT="/srv/abs/" -# DON'T CHANGE. WE NEED IT FOR ABSLIBRE -SYNCSERVER="rsync.archlinux.org" -ARCH="i686" -MIRRORLIST="/etc/pacman.d/mirrorlist" -REPOS=(core extra community testing community-testing !staging !community-staging) - -# Steps -# * Sync abs -# * Download blacklist.txt -# * Sync abslibre from abs excluding from blacklist -# * Create repo.abs.tar.gz tarballs - -function sync_abs() { - for ARCH in any i686 x86_64; do - rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? - done - - # fix some permissions - find "${ABSROOT}" -type d -print0 | xargs -0 chmod 755 - find "${ABSROOT}" -type f -print0 | xargs -0 chmod 644 -} - -function get_blacklist() { - printf ":: Updating blacklist...\t" - cat "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ - sed "s/^/**\//" > ${BLFILE} || { - printf "[FAILED]\n" - return 1 - } - - # Prevent using an empty blacklist - [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 - - printf "[OK]\n" -} - -function sync_abs_libre() { - - # Clone ABSLibre git repo - rm -rf /tmp/abslibre - git clone "$ABSGIT" /tmp/abslibre - - # Sync from ABS and then sync from ABSLibre - printf ":: Syncing ABSLibre...\t" - (rsync ${SYNCARGS} --delete-excluded \ - --exclude-from=${BLFILE} \ - ${ABSROOT} \ - ${ABSLIBRE} \ - && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { - printf "[FAILED]\n" - return 1 - } - - # fix some permissions - find "${ABSLIBRE}" -type d -print0 | xargs -0 chmod 755 - find "${ABSLIBRE}" -type f -print0 | xargs -0 chmod 644 - - printf "[OK]\n" -} - -# This part is very hacky and particular to the current setup :P -sync_pre_mips64el() { - pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null - - sudo -u fauno sh -c " - rsync ${SYNCARGS} \ - --exclude=.git* \ - --exclude=community-staging \ - --exclude=community-testing \ - --exclude=gnome-unstable \ - --exclude=kde-unstable \ - --exclude=multilib \ - --exclude=multilib-testing \ - --exclude=multilib-staging \ - --exclude=staging \ - --exclude=testing \ - ${ABSLIBRE}/x86_64/ \ - /home/fauno/Repos/abslibre-pre-mips64el/ && - git add . && - git commit -m \"$(date)\" -a - git push origin master - git gc - " -} - -# Create .abs.tar.gz tarballs -create_tarballs() { - for repo in ${ABSLIBRE}/{i686,x86_64}/*; do - baserepo=${repo##*/} - arch=$(basename $(dirname $repo)) - - # Remove the old one - mkdir -p $FTP_BASE/$baserepo/os/$arch/ - rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz - # Create a new one joining arch and any - # Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ - -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ - $repo/* ${ABSLIBRE}/any/${baserepo}/* - - done -} - -sync_abs -get_blacklist -sync_abs_libre -#sync_pre_mips64el -create_tarballs -- cgit v1.2.3 From b56bcf5030f7eb8af196edb0aabb6bf75e5283ec Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 13:55:57 -0400 Subject: oops --- config | 1 - 1 file changed, 1 deletion(-) diff --git a/config b/config index 38cdae5..ec56c65 100644 --- a/config +++ b/config @@ -20,7 +20,6 @@ PKGPOOLS=(pool/parabola_gnu+linux # Parabola GNU/Linux-libre pool/{packages,community} # Arch Linux ) # Directories where sources are stored -OURPKGPOOLS=( SRCPOOLS=(src/parabola_gnu+linux # Parabola GNU/Linux-libre src/arch_gnu+linux_arm # Arch Linux ARM sources/{packages,community} # Arch Linux -- cgit v1.2.3 From f3add3749351c9070e0fbe43f37b5a94ac1e7123 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 13:58:19 -0400 Subject: more config file cleanup --- config | 12 ------------ cron-jobs/db-cleanup | 1 + cron-jobs/db-cleanup.conf | 11 +++++++++++ 3 files changed, 12 insertions(+), 12 deletions(-) create mode 100644 cron-jobs/db-cleanup.conf diff --git a/config b/config index ec56c65..041c9eb 100644 --- a/config +++ b/config @@ -13,18 +13,6 @@ PKGREPOS=( PKGPOOL='pool/parabola_gnu+linux' SRCPOOL='src/parabola_gnu+linux' -# Directories where packages are shared between repos -# *relative to FTP_BASE* -PKGPOOLS=(pool/parabola_gnu+linux # Parabola GNU/Linux-libre - pool/arch_gnu+linux_arm # Arch Linux ARM - pool/{packages,community} # Arch Linux - ) -# Directories where sources are stored -SRCPOOLS=(src/parabola_gnu+linux # Parabola GNU/Linux-libre - src/arch_gnu+linux_arm # Arch Linux ARM - sources/{packages,community} # Arch Linux - ) - CLEANUP_DESTDIR="$FTP_BASE/old/pkg" CLEANUP_DRYRUN=false # Time in days to keep moved packages diff --git a/cron-jobs/db-cleanup b/cron-jobs/db-cleanup index ce0e5f7..12d3615 100755 --- a/cron-jobs/db-cleanup +++ b/cron-jobs/db-cleanup @@ -16,6 +16,7 @@ trap_exit() { } source "$(dirname $(dirname "$(readlink -e "$0")"))/config" +source "$(dirname "$(readlink -e "$0")")/db-cleanup.conf" source "$(librelib messages)" # From makepkg diff --git a/cron-jobs/db-cleanup.conf b/cron-jobs/db-cleanup.conf new file mode 100644 index 0000000..7c2344c --- /dev/null +++ b/cron-jobs/db-cleanup.conf @@ -0,0 +1,11 @@ +# Directories where packages are shared between repos +# *relative to FTP_BASE* +PKGPOOLS=(pool/parabola_gnu+linux # Parabola GNU/Linux-libre + pool/arch_gnu+linux_arm # Arch Linux ARM + pool/{packages,community} # Arch Linux + ) +# Directories where sources are stored +SRCPOOLS=(src/parabola_gnu+linux # Parabola GNU/Linux-libre + src/arch_gnu+linux_arm # Arch Linux ARM + sources/{packages,community} # Arch Linux + ) -- cgit v1.2.3 From 0d7d26fb58525411847795e65ce4ce6260349732 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 14:57:38 -0400 Subject: tidy --- cron-jobs/check_archlinux/check_packages.py | 2 +- cron-jobs/db-update-mailer | 26 -------------------------- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 cron-jobs/db-update-mailer diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index ac0194f..d233bf6 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/python2 # # check_archlinux.py # diff --git a/cron-jobs/db-update-mailer b/cron-jobs/db-update-mailer deleted file mode 100644 index 35ff204..0000000 --- a/cron-jobs/db-update-mailer +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Dummy helper to send email to parabola-dev -# It does nothing if no output -# Aurélien DESBRIERES -# GPL v3 or later. -# testing version !!! - -LIST="maintenance@lists.parabolagnulinux.org" -FROM="maintenance@lists.parabolagnulinux.org" - -SUBJECT="Database Updated $(date +"%d-%m-%Y")" -if [ $db-update 1 ]; then - SUBJECT="$1 $(date +"%d-%m-%Y")" -fi - -stdin="$(cat)" -#echo used to strip whitespace for checking for actual data -if [ -n "$(echo $stdin)" ]; then - -echo "Subject: $SUBJECT -To: $LIST -From: $FROM - -$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" - -fi -- cgit v1.2.3