diff options
41 files changed, 692 insertions, 1137 deletions
diff --git a/abslibre b/abslibre deleted file mode 100755 index 6b21d24..0000000 --- a/abslibre +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - -set -e - -FTP_BASE=/srv/repo/main -ABSLIBRE=/srv/abslibre -ABSGIT=/srv/git/abslibre/abslibre.git -# Remote -# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git -BLACKLIST=/home/repo/blacklist/blacklist.txt -SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' -BLFILE=/tmp/blacklist.txt - -# Variables from abs.conf -ABSROOT="/srv/abs/" -# DON'T CHANGE. WE NEED IT FOR ABSLIBRE -SYNCSERVER="rsync.archlinux.org" -ARCH="i686" -MIRRORLIST="/etc/pacman.d/mirrorlist" -REPOS=(core extra community testing community-testing !staging !community-staging) - -# Steps -# * Sync abs -# * Download blacklist.txt -# * Sync abslibre from abs excluding from blacklist -# * Create repo.abs.tar.gz tarballs - -function sync_abs() { - for ARCH in any i686 x86_64; do - rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? - done - - # fix some permissions - find "${ABSROOT}" -type d -print0 | xargs -0 chmod 755 - find "${ABSROOT}" -type f -print0 | xargs -0 chmod 644 -} - -function get_blacklist() { - printf ":: Updating blacklist...\t" - cat "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ - sed "s/^/**\//" > ${BLFILE} || { - printf "[FAILED]\n" - return 1 - } - - # Prevent using an empty blacklist - [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 - - printf "[OK]\n" -} - -function sync_abs_libre() { - - # Clone ABSLibre git repo - if [ -d /tmp/abslibre/.git ]; then - pushd /tmp/abslibre >/dev/null 2>&1 - git pull - popd >/dev/null 2>&1 - else - git clone "$ABSGIT" /tmp/abslibre - fi - - # Sync from ABS and then sync from ABSLibre - printf ":: Syncing ABSLibre...\t" - (rsync ${SYNCARGS} --delete-excluded \ - --exclude-from=${BLFILE} \ - ${ABSROOT} \ - ${ABSLIBRE} \ - && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { - printf "[FAILED]\n" - return 1 - } - - # fix some permissions - find "${ABSLIBRE}" -type d -print0 | xargs -0 chmod 755 - find "${ABSLIBRE}" -type f -print0 | xargs -0 chmod 644 - - printf "[OK]\n" -} - -# This part is very hacky and particular to the current setup :P -sync_pre_mips64el() { - pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null - - sudo -u fauno sh -c " - rsync ${SYNCARGS} \ - --exclude=.git* \ - --exclude=community-staging \ - --exclude=community-testing \ - --exclude=gnome-unstable \ - --exclude=kde-unstable \ - --exclude=multilib \ - --exclude=multilib-testing \ - --exclude=multilib-staging \ - --exclude=staging \ - --exclude=testing \ - ${ABSLIBRE}/x86_64/ \ - /home/fauno/Repos/abslibre-pre-mips64el/ && - git add . && - git commit -m \"$(date)\" -a - git push origin master - git gc - " -} - -# Create .abs.tar.gz tarballs -create_tarballs() { - for repo in ${ABSLIBRE}/{i686,x86_64}/*; do - baserepo=${repo##*/} - arch=$(basename $(dirname $repo)) - - # Remove the old one - mkdir -p $FTP_BASE/$baserepo/os/$arch/ - rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz - # Create a new one joining arch and any - # Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ - -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ - $repo/* ${ABSLIBRE}/any/${baserepo}/* - - done -} - -sync_abs -get_blacklist -sync_abs_libre -#sync_pre_mips64el -create_tarballs diff --git a/any-to-ours b/any-to-ours deleted file mode 100755 index a901d54..0000000 --- a/any-to-ours +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -# Releases 'any' packages from Arch arches to ours - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -# The architecture to compare with -BASEARCH='x86_64' - -# Traverse all Arch repos -for _repo in "${ARCHREPOS[@]}"; do - msg "Processing %s..." "${_repo}" - - # Find 'any' packages - # This is hardcoded but it could release other arches... - PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - -iname '*-any.pkg.tar.?z' \ - -printf "%f ")) - - if [ ${#PKGS[@]} -eq 0 ]; then - msg2 "No '%s' packages here" any - continue - fi - - for _arch in "${OURARCHES[@]}"; do - msg2 "Syncing %s..." "${_arch}" - - # Sync 'any' only and extract the synced packages - SYNCED=($( - rsync -av \ - --include='*-any.pkg.tar.?z' \ - --include='*-any.pkg.tar.?z.sig' \ - --exclude='*' \ - "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \ - grep 'any\.pkg\.tar\..z$' | \ - cut -d ' ' -f 1 )) - - if [ ${#SYNCED[@]} -eq 0 ]; then - msg2 "Already synced (or error happened)" - continue - fi - - msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" - - msg2 "Adding to db..." - - pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null - - # Add the packages to the db - repo-add "${_repo}${DBEXT}" "${SYNCED[@]}" - - popd >/dev/null - - # Avoid mixups - unset SYNCED PKGS - done -done @@ -1,39 +1,25 @@ #!/hint/bash +case "$USER" in + db-import-packages) _name=import-packages;; + db-import-community) _name=import-community;; + *) _name=parabola;; +esac + FTP_BASE="/srv/repo/main" +PKGREPOS=() +PKGPOOL='' +SRCPOOL='' -# Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') -# Official Parabola repos -OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') -# User repos -USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') -# Community project repos -PROJREPOS=('nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') -# Remote repos -PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") -PKGPOOL='pool/parabola' -SRCPOOL='sources/parabola' - -# Directories where packages are shared between repos -# *relative to FTP_BASE* -ARCHPKGPOOLS=(pool/{packages,community}) -OURPKGPOOLS=(pool/parabola) -PKGPOOLS=(${OURPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) -# Directories where sources are stored -ARCHSRCPOOLS=(sources/{packages,community}) -OURPKGPOOLS=(sources/parabola) -SRCPOOLS=(${OURSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) - -CLEANUP_DESTDIR="$FTP_BASE/old/packages" +CLEANUP_DESTDIR="/srv/repo/private/${_name}/package-cleanup" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" -SOURCE_CLEANUP_DRYRUN=true +SOURCE_CLEANUP_DESTDIR="/srv/repo/private/${_name}/source-cleanup" +SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=30 +SOURCE_CLEANUP_KEEP=14 REQUIRE_SIGNATURE=true @@ -41,17 +27,22 @@ LOCK_DELAY=10 LOCK_TIMEOUT=300 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" -TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) -OURARCHES=() -ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) +export TMPDIR="${TMPDIR:-/tmp}" +ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" -MAKEPKGCONF="~/.makepkg.conf" -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" +# Allowed licenses: get sourceballs only for licenses in this array +# Empty (commented out) to get sourceballs for all packages +#ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') + +# Where to send error emails, and who they are from +LIST="dev@lists.parabola.nu" +FROM="dbscripts+${_name}@$(hostname -f)" -# parabolaweb root -WEB_DIR=/srv/http/parabolagnulinux.org/web +# Override default config with config.local +[ -f "$(dirname "${BASH_SOURCE[0]}")/config.local" ] && . "$(dirname "${BASH_SOURCE[0]}")/config.local" +[ -f "$(dirname "${BASH_SOURCE[0]}")/config.local.${_name}" ] && . "$(dirname "${BASH_SOURCE[0]}")/config.local.${_name}" +unset _name diff --git a/config.local.import-community b/config.local.import-community new file mode 100644 index 0000000..393fd57 --- /dev/null +++ b/config.local.import-community @@ -0,0 +1,5 @@ +#!/hint/bash + +PKGREPOS=({community,multilib}{,-testing,-staging}) +PKGPOOL='pool/community' +SRCPOOL='sources/community' diff --git a/config.local.import-packages b/config.local.import-packages new file mode 100644 index 0000000..c699b28 --- /dev/null +++ b/config.local.import-packages @@ -0,0 +1,5 @@ +#!/hint/bash + +PKGREPOS=('core' 'extra' 'testing' 'staging' {kde,gnome}-unstable) +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' diff --git a/config.local.parabola b/config.local.parabola new file mode 100644 index 0000000..648dfbb --- /dev/null +++ b/config.local.parabola @@ -0,0 +1,14 @@ +#!/hint/bash + +PKGREPOS=( + # Main repos + libre{,-testing} + libre-multilib{,-testing} + # Community project repos + {nonsystemd,nonprism}{,-testing} + pcr kernels cross java + # User repos + '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' +) +PKGPOOL='pool/parabola' +SRCPOOL='sources/parabola' diff --git a/config.local.svn-community b/config.local.svn-community new file mode 100644 index 0000000..105ea66 --- /dev/null +++ b/config.local.svn-community @@ -0,0 +1,11 @@ +PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') +PKGPOOL='pool/community' +SRCPOOL='sources/community' +SVNREPO='file:///srv/repos/svn-community/svn' +SVNUSER='svn-community' +TESTING_REPO='community-testing' +STABLE_REPOS=('community') + +CLEANUP_DESTDIR="/srv/repos/svn-community/package-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-community/source-cleanup" +TMPDIR="/srv/repos/svn-community/tmp" diff --git a/config.local.svn-packages b/config.local.svn-packages new file mode 100644 index 0000000..958a483 --- /dev/null +++ b/config.local.svn-packages @@ -0,0 +1,11 @@ +PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' +SVNREPO='file:///srv/repos/svn-packages/svn' +SVNUSER='svn-packages' +TESTING_REPO='testing' +STABLE_REPOS=('core' 'extra') + +CLEANUP_DESTDIR="/srv/repos/svn-packages/package-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup" +TMPDIR="/srv/repos/svn-packages/tmp" diff --git a/create-repo b/create-repo deleted file mode 100755 index 3feb098..0000000 --- a/create-repo +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Creates repository structure - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -eq 0 ]; then - msg "Usage: %s repo1 [repo2 ... repoX]" "${0##*/}" - exit 1 -fi - -msg "Creating repos..." -for _repo in "$@"; do - msg2 "Creating [%s]" "${_repo}" - for _arch in "${ARCHES[@]}"; do - mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ - error "Failed creating %s dir" "${_arch}" - done -done - -msg "Don't forget to add them to the PKGREPOS array on %s" "$(dirname "$(readlink -e "$0")")/config" diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 1a05521..7f298b9 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -2,9 +2,8 @@ #Dummy helper to send email to arch-dev # It does nothing if no output -LIST="arch-dev-public@archlinux.org" -#LIST="aaronmgriffin@gmail.com" -FROM="repomaint@archlinux.org" +# Load $LIST and $FROM from the config file +. "$(dirname "$(readlink -e "$0")")/../config" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if [ $# -ge 1 ]; then diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 4a2b418..4063c09 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -22,6 +22,14 @@ clean_pkg() { fi } +script_lock + +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 + done +done + "${CLEANUP_DRYRUN}" && warning 'dry run mode is active' for repo in "${PKGREPOS[@]}"; do @@ -56,7 +64,7 @@ done # get a list of all available packages in the pacakge pool find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" # create a list of packages in our db -cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" +find "${WORKDIR}" -maxdepth 1 -type f -name 'db-*' -exec cat {} \; | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if [ ${#old_pkgs[@]} -ge 1 ]; then diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check deleted file mode 100755 index 239f042..0000000 --- a/cron-jobs/repo-sanity-check +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Solves issue165... on the old roundup install. From the database -# backups, the title was "Older/deprecated packages never leave the -# repo", I don't know how the body of the issue is stored in the DB, -# but the title says enough, I think. - -. "$(dirname "$(readlink -e "$0")")/../config" -. "$(dirname "$(readlink -e "$0")")/../db-functions" - -# Traverse all repos -for _repo in "${PKGREPOS[@]}"; do - msg "Cleaning up [%s]" "${_repo}" - - # Find all pkgnames on this repo's abs - on_abs=($( - find "${SVNREPO}/${_repo}" -name PKGBUILD | \ - while read pkgbuild; do - source "${pkgbuild}" >/dev/null 2>&1 - # cleanup to save memory - unset build package source md5sums pkgdesc pkgver pkgrel epoch \ - url license arch depends makedepends optdepends options \ - >/dev/null 2>&1 - - # also cleanup package functions - for _pkg in "${pkgname[@]}"; do - unset "package_${pkg}" >/dev/null 2>&1 - done - - # this fills the on_abs array - echo "${pkgname[@]}" - done - )) - - # quit if abs is empty - if [ ${#on_abs[*]} -eq 0 ]; then - warning "[%s]'s ABS tree is empty, skipping" "${_repo}" - break - fi - - # Find all pkgnames on repos - on_repo=($( - find "${FTP_BASE}/${_repo}" -name "*.pkg.tar.?z" \ - -printf "%f\n" | sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" - )) - - # Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 \ - <(printf '%s\n' "${on_abs[@]}" | sort -u) \ - <(printf '%s\n' "${on_repo[@]}" | sort -u) )) - - # Remove them from databases, ftpdir-cleanup will take care of the rest - find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \ - -exec repo-remove {} "${remove[@]}" \; >/dev/null 2>&1 - - msg2 "Removed the following packages:" - plain '%s' "${remove[@]}" -done diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index c12a128..c02912a 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -63,11 +63,10 @@ for repo in "${PKGREPOS[@]}"; do if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi - # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package -# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then -# continue -# fi + if ! ([[ -z ${ALLOWED_LICENSES[*]} ]] || chk_license "${pkglicense[@]}" || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + continue + fi # Store the expected file name of the source package echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" @@ -118,8 +117,8 @@ for repo in "${PKGREPOS[@]}"; do done # Cleanup old source packages -cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'expected-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'available-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/available-src-pkgs.sort" old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) if [ ${#old_pkgs[@]} -ge 1 ]; then diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 14d6f4b..0e1731c 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1,14 +1,28 @@ -nexuiz-data +0ad-data +alienarena-data +blobwars-data +btanks-data +dangerdeep-data +egoboo-data +fillets-ng-data +flightgear-data +frogatto-data +gcompris-data +naev-data +openarena-data +rocksndiamonds-data +smc-data +speed-dreams-data torcs-data tremulous-data ufoai-data -frogatto-data vdrift-data -naev-data -btanks-data +warmux-data wesnoth-data -texlive-bin +widelands-data +xonotic-data texlive-bibtexextra +texlive-bin texlive-core texlive-fontsextra texlive-formatsextra diff --git a/db-check-nonfree b/db-check-nonfree deleted file mode 100755 index 37b7cf6..0000000 --- a/db-check-nonfree +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -ge 1 ]; then - error "Calling %s with a specific repository is not supported" "${0##*/}" - exit 1 -fi - -# TODO: this might lock too much (architectures) -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${pkgarch}" || exit 1 - done -done - -msg "Check nonfree in repo:" -nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u)) -for repo in "${ARCHREPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - msg2 "%s %s" "$repo" "$pkgarch" - if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then - continue - fi - unset dbpkgs - unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in "${dbpkgs[@]}"; do - if in_array "${pkgname}" "${nonfree[@]}"; then - cleanpkgs+=("${pkgname}") - fi - done - if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: %s" "${cleanpkgs[*]}" - arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" - fi - done -done - -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${pkgarch}" - done -done diff --git a/check-package-libraries.py b/db-check-package-libraries.py index bc2349b..bc2349b 100755 --- a/check-package-libraries.py +++ b/db-check-package-libraries.py diff --git a/db-cleanup b/db-cleanup deleted file mode 100755 index ffa2601..0000000 --- a/db-cleanup +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Syncs pools against themselves using database contents as filter to cleanup -# them up -# License: GPLv3 - -# Principles -# * Get repos dbs contents -# * Make them a include list -# * Rsync pools against themselves removing excluded files -# * Instant cleanup! - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -EXTRAFLAGS=() -"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) - -filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") -trap "rm -f -- $(printf %q "$filter")" EXIT - -for _repo in "${PKGREPOS[@]}"; do - for _arch in "${ARCHES[@]}"; do - msg "Getting %s-%s database" "${_repo}" "${_arch}" - - dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" - - if [ ! -r "${dbfile}" ]; then - warning "Not found" - continue - fi - - # Echo the contents into a filter file - bsdtar tf "${dbfile}" | \ - cut -d'/' -f1 | \ - sort -u | \ - sed "s|$|*|" >> "$filter" - - done -done - -msg "Removing old files:" - -for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do - msg2 '%s' "${POOL}" - - rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ - --include-from="$filter" \ - --exclude="*" \ - "${FTP_BASE}/${POOL}/" \ - "${FTP_BASE}/${POOL}/" -done - -msg "Removing dead symlinks:" -actions=(-print) -"${CLEANUP_DRYRUN}" || actions+=(-delete) -find -L "${FTP_BASE}/" -type l "${actions[@]}" diff --git a/db-functions b/db-functions index d76aa41..62260bb 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,7 @@ #!/hint/bash # Some PKGBUILDs need CARCH to be set -CARCH="x86_64" +CARCH=$(. "$(librelib conf.sh)"; load_files makepkg; echo "$CARCH") # Useful functions UMASK="" @@ -26,73 +26,21 @@ mv_acl() { # set up general environment WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +if [ -n "${SVNUSER}" ]; then + setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" + setfacl -m d:u:"${USER}":rwx "${WORKDIR}" + setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" +fi LOCKS=() REPO_MODIFIED=0 -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW -if [[ -t 2 ]]; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" -fi -readonly ALL_OFF BOLD BLUE GREEN RED YELLOW - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> WARNING:${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> ERROR${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -## -# usage : in_array( $needle, $haystack ) -# return : 0 - found -# 1 - not found -## -in_array() { - local needle=$1; shift - [[ -z $1 ]] && return 1 # Not Found - local item - for item in "$@"; do - [[ $item = "$needle" ]] && return 0 # Found - done - return 1 # Not Found -} - -## -# usage : get_full_version( $epoch, $pkgver, $pkgrel ) -# return : full version spec, including epoch (if necessary), pkgver, pkgrel -## -get_full_version() { - if [[ $1 -eq 0 ]]; then - # zero epoch case, don't include it in version - echo "$2-$3" - else - echo "$1:$2-$3" - fi -} +# Used: plain, msg, msg2, warning, error, in_array, get_full_version, abort, die +# Overwritten: cleanup +# Ignored: stat_busy, stat_done, +# setup_workdir, trap_abort, trap_exit, +# lock, slock, lock_close +# pkgver_equal, find_cached_package, check_root +. "$(librelib common)" script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" @@ -141,22 +89,11 @@ cleanup() { if (( REPO_MODIFIED )); then date +%s > "${FTP_BASE}/lastupdate" - date -u +%s > "${FTP_BASE}/lastsync" fi [ "$1" ] && exit "$1" } -abort() { - msg 'Aborting...' - cleanup 0 -} - -die() { - error "$@" - cleanup 1 -} - trap abort INT QUIT TERM HUP trap cleanup EXIT @@ -293,6 +230,13 @@ getpkgarch() { echo "$_ver" } +check_packager() { + local _packager + + _packager=$(_grep_pkginfo "$1" "packager") + [[ $_packager && $_packager != 'Unknown Packager' ]] +} + getpkgfile() { if [[ ${#} -ne 1 ]]; then error 'No canonical package found!' @@ -384,7 +328,6 @@ check_splitpkgs() { for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase "${pkgfile}")" - msg2 "Checking %s" "$_pkgbase" local _pkgname="$(getpkgname "${pkgfile}")" local _pkgarch="$(getpkgarch "${pkgfile}")" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" @@ -422,17 +365,6 @@ check_pkgrepos() { [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 - local repo - local arch - for repo in "${PKGREPOS[@]}"; do - for arch in "${ARCHES[@]}"; do - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}.sig" ] && return 1 - done - done - return 0 } @@ -492,7 +424,7 @@ arch_repo_add() { printf -v pkgs_str -- '%q ' "${pkgs[@]}" # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" \ || error 'repo-add %q %s' "${repo}${DBEXT}" "${pkgs_str% }" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ || error 'repo-add -f %q %s' "${repo}${FILESEXT}" "${pkgs_str% }" @@ -514,7 +446,7 @@ arch_repo_remove() { return 1 fi printf -v pkgs_str -- '%q ' "${pkgs[@]}" - /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" \ || error 'repo-remove %q %s' "${dbfile}" "${pkgs_str% }" /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ || error 'repo-remove %q %s' "${filesfile}" "${pkgs_str% }" diff --git a/db-import b/db-import new file mode 100755 index 0000000..a8a073d --- /dev/null +++ b/db-import @@ -0,0 +1,288 @@ +#!/bin/bash +set -euE +# Imports Arch-like repos, running them through a blacklist +# License: GPLv3 + +. "$(dirname "$(readlink -e "$0")")/config" # for: FTP_BASE DBEXT +. "$(dirname "$(readlink -e "$0")")/db-import.conf" # for: IMPORTDIR IMPORTS +. "$(librelib messages)" +. "$(librelib blacklist)" + +# DBs = pacman DataBases + +# This replaces two scripts: +# - abslibre : imported ABS tree from Arch +# - db-sync : imported pacman DBs from Arch + +# The flow here is: +# 1. "${IMPORTDIR}/cache/${name}/dbs/" # Download the pacman databases +# 2. "${IMPORTDIR}/cache/${name}/abs/" # Download the ABS tree +# 3. "${IMPORTDIR}/clean/${name}/dbs/" # Run the pacman DBs through the blacklist +# 4. "${IMPORTDIR}/clean/${name}/pkgs/" # Download the pkg files mentioned in "clean/${name}/dbs/" +# 5. "${IMPORTDIR}/staging/${tag}" # Copy all the package files we just downloaded to here +# 6. Run `db-update on` with STAGING="${IMPORTDIR}/staging/${tag}" + +# generic arguments to pass to rsync, borrowed from `abs` +SYNCARGS='-mrtvlH --no-motd --no-p --no-o --no-g' + +main() { + blacklist-update + + local importStr + for importStr in "${IMPORTS[@]}"; do + local importAry=($importStr) + local name=${importAry[0]} + local pkgmirror=${importAry[1]} + local absmirror=${importAry[2]} + local tags=("${importAry[@]:3}") + + msg "Fetching remote package source: %s" "$name" + fetch_dbs "$name" "$pkgmirror" + fetch_abs "$name" "$absmirror" "${tags[@]}" + msg "Filtering blacklisted packages from remote package source: %s" "$name" + clean_dbs "$name" "${tags[@]}" + fetch_pkgs "$name" "${tags[@]}" + msg "Publishing changes from remote package source: %s" "$name" + publish "$name" "${tags[@]}" + done + return $r +} + +fetch_dbs() { + local name=$1 + local pkgmirror=$2 + + msg2 'Synchronizing package databases...' + + mkdir -p -- "${IMPORTDIR}/cache/${name}/dbs" + # Grab just the .db files from $pkgmirror + rsync $SYNCARGS --delete-after \ + --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --exclude="*" \ + "rsync://${pkgmirror}/" "${IMPORTDIR}/cache/${name}/dbs" +} + +fetch_abs() { + local name=$1 + local absmirror=$2 + local tags=("${@:3}") + + local fake_home + local absroot + + # Sync the ABS tree from $absmirror + local arch + for arch in $(list_arches "${tags[@]}"); do + msg2 'Synchronizing %s ABS tree...' "$arch" + + absroot="${IMPORTDIR}/cache/${name}/abs/${arch}" + mkdir -p -- "$absroot" + + # Configure `abs` for this mirror + fake_home="${IMPORTDIR}/homes/${name}/${arch}" + mkdir -p -- "$fake_home" + { + printf "ABSROOT='%s'\n" "$absroot" + printf "SYNCSERVER='%s'\n" "$absmirror" + printf "ARCH='%s'\n" "$arch" + printf 'REPOS=(\n' + list_repos "$arch" "${tags[@]}" + printf ')\n' + } > "${fake_home}/.abs.conf" + + # Run `abs` + HOME=$fake_home abs + done +} + +clean_dbs() { + local name=$1 + local tags=("${@:2}") + + rm -rf -- "${IMPORTDIR}/clean/$name" + + local tag + for tag in "${tags[@]}"; do + msg2 'Creating clean version of %s package database...' "$tag" + + local cache="${IMPORTDIR}/cache/$name/dbs/$(db_file "$tag")" + local clean="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + install -Dm644 "$cache" "$clean" + + blacklist-cat | blacklist-get-pkg | xargs -d '\n' repo-remove "$clean" + done +} + +fetch_pkgs() { + local name=$1 + local tags=("${@:2}") + + local repo arch dbfile whitelist + + local tag + for tag in "${tags[@]}"; do + msg2 'Syncronizing package files for %s...' "$tag" + repo=${tag%-*} + arch=${tag##*-} + + dbfile="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + whitelist="${IMPORTDIR}/clean/$name/dbs/$tag.whitelist" + + list_pkgs "$dbfile" > "$whitelist" + + # fetch the architecture-specific packages + rsync $SYNCARGS --delete-after --delete-excluded \ + --delay-updates \ + --include-from=<(sed "s|\$|-$arch.tar.?z|" "$whitelist") \ + --exclude='*' \ + "rsync://${pkgmirror}/$(db_dir "$tag")/" \ + "${IMPORTDIR}/clean/${name}/pkgs/${tag}/" + + # fetch the architecture-independent packages + rsync $SYNCARGS --delete-after --delete-excluded \ + --delay-updates \ + --include-from=<(sed "s|\$|-any.tar.?z|" "$whitelist") \ + --exclude='*' \ + "rsync://${pkgmirror}/$(db_dir "$tag")/" \ + "${IMPORTDIR}/clean/${name}/pkgs/${repo}-any/" + done +} + +publish() { + local name=$1 + local tags=("${@:2}") + + local tag + for tag in "${tags[@]}"; do + msg2 'Publishing changes to %s...' "$tag" + publish_tag "$name" "$tag" + done +} + +publish_tag() { + local name=$1 + local tag=$2 + + local repo=${tag%-*} + local arch=${tag##*-} + local dir="${IMPORTDIR}/clean/${name}/pkgs/${tag}" + + local found + local error=false + local files=() + + local pkgid pkgarch + for pkgid in $(list_added_pkgs "$name" "$tag"); do + found=false + + for pkgarch in "${arch}" any; do + file="${dir}/${pkgid}-${arch}".pkg.tar.?z + if ! $found && [[ -r $file ]]; then + files+=("$file") + found=true + fi + done + + if ! $found; then + error 'Could not find package file for %s' "$pkgid" + error=true + fi + done + + if $error; then + error 'Quitting...' + return 1 + fi + + mkdir -p -- "${IMPORTDIR}/staging/${tag}/${repo}" + cp -al -- "${files[@]}" "${IMPORTDIR}/staging/${tag}/${repo}/" + STAGING="${IMPORTDIR}/staging/${tag}" db-update + + # XXX: db-remove wants pkgbase, not pkgname + list_removed_pkgs "$name" "$tag" | xargs -d '\n' db-remove "$repo" "$arch" +} + +################################################################################ + +# Usage: list_arches repo-arch... +# Returns a list of the architectures mentioned in a list of "repo-arch" pairs. +list_arches() { + local tags=("$@") + printf '%s\n' "${tags[@]##*-}" | sort -u +} + +# Usage: list_repos arch repo-arch... +# Returns a list of all the repositories mentioned for a given architecture in a +# list of "repo-arch" pairs. +list_repos() { + local arch=$1 + local tags=("${@:2}") + printf '%s\n' "${tags[@]}" | sed -n "s/-$arch\$//p" +} + +# Usage: db_dir repo-arch +db_dir() { + local tag=$1 + local repo=${tag%-*} + local arch=${tag##*-} + echo "${repo}/os/${arch}" +} + +# Usage; db_file repo-arch +db_file() { + local tag=$1 + local repo=${tag%-*} + local arch=${tag##*-} + echo "${repo}/os/${arch}/${repo}${DBEXT}" +} + +# Usage: list_pkgs dbfile +# Prints "$pkgname-$(get_full_version "$pkgname")" for every package in $dbfile +list_pkgs() { + local dbfile=$1 + bsdtar tf "$dbfile" | cut -d/ -f1 +} + +# Usage: list_pkgs | sep_ver +# Separates the pkgname from the version (replaces the '-' with ' ') for the +# list provided on stdin. +sep_ver() { + sed -r 's/-([^-]*-[^-]*)$/ \1/' +} + +# Usage: list_removed_pkgs importsrc repo-arch +# Prints "$pkgname-$(get_full_version "$pkgname")" for every removed package. +list_removed_pkgs() { + local name=$1 + local tag=$2 + + local old="${FTP_BASE}/$(db_file "$tag")" + local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + + # make a list of: + # pkgname oldver[ newver] + # It will include removed or updated packages (changed packages) + join -a1 \ + <(list_pkgs "$old"|sep_ver|sort) \ + <(list_pkgs "$new"|sep_ver|sort) + | grep -v ' .* ' # remove updated packages + | sed 's/ /-/' # re-combine the pkgname and version +} + +# Usage: list_added_pkgs importsrc repo-arch +# slightly a misnomer; added and updated +# Prints "$pkgname-$(get_full_version "$pkgname")" for every added or updated +# package. +list_added_pkgs() { + local name=$1 + local tag=$2 + + local old="${FTP_BASE}/$(db_file "$tag")" + local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + + comm -13 <(list_pkgs "$old") <(list_pkgs "$new") +} + +main "$@" diff --git a/db-import.conf b/db-import.conf new file mode 100644 index 0000000..aaf7b0f --- /dev/null +++ b/db-import.conf @@ -0,0 +1,23 @@ +#!/hint/bash + +IMPORTDIR=/srv/repo/import + +case "$USER" in + db-import-packages) + _archrepos=( + {core,extra,testing,staging}-{i686,x86_64} + {gnome,kde}-unstable-{i686,x86_64} + );; + db-import-community) + _archrepos=( + community{,-testing,-staging}-{i686,x86_64} + multilib{,-testing,-staging}-x86_64 + );; +esac + +_archpkgmirror=$(db-pick-mirror rsync https://www.archlinux.org/mirrors/status/tier/1/json/) + +# name pkgmirror absmirror repo-arch... +IMPORTS=("archlinux ${_archpkgmirror} rsync.archlinux.org ${_archrepos[*]}") + +unset _archrepos _archpkgmirror @@ -4,5 +4,3 @@ source "$(dirname "$(readlink -e "$0")")/config" mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" - -"$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/db-libremessages b/db-libremessages deleted file mode 100755 index 37df149..0000000 --- a/db-libremessages +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2006-2010 Pacman Development Team <pacman-dev@archlinux.org> -# Copyright (c) 2002-2006 by Judd Vinet <jvinet@zeroflux.org> -# Copyright (c) 2005 by Aurelien Foret <orelien@chez.com> -# Copyright (c) 2006 by Miklos Vajna <vmiklos@frugalware.org> -# Copyright (c) 2005 by Christian Hamar <krics@linuxforum.hu> -# Copyright (c) 2006 by Alex Smith <alex@alex-smith.me.uk> -# Copyright (c) 2006 by Andras Voroskoi <voroskoi@frugalware.org> -# Copyright (c) 2011 by Joshua Haase <hahj87@gmail.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -# gettext initialization -export TEXTDOMAIN='libretools' -export TEXTDOMAINDIR='/usr/share/locale' - -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW - -if tput setaf 0 &>/dev/null; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" - PURPLE="${ALL_OFF}$(tput setaf 5)" -else - ALL_OFF="\033[1;0m" - BOLD="\033[1;1m" - BLUE="${BOLD}\033[1;34m" - GREEN="${BOLD}\033[1;32m" - RED="${BOLD}\033[1;31m" - YELLOW="${BOLD}\033[1;33m" - PURPLE="${BOLD}\033[1;30;40m" -fi - -stdnull() { - local action=$1; - eval "${action} >/dev/null 2>&1" -} - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -fatal_error() { - local mesg=$1; shift - error "$mesg" "$@" - - exit 1 -} diff --git a/list_nonfree_in_db.py b/db-list-nonfree.py index a486fa5..a486fa5 100755 --- a/list_nonfree_in_db.py +++ b/db-list-nonfree.py @@ -34,11 +34,6 @@ for pkgbase in "${args[@]:2}"; do die "Could not read pkgname" fi - pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") - if [ -z "${pkgver}" ]; then - die "Could not read pkgver" - fi - if [ "${pkgarch}" == 'any' ]; then tarches=("${ARCHES[@]}") else @@ -46,6 +41,10 @@ for pkgbase in "${args[@]:2}"; do fi for pkgname in "${pkgnames[@]}"; do + pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version "${pkgname}") + if [ -z "${pkgver}" ]; then + die "Could not read pkgver" + fi for tarch in "${tarches[@]}"; do getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} >/dev/null done @@ -72,11 +71,11 @@ for pkgbase in "${args[@]:2}"; do else tarches=("${pkgarch}") fi - msg2 '%s (%s)' "${pkgbase}" "${tarches[*]}" + msg2 "%s (%s)" "${pkgbase}" "${tarches[*]}" pkgnames=($(. "${xbsrepo_to}/PKGBUILD"; echo "${pkgname[@]}")) - pkgver=$(. "${xbsrepo_to}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") for pkgname in "${pkgnames[@]}"; do + pkgver=$(. "${xbsrepo_to}/PKGBUILD"; get_full_version "${pkgname}") for tarch in "${tarches[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}) pkgfile="${pkgpath##*/}" diff --git a/db-pick-mirror b/db-pick-mirror new file mode 100755 index 0000000..9474ed7 --- /dev/null +++ b/db-pick-mirror @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby + +require 'json' +require 'rest_client' + +protocol = ARGV[0] +jsonurl = ARGV[1] + +data = JSON::parse(RestClient.get(jsonurl)) + +if data["version"] != 3 + print "Data format version != 3" + exit 1 +end + +urls = data["urls"] +rsync_urls = urls.select{|a| a["protocol"]==protocol} + +# By score ( (delay+speed)/completion ) +#best = rsync_urls.sort{|a,b| (a["score"] || Float::INFINITY) <=> (b["score"] || Float::INFINITY) }.first +# By delay/completion +best = rsync_urls.sort{|a,b| a["delay"]/a["completion_pct"] <=> b["delay"]/b["completion_pct"] }.first + +puts best["url"] diff --git a/db-sync b/db-sync deleted file mode 100755 index 2194fe6..0000000 --- a/db-sync +++ /dev/null @@ -1,208 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics - -# Run as `V=true db-sync` to get verbose output -VERBOSE=${V} -extra=() -${VERBOSE} && extra+=(-v) - -WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT - -# Returns contents of a repo -get_repos() { - # Exclude everything but db files - rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" -} - -get_repo_content() { - # Return all contents - bsdtar tf "${1}" | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${WORKDIR}/${1}/os/${2}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "%d packages in blacklist" ${#blacklist[@]} - - test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in "${ARCHREPOS[@]}"; do - for _arch in "${ARCHARCHES[@]}"; do - msg "Processing %s-%s" "${_repo}-${_arch}" - - db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} - files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - # Get db contents - db=($(get_repo_content "${db_file}")) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - # IMPORTANT: the . in the sed command is needed because an empty - # whitelist would consist of a single * allowing any package to - # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" - - msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync "${extra[@]}" --no-motd -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from="/tmp/${_repo}-${_arch}.whitelist" \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - "${WORKDIR}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists, check for single *s just in case - cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist - - msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local pkgpool - for pkgpool in "${ARCHPKGPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ - "${FTP_BASE}/${pkgpool}/" - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local srcpool - for srcpool in "${ARCHSRCPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ - "${FTP_BASE}/${srcpool}/" - done - - date -u +%s > "${FTP_BASE}/lastsync" - - # Cleanup - unset blacklist whitelists _arch _repo repo_file -} - -trap_exit() { - local signal=$1; shift - echo - error "$@" - trap -- "$signal" - kill "-$signal" "$$" -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-sync.conf" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do - test -z "${!var}" && fatal_error "Empty %s" "${var}" -done - -# From makepkg -set -E -for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" -done -trap 'trap_exit INT "Aborted by user! Exiting..."' INT -trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR - -init diff --git a/db-sync.conf b/db-sync.conf deleted file mode 100644 index f7748c3..0000000 --- a/db-sync.conf +++ /dev/null @@ -1,11 +0,0 @@ -#mirror="mirrors.kernel.org" -mirror="mirrors.niyawe.de" - -## mirrors without sources folder -#mirror="mirror.nl.leaseweb.net" -#mirror="mirror.one.com" -#mirror="mirror.us.leaseweb.net" -#mirror="mirror.bytemark.co.uk" -#mirror="mirror.de.leaseweb.net" - -mirrorpath="archlinux" @@ -42,14 +42,22 @@ for repo in "${repos[@]}"; do if ! check_pkgfile "${pkg}"; then die "Package %s is not consistent with its meta data" "${repo}/${pkg##*/}" fi + if "${REQUIRE_SIGNATURE}" && ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then + die "Package %s does not have a valid signature" "${repo}/${pkg##*/}" + fi + if ! check_pkgxbs "${pkg}" "${repo}"; then + die "Package %s is not consistent with %s" "${repo}/${pkg##*/}" "$(xbs name)" + fi if ! check_pkgrepos "${pkg}"; then die "Package %s already exists in another repository" "${repo}/${pkg##*/}" fi + if ! check_packager "${pkg}"; then + die "Package ${repo}/${pkg##*/} does not have a valid packager" + fi done - # This is fucking obnoxious - #if ! check_splitpkgs ${repo} "${pkgs[@]}"; then - # die "Missing split packages for %s" "${repo}" - #fi + if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then + die "Missing split packages for %s" "${repo}" + fi else die "Could not read %s" "${STAGING}" fi @@ -1,15 +0,0 @@ -#!/bin/bash -# Author: Nicolás Reynolds <fauno@kiwwwi.com.ar> -# License: GPLv3+ -# Description: A script to quickly create new [repos] - -source "$(dirname "$(readlink -e "$0")")/config" - -for repo in "$@"; do - echo ":: Creating [$repo]" - for arch in "${ARCHES[@]}"; do - mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" - done -done - -echo ":: All done. Add the repo to the ParabolaWeb admin page." diff --git a/test/lib/common.inc b/test/lib/common.inc index a2dee10..b9a2f9d 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -1,45 +1,83 @@ set -E +# override the default TMPDIR +init_tmpdir() { + [[ -n $MASTER_TMPDIR ]] || export MASTER_TMPDIR="$(mktemp -dqt ${0##*/}.XXXXXXXXXX)" + export TMPDIR=$MASTER_TMPDIR +} + +_TMPDIR=$TMPDIR . "$(dirname ${BASH_SOURCE[0]})/../../config" +TMPDIR=$_TMPDIR +init_tmpdir + . "$(dirname ${BASH_SOURCE[0]})/../../db-functions" +arches() ( + . $(librelib conf.sh) + load_files libretools + printf '%s\n' "${ARCHES[*]}" +) + +arch_svn() { + /usr/bin/svn "$@" +} + +signpkg() { + if [[ -r '/etc/makepkg.conf' ]]; then + source '/etc/makepkg.conf' + else + die '/etc/makepkg.conf not found!' + fi + if [[ -r ~/.makepkg.conf ]]; then + . ~/.makepkg.conf + fi + if [[ -n $GPGKEY ]]; then + SIGNWITHKEY=(-u "${GPGKEY}") + fi + gpg --detach-sign --use-agent "${SIGNWITHKEY[@]}" "${@}" || die +} + oneTimeSetUp() { local p local d local a + local arches=($(arches)) local pkgname local pkgarch local pkgversion local build - pkgdir="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)" - cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" + pkgdir="$(mktemp -dt "${0##*/}.XXXXXXXXXX")" + cp -Lr "$(dirname "${BASH_SOURCE[0]}")"/../packages/* "${pkgdir}" msg 'Building packages...' for d in "${pkgdir}"/*; do pushd $d >/dev/null - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - pkgarch=($(. PKGBUILD; echo ${arch[@]})) - pkgversion=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgname=($(. PKGBUILD; echo "${pkgname[@]}")) + pkgarch=($(. PKGBUILD; echo "${arch[@]}")) + pkgversion=$(. PKGBUILD; get_full_version) build=true - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - [ ! -f ${p}-${pkgversion}-${a}${PKGEXT} ] && build=false + for a in "${pkgarch[@]}"; do + for p in "${pkgname[@]}"; do + [ ! -f "${p}-${pkgversion}-${a}"${PKGEXT} ] && build=false done done - if ! ${build}; then + if ! "${build}"; then if [ "${pkgarch[0]}" == 'any' ]; then - sudo extra-x86_64-build || die 'extra-x86_64-build failed' + sudo libremakepkg || die 'libremakepkg failed' else - for a in ${pkgarch[@]}; do - sudo extra-${a}-build || die "extra-${a}-build failed" + for a in "${pkgarch[@]}"; do + if in_array "$a" "${arches[@]}"; then + sudo setarch "$a" libremakepkg -n "$a" || die "setarch ${a} libremakepkg -n ${a} failed" + for p in "${pkgname[@]}"; do + cp "${p}-${pkgversion}-${a}"${PKGEXT} "$(dirname "${BASH_SOURCE[0]})/../packages/${d##*/}")" + done + else + warning "skipping arch %s" "$a" + fi done fi - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/$(basename ${d}) - done - done fi popd >/dev/null done @@ -55,17 +93,19 @@ setUp() { local r local a - [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" - TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" + [ -f "$(dirname "${BASH_SOURCE[0]}")/../../config.local" ] && die "$(dirname "${BASH_SOURCE[0]}")/../../config.local exists" + init_tmpdir + TMP="$(mktemp -dt "${0##*/}.XXXXXXXXXX")" #msg "Using ${TMP}" PKGREPOS=('core' 'extra' 'testing') PKGPOOL='pool/packages' + SRCPOOL='pool/sources' mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} - for r in ${PKGREPOS[@]}; do + for r in "${PKGREPOS[@]}"; do mkdir -p "${TMP}/staging/${r}" - for a in ${ARCHES[@]} any; do + for a in "${ARCHES[@]}"; do mkdir -p "${TMP}/ftp/${r}/os/${a}" done done @@ -74,21 +114,33 @@ setUp() { msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" - svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + arch_svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" for p in "${pkgdir}"/*; do - pkg=$(basename $p) + pkg=${p##*/} mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} - cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ - svn add -q "${TMP}/svn-packages-copy"/${pkg} - svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" + cp "${p}"/* "${TMP}/svn-packages-copy/${pkg}/trunk/" + arch_svn add -q "${TMP}/svn-packages-copy/${pkg}" + arch_svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done - cat <<eot > "$(dirname ${BASH_SOURCE[0]})/../../config.local" + mkdir -p "${TMP}/home/.config/libretools" + export XDG_CONFIG_HOME="${TMP}/home/.config" + printf '%s\n' \ + 'SVNURL=foo' \ + "SVNREPO=\"${TMP}/svn-packages-copy\"" \ + "ARCHES=($(arches))" \ + > "$XDG_CONFIG_HOME/libretools/xbs-abs.conf" + printf '%s\n' 'BUILDSYSTEM=abs' > "$XDG_CONFIG_HOME/xbs.conf" + + cat <<eot > "$(dirname "${BASH_SOURCE[0]}")/../../config.local" FTP_BASE="${TMP}/ftp" - SVNREPO="file://${TMP}/svn-packages-repo" - PKGREPOS=(${PKGREPOS[@]}) + SVNREPO="${TMP}/svn-packages-copy" + PKGREPOS=("${PKGREPOS[@]}") PKGPOOL="${PKGPOOL}" + SRCPOOL="${SRCPOOL}" + TESTING_REPO='testing' + STABLE_REPOS=('core' 'extra') CLEANUP_DESTDIR="${TMP}/package-cleanup" SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" @@ -97,12 +149,12 @@ setUp() { SOURCE_CLEANUP_DRYRUN=false REQUIRE_SIGNATURE=true eot - . "$(dirname ${BASH_SOURCE[0]})/../../config" + . "$(dirname "${BASH_SOURCE[0]}")/../../config" } tearDown() { rm -rf "${TMP}" - rm -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" + rm -f "$(dirname "${BASH_SOURCE[0]}")/../../config.local" echo } @@ -110,18 +162,24 @@ releasePackage() { local repo=$1 local pkgbase=$2 local arch=$3 + local a + local p + local pkgver + local pkgname - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null - archrelease ${repo}-${arch} >/dev/null 2&>1 - pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" >/dev/null + xbs release "${repo}" "${arch}" >/dev/null 2>&1 + pkgver=$(. PKGBUILD; get_full_version) + pkgname=($(. PKGBUILD; echo "${pkgname[@]}")) popd >/dev/null - cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ + cp "${pkgdir}/${pkgbase}"/*-"${pkgver}-${arch}"${PKGEXT} "${STAGING}/${repo}/" - if ${REQUIRE_SIGNATURE}; then - # TODO: really sign the packages with a valid key - find "${STAGING}"/${repo}/ -type f \ - -name "*-${pkgver}-${arch}${PKGEXT}" \ - -exec touch {}.sig \; + if "${REQUIRE_SIGNATURE}"; then + for a in "${arch[@]}"; do + for p in "${pkgname[@]}"; do + signpkg "${STAGING}/${repo}/${p}-${pkgver}-${a}"${PKGEXT} + done + done fi } @@ -132,32 +190,29 @@ checkAnyPackageDB() { local db [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - if ${REQUIRE_SIGNATURE}; then + if "${REQUIRE_SIGNATURE}"; then [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" fi - for arch in i686 x86_64; do + for arch in $(arches); do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkg}")" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" if ${REQUIRE_SIGNATURE}; then [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkg}.sig")" ] \ || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" fi + + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep ${pkg} &>/dev/null) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done done [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done - - [ -r "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" - [ -r "${FTP_BASE}/${repo}/os/any/${pkg}.sig" ] && fail "${repo}/os/any/${pkg}.sig should not exist" } checkAnyPackage() { @@ -167,7 +222,7 @@ checkAnyPackage() { checkAnyPackageDB $repo $pkg local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" } @@ -182,7 +237,7 @@ checkPackageDB() { [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkg}")" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" if ${REQUIRE_SIGNATURE}; then @@ -190,13 +245,13 @@ checkPackageDB() { [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkg}.sig")" ] \ || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" fi for db in ${DBEXT} ${FILESEXT}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep ${pkg} &>/dev/null) \ || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" done } @@ -209,7 +264,7 @@ checkPackage() { checkPackageDB $repo $pkg $arch local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" } @@ -222,7 +277,7 @@ checkRemovedPackageDB() { for db in ${DBEXT} ${FILESEXT}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep ${pkgbase} &>/dev/null) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" done } @@ -234,7 +289,7 @@ checkRemovedPackage() { checkRemovedPackageDB $repo $pkgbase $arch - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" } @@ -246,9 +301,9 @@ checkRemovedAnyPackageDB() { local db for db in ${DBEXT} ${FILESEXT}; do - for arch in i686 x86_64; do + for arch in $(arches); do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep ${pkgbase} &>/dev/null) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" done done @@ -260,7 +315,7 @@ checkRemovedAnyPackage() { checkRemovedAnyPackageDB $repo $pkgbase - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" } diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 49734c4..b5ec5c8 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -1,24 +1,24 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(readlink -e "$(dirname "$0")")" . "${curdir}/../lib/common.inc" testCreateSimpleFileLists() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${arches[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then + for pkgbase in "${pkgs[@]}"; do + for arch in "${arches[@]}"; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/${pkgbase}" &>/dev/null; then fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}" fi done @@ -26,19 +26,19 @@ testCreateSimpleFileLists() { } testCreateAnyFileLists() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" any done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then + for pkgbase in "${pkgs[@]}"; do + for arch in "${arches[@]}"; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/share/${pkgbase}/test" &>/dev/null; then fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}" fi done @@ -46,7 +46,7 @@ testCreateAnyFileLists() { } testCreateSplitFileLists() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -54,18 +54,18 @@ testCreateSplitFileLists() { local pkgnames local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${arches[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do - pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) - for pkgname in ${pkgnames[@]}; do - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then + for pkgbase in "${pkgs[@]}"; do + pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo "${pkgname[@]}")) + for pkgname in "${pkgnames[@]}"; do + for arch in "${arches[@]}"; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/${pkgname}" &>/dev/null; then fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}" fi done @@ -75,7 +75,7 @@ testCreateSplitFileLists() { testCleanupFileLists() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -92,10 +92,10 @@ testCleanupFileLists() { done for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/pkg-simple-b" &>/dev/null; then fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}" fi - if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-a"; then + if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/pkg-simple-a" &>/dev/null; then fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}" fi done diff --git a/test/test.d/db-move.sh b/test/test.d/db-move.sh index 9d7c1f6..3cf355b 100755 --- a/test/test.d/db-move.sh +++ b/test/test.d/db-move.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testMoveSimplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -28,7 +28,7 @@ testMoveSimplePackages() { } testMoveMultiplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -52,7 +52,7 @@ testMoveMultiplePackages() { } testMoveEpochPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-epoch') local pkgbase local arch @@ -90,7 +90,7 @@ testMoveAnyPackages() { } testMoveSplitPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -107,12 +107,12 @@ testMoveSplitPackages() { for arch in ${arches[@]}; do for pkg in "${pkgdir}/pkg-split-a"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + checkPackage extra ${pkg##*/} ${arch} done done for arch in ${arches[@]}; do for pkg in "${pkgdir}/pkg-split-b"/*-${arch}${PKGEXT}; do - checkPackage testing $(basename ${pkg}) ${arch} + checkPackage testing ${pkg##*/} ${arch} done done diff --git a/test/test.d/db-remove.sh b/test/test.d/db-remove.sh index 416e693..d79605e 100755 --- a/test/test.d/db-remove.sh +++ b/test/test.d/db-remove.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testRemovePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') local pkgbase local arch @@ -31,7 +31,7 @@ testRemovePackages() { } testRemoveMultiplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') local pkgbase local arch diff --git a/test/test.d/db-repo-add.sh b/test/test.d/db-repo-add.sh index 8603104..09fc52f 100755 --- a/test/test.d/db-repo-add.sh +++ b/test/test.d/db-repo-add.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testAddSimplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -27,7 +27,7 @@ testAddSimplePackages() { } testAddMultiplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch diff --git a/test/test.d/db-repo-remove.sh b/test/test.d/db-repo-remove.sh index 315d63d..eec0109 100755 --- a/test/test.d/db-repo-remove.sh +++ b/test/test.d/db-repo-remove.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testRemovePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch @@ -31,7 +31,7 @@ testRemovePackages() { } testRemoveMultiplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index e38c328..7f1874b 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testAddSimplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -52,7 +52,7 @@ testAddAnyPackages() { } testAddSplitPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -69,7 +69,7 @@ testAddSplitPackages() { for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + checkPackage extra ${pkg##*/} ${arch} done done done @@ -81,8 +81,8 @@ testUpdateAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 + arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo libremakepkg mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -99,8 +99,8 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 + arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo libremakepkg mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -132,16 +132,16 @@ testUpdateSameAnyPackageToDifferentRepositories() { ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) local arch - for arch in i686 x86_64; do + for arch in $(arches); do ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep ${pkgbase} &>/dev/null) \ && fail "${pkgbase} should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" done } testAddIncompleteSplitPackage() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local repo='extra' local pkgbase='pkg-split-a' local arch @@ -157,9 +157,19 @@ testAddIncompleteSplitPackage() { for arch in ${arches[@]}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep ${pkgbase} &>/dev/null) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" done } +testUnknownRepo() { + mkdir "${STAGING}/unknown/" + releasePackage extra 'pkg-simple-a' 'i686' + releasePackage unknown 'pkg-simple-b' 'i686' + ../db-update + checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + [ -e "${FTP_BASE}/unknown" ] && fail "db-update pushed a package into an unknown repository" + rm -rf "${STAGING}/unknown/" +} + . "${curdir}/../lib/shunit2" diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh index 20026b4..630b88f 100755 --- a/test/test.d/ftpdir-cleanup.sh +++ b/test/test.d/ftpdir-cleanup.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testCleanupSimplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -35,7 +35,7 @@ testCleanupSimplePackages() { } testCleanupEpochPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-epoch') local pkgbase local arch @@ -85,7 +85,7 @@ testCleanupAnyPackages() { } testCleanupSplitPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -113,7 +113,7 @@ testCleanupSplitPackages() { done for pkg in "${pkgdir}/${pkgs[1]}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + checkPackage extra ${pkg##*/} ${arch} done done } diff --git a/test/test.d/pool-transition.sh b/test/test.d/pool-transition.sh deleted file mode 100755 index 5873f00..0000000 --- a/test/test.d/pool-transition.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testMovePackagesWithoutPool() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') - local pkgbase - local arch - local pkg - local old - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - - # transform two packages to old style layout - for arch in ${arches[@]}; do - for old in 0 2; do - for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}${PKGEXT}; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} - done - checkRemovedPackage testing ${pkgbase} ${arch} - done - done -} - -testUpdateAnyPackageWithoutPool() { - local pkgname='pkg-any-a' - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - local pkg2='pkg-any-a-1-2-any.pkg.tar.xz' - local arch - - - releasePackage extra pkg-any-a any - ../db-update - # transform two packages to old style layout - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" "${FTP_BASE}/extra/os/any" - for arch in i686 x86_64; do - ln -sf "../any/${pkg1}" "${FTP_BASE}/extra/os/${arch}" - done - - pushd "${TMP}/svn-packages-copy/${pkgname}/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 - mv "${pkg2}" "${pkgdir}/${pkgname}/" - popd >/dev/null - - releasePackage extra ${pkgname} any - ../db-update - rm -f "${pkgdir}/${pkgname}/${pkg2}" - - ../cron-jobs/ftpdir-cleanup >/dev/null - - checkAnyPackage extra "${pkg2}" - - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/extra/os/${arch}/${pkg1}" ] && fail "extra/os/${arch}/${pkg1} found" - done -} - -testMoveAnyPackagesWithoutPool() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch - local pkg - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any - done - - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-any${PKGEXT}; do - checkAnyPackage extra $(basename ${pkg}) - done - checkRemovedAnyPackage testing ${pkgbase} - done - - for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=$(basename $pkg) - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/testing/os/${arch}/${pkg}" ] && fail "testing/os/${arch}/${pkg} found" - done - done -} - -testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { - local pkg - local arch - - releasePackage extra pkg-any-a any - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/pkg-any-a"/*-any${PKGEXT}; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" - done - done - - releasePackage testing pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) - - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q pkg-any-a) \ - && fail "pkg-any-a should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" - done -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/signed-packages.sh b/test/test.d/signed-packages.sh index 5d6f4ff..03566ef 100755 --- a/test/test.d/signed-packages.sh +++ b/test/test.d/signed-packages.sh @@ -3,11 +3,34 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" +testAddSignedPackage() { + releasePackage extra 'pkg-simple-a' 'i686' + ../db-update || fail "db-update failed!" +} + testAddUnsignedPackage() { releasePackage extra 'pkg-simple-a' 'i686' - # remove any signature rm "${STAGING}"/extra/*.sig ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" } +testAddInvalidSignedPackage() { + local p + releasePackage extra 'pkg-simple-a' 'i686' + for p in "${STAGING}"/extra/*${PKGEXT}; do + unxz $p + xz -0 ${p%%.xz} + done + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is invalid!" +} + +testAddBrokenSignature() { + local s + releasePackage extra 'pkg-simple-a' 'i686' + for s in "${STAGING}"/extra/*.sig; do + echo 0 > $s + done + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" +} + . "${curdir}/../lib/shunit2" diff --git a/test/test.d/sourceballs.sh b/test/test.d/sourceballs.sh index fdcf08c..472cb30 100755 --- a/test/test.d/sourceballs.sh +++ b/test/test.d/sourceballs.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testSourceballs() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch @@ -38,7 +38,7 @@ testAnySourceballs() { } testSplitSourceballs() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -59,7 +59,7 @@ testSplitSourceballs() { } testSourceballsCleanup() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index eda6cd6..0c2fa83 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -9,8 +9,8 @@ testTesting2xAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 + arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo libremakepkg mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null |