From 8eafe04ec5eaf917d9e2d58993604c454f1a26d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Wed, 26 Aug 2015 03:33:26 -0300 Subject: re-estructure dbscrips --- Makefile | 0 TODO | 10 - any-to-ours | 123 --- bin | 1 + config | 36 - config_abslibre_gnu+hurd | 69 -- config_abslibre_gnu+linux | 69 -- config_abslibre_gnu+linux_arm | 69 -- config_abslibre_gnu+linux_legacy | 69 -- config_gnu+hurd | 59 -- config_gnu+linux | 59 -- config_platforms | 6 - create-repo | 61 -- createrepos | 36 - cron-jobs/check_archlinux/check_packages.py | 508 ---------- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 153 --- cron-jobs/devlist-mailer | 28 - cron-jobs/ftpdir-cleanup | 88 -- cron-jobs/integrity-check | 32 - cron-jobs/make_repo_torrents | 70 -- cron-jobs/repo-sanity-check | 57 -- cron-jobs/sourceballs | 151 --- cron-jobs/sourceballs.force | 4 - cron-jobs/sourceballs.skip | 29 - db-check-nonfree | 46 - db-check-package-libraries.py | 193 ---- db-cleanup | 69 -- db-functions | 524 ---------- db-libremessages | 83 -- db-list-unsigned-packages | 38 - db-list-unsigned-packages.py | 96 -- db-move | 106 -- db-remove | 49 - db-repo-add | 41 - db-repo-remove | 37 - db-sync | 208 ---- db-sync.conf | 11 - db-update | 118 --- etc | 1 + extra/coadde/TODO | 10 + extra/coadde/any-to-ours | 123 +++ extra/coadde/create-repo | 61 ++ extra/coadde/createrepos | 36 + .../cron-jobs/check_archlinux/check_packages.py | 508 ++++++++++ .../cron-jobs/check_archlinux/parse_pkgbuilds.sh | 153 +++ extra/coadde/cron-jobs/devlist-mailer | 28 + extra/coadde/cron-jobs/ftpdir-cleanup | 88 ++ extra/coadde/cron-jobs/integrity-check | 32 + extra/coadde/cron-jobs/make_repo_torrents | 70 ++ extra/coadde/cron-jobs/repo-sanity-check | 57 ++ extra/coadde/cron-jobs/sourceballs | 151 +++ extra/coadde/cron-jobs/sourceballs.force | 4 + extra/coadde/cron-jobs/sourceballs.skip | 29 + extra/coadde/db-check-nonfree | 46 + extra/coadde/db-check-package-libraries.py | 193 ++++ extra/coadde/db-cleanup | 69 ++ extra/coadde/db-functions | 524 ++++++++++ extra/coadde/db-libremessages | 83 ++ extra/coadde/db-list-unsigned-packages | 38 + extra/coadde/db-list-unsigned-packages.py | 96 ++ extra/coadde/db-move | 106 ++ extra/coadde/db-remove | 49 + extra/coadde/db-repo-add | 41 + extra/coadde/db-repo-remove | 37 + extra/coadde/db-sync | 208 ++++ extra/coadde/db-sync.conf | 11 + extra/coadde/db-update | 118 +++ extra/coadde/list_nonfree_in_db.py | 28 + extra/coadde/make_individual_torrent | 52 + extra/coadde/mkrepo | 15 + extra/coadde/test/__init__.py | 0 extra/coadde/test/blacklist_sample | 2 + extra/coadde/test/core.db.tar.gz | Bin 0 -> 1345 bytes extra/coadde/test/depends | 4 + extra/coadde/test/desc | 39 + extra/coadde/test/lib/common.inc | 266 +++++ extra/coadde/test/lib/shunit2 | 1048 ++++++++++++++++++++ extra/coadde/test/packages/pkg-any-a/PKGBUILD | 12 + extra/coadde/test/packages/pkg-any-b/PKGBUILD | 12 + extra/coadde/test/packages/pkg-simple-a/Makefile | 1 + extra/coadde/test/packages/pkg-simple-a/PKGBUILD | 22 + extra/coadde/test/packages/pkg-simple-a/test.c | 1 + extra/coadde/test/packages/pkg-simple-b/Makefile | 1 + extra/coadde/test/packages/pkg-simple-b/PKGBUILD | 22 + extra/coadde/test/packages/pkg-simple-b/test.c | 1 + .../coadde/test/packages/pkg-simple-epoch/Makefile | 1 + .../coadde/test/packages/pkg-simple-epoch/PKGBUILD | 23 + extra/coadde/test/packages/pkg-simple-epoch/test.c | 1 + extra/coadde/test/packages/pkg-split-a/Makefile | 1 + extra/coadde/test/packages/pkg-split-a/PKGBUILD | 28 + extra/coadde/test/packages/pkg-split-a/test.c | 1 + extra/coadde/test/packages/pkg-split-b/Makefile | 1 + extra/coadde/test/packages/pkg-split-b/PKGBUILD | 29 + extra/coadde/test/packages/pkg-split-b/test.c | 1 + extra/coadde/test/rsync_output_sample | 14 + extra/coadde/test/runTest | 15 + extra/coadde/test/src/Makefile | 5 + extra/coadde/test/src/test.c | 7 + extra/coadde/test/test.d/create-filelists.sh | 105 ++ extra/coadde/test/test.d/db-move.sh | 122 +++ extra/coadde/test/test.d/db-remove.sh | 77 ++ extra/coadde/test/test.d/db-repo-add.sh | 54 + extra/coadde/test/test.d/db-repo-remove.sh | 58 ++ extra/coadde/test/test.d/db-update.sh | 165 +++ extra/coadde/test/test.d/ftpdir-cleanup.sh | 121 +++ extra/coadde/test/test.d/packages.sh | 11 + extra/coadde/test/test.d/pool-transition.sh | 152 +++ extra/coadde/test/test.d/signed-packages.sh | 13 + extra/coadde/test/test.d/sourceballs.sh | 84 ++ extra/coadde/test/test.d/testing2x.sh | 27 + extra/coadde/test/test_filter.py | 196 ++++ extra/coadde/xbslibre | 93 ++ list_nonfree_in_db.py | 28 - make_individual_torrent | 52 - mkrepo | 15 - share | 1 + src/etc/dbscripts.cfg | 6 + src/etc/dbscripts.d/gnu+hurd_add.cfg | 67 ++ src/etc/dbscripts.d/gnu+hurd_native.cfg | 57 ++ src/etc/dbscripts.d/gnu+linux_add.cfg | 67 ++ src/etc/dbscripts.d/gnu+linux_add_arm.cfg | 67 ++ src/etc/dbscripts.d/gnu+linux_add_legacy.cfg | 67 ++ src/etc/dbscripts.d/gnu+linux_native.cfg | 57 ++ src/etc/dbscripts.d/main.cfg | 36 + test/__init__.py | 0 test/blacklist_sample | 2 - test/core.db.tar.gz | Bin 1345 -> 0 bytes test/depends | 4 - test/desc | 39 - test/lib/common.inc | 266 ----- test/lib/shunit2 | 1048 -------------------- test/packages/pkg-any-a/PKGBUILD | 12 - test/packages/pkg-any-b/PKGBUILD | 12 - test/packages/pkg-simple-a/Makefile | 1 - test/packages/pkg-simple-a/PKGBUILD | 22 - test/packages/pkg-simple-a/test.c | 1 - test/packages/pkg-simple-b/Makefile | 1 - test/packages/pkg-simple-b/PKGBUILD | 22 - test/packages/pkg-simple-b/test.c | 1 - test/packages/pkg-simple-epoch/Makefile | 1 - test/packages/pkg-simple-epoch/PKGBUILD | 23 - test/packages/pkg-simple-epoch/test.c | 1 - test/packages/pkg-split-a/Makefile | 1 - test/packages/pkg-split-a/PKGBUILD | 28 - test/packages/pkg-split-a/test.c | 1 - test/packages/pkg-split-b/Makefile | 1 - test/packages/pkg-split-b/PKGBUILD | 29 - test/packages/pkg-split-b/test.c | 1 - test/rsync_output_sample | 14 - test/runTest | 15 - test/src/Makefile | 5 - test/src/test.c | 7 - test/test.d/create-filelists.sh | 105 -- test/test.d/db-move.sh | 122 --- test/test.d/db-remove.sh | 77 -- test/test.d/db-repo-add.sh | 54 - test/test.d/db-repo-remove.sh | 58 -- test/test.d/db-update.sh | 165 --- test/test.d/ftpdir-cleanup.sh | 121 --- test/test.d/packages.sh | 11 - test/test.d/pool-transition.sh | 152 --- test/test.d/signed-packages.sh | 13 - test/test.d/sourceballs.sh | 84 -- test/test.d/testing2x.sh | 27 - test/test_filter.py | 196 ---- xbslibre | 93 -- 166 files changed, 6327 insertions(+), 6336 deletions(-) create mode 100644 Makefile delete mode 100644 TODO delete mode 100755 any-to-ours create mode 120000 bin delete mode 100644 config delete mode 100644 config_abslibre_gnu+hurd delete mode 100644 config_abslibre_gnu+linux delete mode 100644 config_abslibre_gnu+linux_arm delete mode 100644 config_abslibre_gnu+linux_legacy delete mode 100644 config_gnu+hurd delete mode 100644 config_gnu+linux delete mode 100644 config_platforms delete mode 100755 create-repo delete mode 100755 createrepos delete mode 100755 cron-jobs/check_archlinux/check_packages.py delete mode 100755 cron-jobs/check_archlinux/parse_pkgbuilds.sh delete mode 100755 cron-jobs/devlist-mailer delete mode 100755 cron-jobs/ftpdir-cleanup delete mode 100755 cron-jobs/integrity-check delete mode 100755 cron-jobs/make_repo_torrents delete mode 100755 cron-jobs/repo-sanity-check delete mode 100755 cron-jobs/sourceballs delete mode 100644 cron-jobs/sourceballs.force delete mode 100644 cron-jobs/sourceballs.skip delete mode 100755 db-check-nonfree delete mode 100755 db-check-package-libraries.py delete mode 100755 db-cleanup delete mode 100644 db-functions delete mode 100755 db-libremessages delete mode 100755 db-list-unsigned-packages delete mode 100755 db-list-unsigned-packages.py delete mode 100755 db-move delete mode 100755 db-remove delete mode 100755 db-repo-add delete mode 100755 db-repo-remove delete mode 100755 db-sync delete mode 100644 db-sync.conf delete mode 100755 db-update create mode 120000 etc create mode 100644 extra/coadde/TODO create mode 100755 extra/coadde/any-to-ours create mode 100755 extra/coadde/create-repo create mode 100755 extra/coadde/createrepos create mode 100755 extra/coadde/cron-jobs/check_archlinux/check_packages.py create mode 100755 extra/coadde/cron-jobs/check_archlinux/parse_pkgbuilds.sh create mode 100755 extra/coadde/cron-jobs/devlist-mailer create mode 100755 extra/coadde/cron-jobs/ftpdir-cleanup create mode 100755 extra/coadde/cron-jobs/integrity-check create mode 100755 extra/coadde/cron-jobs/make_repo_torrents create mode 100755 extra/coadde/cron-jobs/repo-sanity-check create mode 100755 extra/coadde/cron-jobs/sourceballs create mode 100644 extra/coadde/cron-jobs/sourceballs.force create mode 100644 extra/coadde/cron-jobs/sourceballs.skip create mode 100755 extra/coadde/db-check-nonfree create mode 100755 extra/coadde/db-check-package-libraries.py create mode 100755 extra/coadde/db-cleanup create mode 100644 extra/coadde/db-functions create mode 100755 extra/coadde/db-libremessages create mode 100755 extra/coadde/db-list-unsigned-packages create mode 100755 extra/coadde/db-list-unsigned-packages.py create mode 100755 extra/coadde/db-move create mode 100755 extra/coadde/db-remove create mode 100755 extra/coadde/db-repo-add create mode 100755 extra/coadde/db-repo-remove create mode 100755 extra/coadde/db-sync create mode 100644 extra/coadde/db-sync.conf create mode 100755 extra/coadde/db-update create mode 100755 extra/coadde/list_nonfree_in_db.py create mode 100755 extra/coadde/make_individual_torrent create mode 100755 extra/coadde/mkrepo create mode 100644 extra/coadde/test/__init__.py create mode 100644 extra/coadde/test/blacklist_sample create mode 100644 extra/coadde/test/core.db.tar.gz create mode 100644 extra/coadde/test/depends create mode 100644 extra/coadde/test/desc create mode 100644 extra/coadde/test/lib/common.inc create mode 100755 extra/coadde/test/lib/shunit2 create mode 100644 extra/coadde/test/packages/pkg-any-a/PKGBUILD create mode 100644 extra/coadde/test/packages/pkg-any-b/PKGBUILD create mode 120000 extra/coadde/test/packages/pkg-simple-a/Makefile create mode 100644 extra/coadde/test/packages/pkg-simple-a/PKGBUILD create mode 120000 extra/coadde/test/packages/pkg-simple-a/test.c create mode 120000 extra/coadde/test/packages/pkg-simple-b/Makefile create mode 100644 extra/coadde/test/packages/pkg-simple-b/PKGBUILD create mode 120000 extra/coadde/test/packages/pkg-simple-b/test.c create mode 120000 extra/coadde/test/packages/pkg-simple-epoch/Makefile create mode 100644 extra/coadde/test/packages/pkg-simple-epoch/PKGBUILD create mode 120000 extra/coadde/test/packages/pkg-simple-epoch/test.c create mode 120000 extra/coadde/test/packages/pkg-split-a/Makefile create mode 100644 extra/coadde/test/packages/pkg-split-a/PKGBUILD create mode 120000 extra/coadde/test/packages/pkg-split-a/test.c create mode 120000 extra/coadde/test/packages/pkg-split-b/Makefile create mode 100644 extra/coadde/test/packages/pkg-split-b/PKGBUILD create mode 120000 extra/coadde/test/packages/pkg-split-b/test.c create mode 100644 extra/coadde/test/rsync_output_sample create mode 100755 extra/coadde/test/runTest create mode 100644 extra/coadde/test/src/Makefile create mode 100644 extra/coadde/test/src/test.c create mode 100755 extra/coadde/test/test.d/create-filelists.sh create mode 100755 extra/coadde/test/test.d/db-move.sh create mode 100755 extra/coadde/test/test.d/db-remove.sh create mode 100755 extra/coadde/test/test.d/db-repo-add.sh create mode 100755 extra/coadde/test/test.d/db-repo-remove.sh create mode 100755 extra/coadde/test/test.d/db-update.sh create mode 100755 extra/coadde/test/test.d/ftpdir-cleanup.sh create mode 100755 extra/coadde/test/test.d/packages.sh create mode 100755 extra/coadde/test/test.d/pool-transition.sh create mode 100755 extra/coadde/test/test.d/signed-packages.sh create mode 100755 extra/coadde/test/test.d/sourceballs.sh create mode 100755 extra/coadde/test/test.d/testing2x.sh create mode 100644 extra/coadde/test/test_filter.py create mode 100755 extra/coadde/xbslibre delete mode 100755 list_nonfree_in_db.py delete mode 100755 make_individual_torrent delete mode 100755 mkrepo create mode 120000 share create mode 100644 src/etc/dbscripts.cfg create mode 100644 src/etc/dbscripts.d/gnu+hurd_add.cfg create mode 100644 src/etc/dbscripts.d/gnu+hurd_native.cfg create mode 100644 src/etc/dbscripts.d/gnu+linux_add.cfg create mode 100644 src/etc/dbscripts.d/gnu+linux_add_arm.cfg create mode 100644 src/etc/dbscripts.d/gnu+linux_add_legacy.cfg create mode 100644 src/etc/dbscripts.d/gnu+linux_native.cfg create mode 100644 src/etc/dbscripts.d/main.cfg delete mode 100644 test/__init__.py delete mode 100644 test/blacklist_sample delete mode 100644 test/core.db.tar.gz delete mode 100644 test/depends delete mode 100644 test/desc delete mode 100644 test/lib/common.inc delete mode 100755 test/lib/shunit2 delete mode 100644 test/packages/pkg-any-a/PKGBUILD delete mode 100644 test/packages/pkg-any-b/PKGBUILD delete mode 120000 test/packages/pkg-simple-a/Makefile delete mode 100644 test/packages/pkg-simple-a/PKGBUILD delete mode 120000 test/packages/pkg-simple-a/test.c delete mode 120000 test/packages/pkg-simple-b/Makefile delete mode 100644 test/packages/pkg-simple-b/PKGBUILD delete mode 120000 test/packages/pkg-simple-b/test.c delete mode 120000 test/packages/pkg-simple-epoch/Makefile delete mode 100644 test/packages/pkg-simple-epoch/PKGBUILD delete mode 120000 test/packages/pkg-simple-epoch/test.c delete mode 120000 test/packages/pkg-split-a/Makefile delete mode 100644 test/packages/pkg-split-a/PKGBUILD delete mode 120000 test/packages/pkg-split-a/test.c delete mode 120000 test/packages/pkg-split-b/Makefile delete mode 100644 test/packages/pkg-split-b/PKGBUILD delete mode 120000 test/packages/pkg-split-b/test.c delete mode 100644 test/rsync_output_sample delete mode 100755 test/runTest delete mode 100644 test/src/Makefile delete mode 100644 test/src/test.c delete mode 100755 test/test.d/create-filelists.sh delete mode 100755 test/test.d/db-move.sh delete mode 100755 test/test.d/db-remove.sh delete mode 100755 test/test.d/db-repo-add.sh delete mode 100755 test/test.d/db-repo-remove.sh delete mode 100755 test/test.d/db-update.sh delete mode 100755 test/test.d/ftpdir-cleanup.sh delete mode 100755 test/test.d/packages.sh delete mode 100755 test/test.d/pool-transition.sh delete mode 100755 test/test.d/signed-packages.sh delete mode 100755 test/test.d/sourceballs.sh delete mode 100755 test/test.d/testing2x.sh delete mode 100644 test/test_filter.py delete mode 100755 xbslibre diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e69de29 diff --git a/TODO b/TODO deleted file mode 100644 index 9dd4b52..0000000 --- a/TODO +++ /dev/null @@ -1,10 +0,0 @@ -* Test Suite for clean_repo.py - - - Review all repo - - Remove all blacklisted packages - - Get pending list right - - Extract licenses all right - -* Fix db-move - - - Make it use abslibre diff --git a/any-to-ours b/any-to-ours deleted file mode 100755 index 7ca2878..0000000 --- a/any-to-ours +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash -# Releases 'any' packages from Arch-derivations arches to ours - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config_platforms" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -for 'platform' in "${DERIVATIVE_PLATFORMS[@]}"; do - - source "$(dirname "$(readlink -e "$0")")/config_${platform}" - - # Traverse all Arch repos - for '_repo' in "${MAIN_REPOS[@]}"; do - msg "Processing %s..." "${_repo}" - - # Find 'any' packages - # This is hardcoded but it could release other arches... - PKGS=($(find "${REPO_DIR}/${_repo}/os/${ARCHES[0]}/" \ - -iname '*-any.pkg.tar.?z' \ - -printf "%f ")) - - if [ ${#PKGS[@]} -eq 0 ]; then - msg2 "No '%s' packages here" any - continue - fi - - for '_arch' in "${ARCHES[@]}"; do - msg2 "Syncing %s..." "${_arch}" - - # Sync 'any' only and extract the synced packages - SYNCED=($( - rsync -av \ - --include='*-any.pkg.tar.?z' \ - --include='*-any.pkg.tar.?z.sig' \ - --exclude='*' \ - "${REPO_DIR}/${_repo}/os/${ARCHES[0]}/" \ - "${REPO_DIR}/${_repo}/os/${_arch}/" 2>&1 | \ - grep 'any\.pkg\.tar\..z$' | \ - cut -d ' ' -f 1 )) - - if [ "${#SYNCED[@]}" -eq 0 ]; then - msg2 "Already synced (or error happened)" - continue - fi - - msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" - - msg2 "Adding to db..." - - pushd "${REPO_DIR}/${_repo}/os/${_arch}/" >/dev/null - - # Add the packages to the db - repo-add "${_repo}${DB_EXT}" "${SYNCED[@]}" - - popd >/dev/null - - # Avoid mixups - unset 'SYNCED' 'PKGS' - done - if [ -n "${MULTILIB_ARCHES}" ]; then - for '_repo_multilib' in "${MULTILIB_REPOS[@]}"; do - msg "Processing %s..." "${_repo}" '(multilib)' - - # Find 'any' packages - # This is hardcoded but it could release other arches... - PKGS_MULTILIB=($(find "${REPO_DIR}/${_repo_multilib}/os/${MULTILIB_ARCHES[0]}/" \ - -iname '*-any.pkg.tar.?z' \ - -printf "%f ")) - - if [ ${#PKGS_MULTILIB[@]} -eq 0 ]; then - msg2 "No '%s' multilib packages here" any - continue - fi - - for '_arch_multilib' in "${MULTILIB_ARCHES[@]}"; do - msg2 "Syncing %s..." "${_arch_multilib}" '(multilib)' - - # Sync 'any' only and extract the synced packages - SYNCED_MULTILIB=($( - rsync -av \ - --include='*-any.pkg.tar.?z' \ - --include='*-any.pkg.tar.?z.sig' \ - --exclude='*' \ - "${REPO_DIR}/${_repo_multilib}/os/${MULTILIB_ARCHES[0]}/" \ - "${REPO_DIR}/${_repo_multilib}/os/${_arch_multilib}/" 2>&1 | \ - grep 'any\.pkg\.tar\..z$' | \ - cut -d ' ' -f 1 )) - - if [ "${#SYNCED_MULTILIB[@]}" -eq 0 ]; then - msg2 "Already synced (or error happened)" - continue - fi - - msg2 "Synced %d packages: %s" "${#SYNCED_MULTILIB[@]}" "${SYNCED_MULTILIB[*]}" - - msg2 "Adding Multilib to db..." - - pushd "${REPO_DIR}/${_repo_multilib}/os/${_arch_multilib}/" >/dev/null - - # Add the packages to the db - repo-add "${_repo_multilib}${DB_EXT}" "${SYNCED_MULTILIB[@]}" - - popd >/dev/null - - # Avoid mixups - unset 'SYNCED_MULTILIB' 'PKGS_MULTILIB' - done - done - fi - done -done diff --git a/bin b/bin new file mode 120000 index 0000000..70feaa8 --- /dev/null +++ b/bin @@ -0,0 +1 @@ +src/bin \ No newline at end of file diff --git a/config b/config deleted file mode 100644 index c1b9ffa..0000000 --- a/config +++ /dev/null @@ -1,36 +0,0 @@ -#!/hint/bash - -# Parabola Software Development -PSD=('rolling' 'testing' 'stable') - -ROLLING_NAME='rolling' -TESTING_NAME='waterbird-v2' -STABLE_NAME='minicat-v1' - -PKG_POOL_DIR='pools' -SRC_POOL_DIR='sources' - -SRV_DIR='/srv' -TMP_DIR='/tmp' -PKG_POOL_DIR='pools' -SRC_POOL_DIR='sources' -GIT_BASE="${SRV_DIR}/git" -SYNC_ARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' - -REQUIRE_SIGNATURE='true' - -LOCK_DELAY='10' -LOCK_TIMEOUT='300' - -[ -n "${STAGING:-}" ] || STAGING="${HOME}/staging/unknown/staging" -DB_EXT='.db.tar.gz' -FILES_EXT='.files.tar.gz' -PKG_EXT=".pkg.tar.?z" -SRC_EXT='.src.tar.gz' - -MIRROR_LIST='/etc/pacman.d/mirrorlist' -MAKEPKG_CONF='/etc/makepkg.conf' -MAKEPKG_ARCH="$(uname -m)" - -# parabolaweb root -WEB_DIR="${SRV_DIR}/http/www.parabola.nu/web" diff --git a/config_abslibre_gnu+hurd b/config_abslibre_gnu+hurd deleted file mode 100644 index c40a207..0000000 --- a/config_abslibre_gnu+hurd +++ /dev/null @@ -1,69 +0,0 @@ -#!/hint/bash - -source "$(dirname "$(readlink -e "$0")")/config" - -# Parabola Platform -PLATFORM_NAME='GNU/Hurd' -PLATFORM='gnu+hurd' - -# Multilib Support Archictectures -MULTILIB_ARCHES=() - -# Archictectures -ARCHES=('i686' "${MULTILIB_ARCHES[@]}") - -# Main Repositories from Arch GNU/Hurd -MAIN_REPOS=('core' 'extra' 'testing' 'staging-core' 'staging-extra') -# Libre Repositories from Parabola GNU/Hurd -LIBRE_MAIN_REPOS=('libre' 'libre-testing') - -# Multilib Repositories from Arch GNU/Hurd -MULTILIB_REPOS=('multilib' 'multilib-testing') -# Libre Multilib Repositories from Parabola GNU/Hurd -LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') - -# Extra Repositories -EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') - -# Community Repositories -COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') - -# Platform Repositories -PLATFORM_REPOS=() - -# Remote Repositories -PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") -PKG_MULTILIB_REPOS=("${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}") - -# Directories where packages are shared between repos -# *relative to REPO_DIR* -PKG_POOL="${PKG_POOL_DIR}/abslibre_${PLATFORM}" - -# Directories where sources are stored -SRC_POOL="${SRC_POOL_DIR}/abslibre_${PLATFORM}" - -# Build System and Repository scripts -BS_MAIN_NAME="abslibre_${PLATFORM}" -BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" -BS_DERIVATION_NAME="abs_${PLATFORM}" -BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" -BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" -# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' -BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" -BLACKLIST_NAME="blacklist_${PLATFORM}.txt" -BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" -BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" -SYNC_DERIVATION_SERVER='rsync.archhurd.org' - -# Base Repository -REPO_DIR="${SRV_DIR}/repo/abslibre_${PLATFORM}" - -CLEANUP_DESTDIR="${REPO_DIR}/old/packages" -CLEANUP_DRYRUN='false' -# Time in days to keep moved packages -CLEANUP_KEEP='30' - -SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" -SOURCE_CLEANUP_DRYRUN='true' -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP='30' diff --git a/config_abslibre_gnu+linux b/config_abslibre_gnu+linux deleted file mode 100644 index ec6fa8b..0000000 --- a/config_abslibre_gnu+linux +++ /dev/null @@ -1,69 +0,0 @@ -#!/hint/bash - -source "$(dirname "$(readlink -e "$0")")/config" - -# Parabola Platform -PLATFORM_NAME='GNU/Linux-libre' -PLATFORM='gnu+linux' - -# Multilib Support Archictectures -MULTILIB_ARCHES=('x86_64') - -# Archictectures -ARCHES=('i686' "${MULTILIB_ARCHES[@]}") - -# Main Repositories from Arch GNU/Linux -MAIN_REPOS=('core' 'extra' 'testing' 'community' 'community-testing') -# Libre Repositories from Parabola GNU/Linux-libre -LIBRE_MAIN_REPOS=('libre' 'libre-testing') - -# Multilib Repositories from Arch GNU/Linux -MULTILIB_REPOS=('multilib' 'multilib-testing') -# Libre Multilib Repositories from Parabola GNU/Linux-libre -LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') - -# Extra Repositories -EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') - -# Community Repositories -COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') - -# Platform Repositories -PLATFORM_REPOS=('nonsystemd' 'nonsystemd-testing') - -# Remote Repositories -PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") -PKG_MULTILIB_REPOS=("${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}") - -# Directories where packages are shared between repos -# *relative to REPO_DIR* -PKG_POOL="${PKG_POOL_DIR}/abslibre_${PLATFORM}" - -# Directories where sources are stored -SRC_POOL="${SRC_POOL_DIR}/abslibre_${PLATFORM}" - -# Build System and Repository scripts -BS_MAIN_NAME="abslibre_${PLATFORM}" -BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" -BS_DERIVATION_NAME="abs_${PLATFORM}" -BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" -BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" -# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' -BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" -BLACKLIST_NAME="blacklist_${PLATFORM}.txt" -BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" -BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" -SYNC_DERIVATION_SERVER='rsync.archlinux.org' - -# Base Repository -REPO_DIR="${SRV_DIR}/repo/abslibre_${PLATFORM}" - -CLEANUP_DESTDIR="${REPO_DIR}/old/packages" -CLEANUP_DRYRUN='false' -# Time in days to keep moved packages -CLEANUP_KEEP='30' - -SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" -SOURCE_CLEANUP_DRYRUN='true' -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP='30' diff --git a/config_abslibre_gnu+linux_arm b/config_abslibre_gnu+linux_arm deleted file mode 100644 index 4e35017..0000000 --- a/config_abslibre_gnu+linux_arm +++ /dev/null @@ -1,69 +0,0 @@ -#!/hint/bash - -source "$(dirname "$(readlink -e "$0")")/config" - -# Parabola Platform -PLATFORM_NAME='GNU/Linux-libre' -PLATFORM='gnu+linux' - -# Multilib Support Archictectures -MULTILIB_ARCHES=() - -# Archictectures -ARCHES=('arm' 'armv6h' 'armv7h' 'aarch64' "${MULTILIB_ARCHES[@]}") - -# Main Repositories from Arch GNU/Linux ARM -MAIN_REPOS=('core' 'extra' 'community' 'aur' 'alarm') -# Libre Repositories from Parabola GNU/Linux-libre -LIBRE_MAIN_REPOS=('libre' 'libre-testing') - -# Multilib Repositories from Arch GNU/Linux ARM -MULTILIB_REPOS=('multilib') -# Libre Multilib Repositories from Parabola GNU/Linux-libre -LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') - -# Extra Repositories -EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') - -# Community Repositories -COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') - -# Platform Repositories -PLATFORM_REPOS=('nonsystemd' 'nonsystemd-testing') - -# Remote Repositories -PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") -PKG_MULTILIB_REPOS=("${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}") - -# Directories where packages are shared between repos -# *relative to REPO_DIR* -PKG_POOL="${PKG_POOL_DIR}/abslibre_${PLATFORM}_arm" - -# Directories where sources are stored -SRC_POOL="${SRC_POOL_DIR}/abslibre_${PLATFORM}_arm" - -# Build System and Repository scripts -BS_MAIN_NAME="abslibre_${PLATFORM}_arm" -BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" -BS_DERIVATION_NAME="abs_${PLATFORM}_arm" -BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" -BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" -# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' -BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" -BLACKLIST_NAME="blacklist_${PLATFORM}_arm.txt" -BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" -BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" -SYNC_DERIVATION_SERVER='rsync.archlinuxarm.org' - -# Base Repository -REPO_DIR="${SRV_DIR}/repo/abslibre_${PLATFORM}_arm" - -CLEANUP_DESTDIR="${REPO_DIR}/old/packages" -CLEANUP_DRYRUN='false' -# Time in days to keep moved packages -CLEANUP_KEEP='30' - -SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" -SOURCE_CLEANUP_DRYRUN='true' -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP='30' diff --git a/config_abslibre_gnu+linux_legacy b/config_abslibre_gnu+linux_legacy deleted file mode 100644 index e9b0492..0000000 --- a/config_abslibre_gnu+linux_legacy +++ /dev/null @@ -1,69 +0,0 @@ -#!/hint/bash - -source "$(dirname "$(readlink -e "$0")")/config" - -# Parabola Platform -PLATFORM_NAME='GNU/Linux-libre' -PLATFORM='gnu+linux' - -# Multilib Support Archictectures -MULTILIB_ARCHES=('x86_64') - -# Archictectures -ARCHES=('i686' "${MULTILIB_ARCHES[@]}") - -# Main Repositories from Arch GNU/Linux -MAIN_REPOS=('core' 'extra' 'testing' 'community' 'community-testing') -# Libre Repositories from Parabola GNU/Linux-libre -LIBRE_MAIN_REPOS=('libre' 'libre-testing') - -# Multilib Repositories from Arch GNU/Linux -MULTILIB_REPOS=('multilib' 'multilib-testing') -# Libre Multilib Repositories from Parabola GNU/Linux-libre -LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') - -# Extra Repositories -EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') - -# Community Repositories -COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') - -# Platform Repositories -PLATFORM_REPOS=('nonsystemd' 'nonsystemd-testing') - -# Remote Repositories -PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") -PKG_MULTILIB_REPOS=("${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}") - -# Directories where packages are shared between repos -# *relative to REPO_DIR* -PKG_POOL="pool/parabola" - -# Directories where sources are stored -SRC_POOL="sources/parabola" - -# Build System and Repository scripts -BS_MAIN_NAME="abslibre" -BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" -BS_DERIVATION_NAME="abs" -BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" -BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" -# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' -BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" -BLACKLIST_NAME="blacklist.txt" -BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" -BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" -SYNC_DERIVATION_SERVER='rsync.archlinux.org' - -# Base Repository -REPO_DIR="${SRV_DIR}/repo/main" - -CLEANUP_DESTDIR="${REPO_DIR}/old/packages" -CLEANUP_DRYRUN='false' -# Time in days to keep moved packages -CLEANUP_KEEP='30' - -SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" -SOURCE_CLEANUP_DRYRUN='true' -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP='30' diff --git a/config_gnu+hurd b/config_gnu+hurd deleted file mode 100644 index eb33dc0..0000000 --- a/config_gnu+hurd +++ /dev/null @@ -1,59 +0,0 @@ -#!/hint/bash - -source "$(dirname "$(readlink -e "$0")")/config" - -# Parabola Platform -PLATFORM_NAME='GNU/Hurd' -PLATFORM='gnu+hurd' - -# Multilib Support Archictectures -MULTILIB_ARCHES=() - -# Archictectures -ARCHES=('i686' "${MULTILIB_ARCHES[@]}") - -# Main Repositories -MAIN_REPOS=('core' 'extra' 'testing') - -# Multilib Repositories -MULTILIB_REPOS=('multilib' 'multilib-testing') - -# Extra Repositories -EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') - -# Community Repositories -COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') - -# Platform Repositories -PLATFORM_REPOS=() - -# Remote Repositories -PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") -PKG_MULTILIB_REPOS=("${MULTILIB_REPOS[@]}") - -# Directories where packages are shared between repos -# *relative to REPO_DIR* -PKG_POOL="${PKG_POOL_DIR}/${PLATFORM}" - -# Directories where sources are stored -SRC_POOL="${SRC_POOL_DIR}/${PLATFORM}" - -# Build System and Repository scripts -BS_MAIN_NAME="pbs_${PLATFORM}" -BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" -BS_GIT="${GIT_BASE}/${BS_MAIN_NAME}/pmr.git" -# BS_GIT='http://projects.parabola.gnu/${BS_MAIN_NAME}/pmr.git' -BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" - -# Base Repository -REPO_DIR="${SRV_DIR}/repo/${PLATFORM}" - -CLEANUP_DESTDIR="${REPO_DIR}/old/packages" -CLEANUP_DRYRUN='false' -# Time in days to keep moved packages -CLEANUP_KEEP='30' - -SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" -SOURCE_CLEANUP_DRYRUN='true' -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP='30' diff --git a/config_gnu+linux b/config_gnu+linux deleted file mode 100644 index bfd421c..0000000 --- a/config_gnu+linux +++ /dev/null @@ -1,59 +0,0 @@ -#!/hint/bash - -source "$(dirname "$(readlink -e "$0")")/config" - -# Parabola Platform -PLATFORM_NAME='GNU/Linux-libre' -PLATFORM='gnu+linux' - -# Multilib Support Archictectures -MULTILIB_ARCHES=() # 'x86_64' - -# Archictectures -ARCHES=('mips64el' "${MULTILIB_ARCHES[@]}") # 'i686' - -# Main Repositories -MAIN_REPOS=('core' 'extra' 'testing') - -# Multilib Repositories -MULTILIB_REPOS=('multilib' 'multilib-testing') - -# Extra Repositories -EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') - -# Community Repositories -COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') - -# Platform Repositories -PLATFORM_REPOS=() - -# Remote Repositories -PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") -PKG_MULTILIB_REPOS=("${MULTILIB_REPOS[@]}") - -# Directories where packages are shared between repos -# *relative to REPO_DIR* -PKG_POOL="${PKG_POOL_DIR}/${PLATFORM}" - -# Directories where sources are stored -SRC_POOL="${SRC_POOL_DIR}/${PLATFORM}" - -# Build System and Repository scripts -BS_MAIN_NAME="pbs_${PLATFORM}" -BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" -BS_GIT="${GIT_BASE}/${BS_MAIN_NAME}/pmr.git" -# BS_GIT='http://projects.parabola.gnu/${BS_MAIN_NAME}/pmr.git' -BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" - -# Base Repository -REPO_DIR="${SRV_DIR}/repo/${PLATFORM}" - -CLEANUP_DESTDIR="${REPO_DIR}/old/packages" -CLEANUP_DRYRUN='false' -# Time in days to keep moved packages -CLEANUP_KEEP='30' - -SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" -SOURCE_CLEANUP_DRYRUN='true' -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP='30' diff --git a/config_platforms b/config_platforms deleted file mode 100644 index 4ef4353..0000000 --- a/config_platforms +++ /dev/null @@ -1,6 +0,0 @@ -#!/hint/bash - -# Parabola Platforms -MAIN_PLATFORMS=('gnu+linux') # 'gnu+hurd' -DERIVATIVE_PLATFORMS=('abslibre_gnu+linux' 'abslibre_gnu+linux_arm') # 'abslibre_gnu+hurd' -PLATFORMS=("${MAIN_PLATFORMS[@]}" "${DERIVATIVE_PLATFORMS[@]}") diff --git a/create-repo b/create-repo deleted file mode 100755 index c977442..0000000 --- a/create-repo +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# Creates repository structure - -source "$(dirname "$(readlink -e "$0")")/config_platforms" -source "$(dirname "$(readlink -e "$0")")/db-functions" - -create-repo-usage() { - msg "Usage: %s [platform] repo1 [repo2 ... repoX]" "${0##*/}" - exit 1 -} - -if [ "$#" -eq '0' ]; then - create-repo-usage -fi - -for 'platform' in "${PLATFORMS[@]}"; do - case "$1" in - --platform|-platform|-p) - - # Don't create [--platform], [-platform] or [-p] and [] repos - _repos=($@) && unset _repos[0] _repos[1] - - # Rename plataform name ($2) to easily script usage - _platform_name="${2/\//+}" _platform_name="${_platform_name,,}" - - if [ "${_platform_name}" == "${platform}" ]; then - source "$(dirname "$(readlink -e "$0")")/config_${2}" - - msg "Creating repos... from ${PLATFORM_NAME} platform" - for '_repo' in "${_repos[@]}"; do - msg2 "Creating [%s]" "${_repo_all}" - for '_arch' in "${ARCHES[@]}"; do - mkdir -p "${REPO_DIR}/${_repo}/os/${_arch}" || \ - error "Failed creating %s dir" "${_arch}" "from ${PLATFORM_NAME} platform" - done - done - - msg "Don't forget to add them to the PKG_REPOS array on %s" "$(dirname "$(readlink -e "$0")")/config_${2}" - else - create-repo-usage - fi - ;; - *) - source "$(dirname "$(readlink -e "$0")")/config_${platform}" - - msg "Creating repos... from all platforms" - for '_repo_all' in "${@}"; do - msg2 "Creating [%s]" "${_repo_all}" - for '_arch' in "${ARCHES[@]}"; do - mkdir -p "${REPO_DIR}/${_repo_all}/os/${_arch}" || \ - error "Failed creating %s dir" "${_arch}" "in all platforms" - done - done - - msg "Don't forget to add them to the PKG_REPOS array on %s" "$(dirname "$(readlink -e "$0")")/config_${platform}" - ;; - esac -done - -# if [ -n "${MULTILIB_ARCHES}" ]; then -# fi diff --git a/createrepos b/createrepos deleted file mode 100755 index 4416196..0000000 --- a/createrepos +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Creates the repo structure defined in config - -source "$(dirname "$(readlink -e "$0")")/config_platforms" -source "$(dirname "$(readlink -e "$0")")/db-functions" - -createrepos-usage() { - msg "Usage: %s [platform1 platform2 ... platformX]" "${0##*/}" - exit 1 -} - -for 'platform' in "${PLATFORMS[@]}"; do - if [ -n $1 ]; then - for '_platform' in "$@"; do - # Rename plataform name ($_platform) to easily script usage - _platform_name="${_platform/\//+}" _platform_name="${_platform_name,,}" - - if [ "${_platform_name}" == "${platform}" ]; then - source "$(dirname "$(readlink -e "$0")")/config_${_platform}" - - mkdir -p -- "${REPO_DIR}"/{"${PKG_POOL}","${SRC_POOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" - - "$(dirname "$(readlink -e "$0")")/create-repo" -p "${_platform}" "${PKG_REPOS[@]}" - else - createrepos-usage - fi - done - else - source "$(dirname "$(readlink -e "$0")")/config_${platform}" - - mkdir -p -- "${REPO_DIR}"/{"${PKG_POOL}","${SRC_POOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" - - "$(dirname "$(readlink -e "$0")")/create-repo" "${PKG_REPOS[@]}" - ;; - fi -done diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py deleted file mode 100755 index ac0194f..0000000 --- a/cron-jobs/check_archlinux/check_packages.py +++ /dev/null @@ -1,508 +0,0 @@ -#!/usr/bin/env python2 -# -# check_archlinux.py -# -# Original script by Scott Horowitz -# Rewritten by Xavier Chantry -# -# This script currently checks for a number of issues in your ABS tree: -# 1. Directories with missing PKGBUILDS -# 2. Invalid PKGBUILDs (bash syntax error for instance) -# 3. PKGBUILD names that don't match their directory -# 4. Duplicate PKGBUILDs -# 5. Valid arch's in PKGBUILDS -# 6. Missing (make-)dependencies -# 7. Hierarchy of repos (e.g., that a core package doesn't depend on -# a non-core package) -# 8. Circular dependencies - -import os,re,commands,getopt,sys,tarfile -import pdb - -import ctypes -_alpm = ctypes.cdll.LoadLibrary("libalpm.so") - -DBEXT='.db.tar.gz' - -packages = {} # pkgname : PacmanPackage -repopkgs = {} # pkgname : PacmanPackage -provisions = {} # provision : PacmanPackage -pkgdeps,makepkgdeps = {},{} # PacmanPackage : list of the PacmanPackage dependencies -invalid_pkgbuilds = [] -missing_pkgbuilds = [] -dups = [] - -dbonly = [] -absonly = [] - -mismatches = [] -missing_deps = [] -missing_makedeps = [] -invalid_archs = [] -dep_hierarchy = [] -makedep_hierarchy = [] -circular_deps = [] # pkgname>dep1>dep2>...>pkgname -checked_deps = [] - -class PacmanPackage: - def __init__(self): - self.name,self.version = "","" - self.base = "" - self.path,self.repo = "","" - self.deps,self.makedeps = [],[] - self.provides,self.conflicts = [],[] - self.archs = [] - -class Depend: - def __init__(self,name,version,mod): - self.name = name - self.version = version - self.mod = mod - -def parse_pkgbuilds(repos,arch): - for absroot in absroots: - for repo in repos: - cmd = os.path.dirname(os.path.realpath(sys.argv[0])) + '/parse_pkgbuilds.sh ' - cmd += arch + ' ' + absroot + '/' + repo - (status,output) = commands.getstatusoutput(cmd) - if status != 0: - print "Error : failed to run '%s'" % cmd - sys.exit() - parse_data(repo,output) - -def parse_data(repo,data): - attrname = None - - for line in data.split('\n'): - if line.startswith('%'): - attrname = line.strip('%').lower() - elif line.strip() == '': - attrname = None - elif attrname == "invalid": - if repo in repos: - invalid_pkgbuilds.append(line) - elif attrname == "missing": - if repo in repos: - missing_pkgbuilds.append(line) - elif attrname == "name": - pkg = PacmanPackage() - pkg.name = line - pkg.repo = repo - dup = None - if pkg.name in packages: - dup = packages[pkg.name] - else: - packages[pkg.name] = pkg - elif attrname == "base": - pkg.base = line - elif attrname == "version": - pkg.version = line - elif attrname == "path": - pkg.path = line - if dup != None and (pkg.repo in repos or dup.repo in repos): - dups.append(pkg.path + " vs. " + dup.path) - elif attrname == "arch": - pkg.archs.append(line) - elif attrname == "depends": - pkg.deps.append(line) - elif attrname == "makedepends": - pkg.makedeps.append(line) - elif attrname == "conflicts": - pkg.conflicts.append(line) - elif attrname == "provides": - pkg.provides.append(line) - -def parse_dbs(repos,arch): - dbpkgs = {} - for repo in repos: - pkgs = set([]) - db = tarfile.open(os.path.join(repodir,repo,'os',arch,repo + DBEXT)) - for line in db.getnames(): - if not '/' in line: - pkgs.add(line.rsplit('-',2)[0]) - dbpkgs[repo] = pkgs - return(dbpkgs) - -def splitdep(dep): - name = dep - version = "" - mod = "" - for char in (">=", "<=", "=", ">", "<"): - pos = dep.find(char) - if pos > -1: - name = dep[:pos] - version = dep[pos:].replace(char, "") - mod = char - break - return Depend(name,version,mod) - -def splitprov(prov): - name = prov - version = "" - pos = prov.find("=") - if pos > -1: - name = prov[:pos] - version = prov[pos:].replace("=", "") - return (name,version) - -def vercmp(v1,mod,v2): - """ - >>> vercmp("1.0", "<=", "2.0") - True - >>> vercmp("1:1.0", ">", "2.0") - True - >>> vercmp("1.0.2", ">=", "2.1.0") - False - """ - s1 = ctypes.c_char_p(v1) - s2 = ctypes.c_char_p(v2) - res = _alpm.alpm_pkg_vercmp(s1,s2) - if res == 0: - return (mod.find("=") > -1) - elif res < 0: - return (mod.find("<") > -1) - elif res > 0: - return (mod.find(">") > -1) - return False - - -def depcmp(name,version,dep): - if name != dep.name: - return False - if dep.version == "" or dep.mod == "": - return True - if version == "": - return False - return vercmp(version,dep.mod,dep.version) - -def provcmp(pkg,dep): - for prov in pkg.provides: - (provname,provver) = splitprov(prov) - if depcmp(provname,provver,dep): - return True - return False - -def verify_dep(dep): - dep = splitdep(dep) - if dep.name in packages: - pkg = packages[dep.name] - if depcmp(pkg.name,pkg.version,dep): - return [pkg] - if dep.name in provisions: - provlist = provisions[dep.name] - results = [] - for prov in provlist: - if provcmp(prov,dep): - results.append(prov) - return results - return [] - -def verify_deps(name,repo,deps): - pkg_deps = [] - missdeps = [] - hierarchy = [] - for dep in deps: - pkglist = verify_dep(dep) - if pkglist == []: - missdeps.append(repo + "/" + name + " --> '" + dep + "'") - else: - valid_repos = get_repo_hierarchy(repo) - pkgdep = None - for pkg in pkglist: - if pkg.repo in valid_repos: - pkgdep = pkg - break - if not pkgdep: - pkgdep = pkglist[0] - hierarchy.append((repo,name,pkgdep)) - - pkg_deps.append(pkgdep) - - return (pkg_deps,missdeps,hierarchy) - -def compute_deplist(pkg): - list = [] - stack = [pkg] - while stack != []: - dep = stack.pop() - if dep in pkgdeps: - for dep2 in pkgdeps[dep]: - if dep2 not in list: - list.append(dep2) - stack.append(dep2) - if dep in makepkgdeps: - for dep2 in makepkgdeps[dep]: - if dep2 not in list: - list.append(dep2) - stack.append(dep2) - return list - -def check_hierarchy(deph): - hierarchy = [] - for (repo,name,pkgdep) in deph: - deplist = compute_deplist(pkgdep) - valid_repos = get_repo_hierarchy(repo) - extdeps = [] - for dep in deplist: - if dep.repo not in valid_repos: - extdeps.append(dep.name) - string = repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name + " (" - string += "%s extra (make)deps to pull" % len(extdeps) - if 0 < len(extdeps) < 10: - string += " : " + ' '.join(extdeps) - string += ")" - hierarchy.append(string) - return hierarchy - -def get_repo_hierarchy(repo): - repo_hierarchy = {'core': ['core'], \ - 'extra': ['core', 'extra'], \ - 'community': ['core', 'extra', 'community'], \ - 'multilib': ['core', 'extra', 'community', 'multilib'] } - if repo in repo_hierarchy: - return repo_hierarchy[repo] - else: - return ['core','extra','community'] - -def verify_archs(name,repo,archs): - valid_archs = ['any', 'i686', 'x86_64'] - invalid_archs = [] - for arch in archs: - if arch not in valid_archs: - invalid_archs.append(repo + "/" + name + " --> " + arch) - return invalid_archs - -def find_scc(packages): - # reset all variables - global index,S,pkgindex,pkglowlink - index = 0 - S = [] - pkgindex = {} - pkglowlink = {} - cycles = [] - for pkg in packages: - tarjan(pkg) - -def tarjan(pkg): - global index,S,pkgindex,pkglowlink,cycles - pkgindex[pkg] = index - pkglowlink[pkg] = index - index += 1 - checked_deps.append(pkg) - S.append(pkg) - deps = [] - if pkg in pkgdeps: - deps = pkgdeps[pkg] - for dep in deps: - if dep not in pkgindex: - tarjan(dep) - pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep]) - elif dep in S: - pkglowlink[pkg] = min(pkglowlink[pkg],pkgindex[dep]) - if pkglowlink[pkg] == pkgindex[pkg]: - dep = S.pop() - if pkg == dep: - return - path = pkg.name - while pkg != dep: - path = dep.repo + "/" + dep.name + ">" + path - dep = S.pop() - path = dep.name + ">" + path - if pkg.repo in repos: - circular_deps.append(path) - -def print_heading(heading): - print "" - print "=" * (len(heading) + 4) - print "= " + heading + " =" - print "=" * (len(heading) + 4) - -def print_subheading(subheading): - print "" - print subheading - print "-" * (len(subheading) + 2) - -def print_missdeps(pkgname,missdeps) : - for d in missdeps: - print pkgname + " : " + d - -def print_result(list, subheading): - if len(list) > 0: - list.sort() - print_subheading(subheading) - for item in list: - print item - -def print_results(): - print_result(missing_pkgbuilds, "Missing PKGBUILDs") - print_result(invalid_pkgbuilds, "Invalid PKGBUILDs") - print_result(mismatches, "Mismatched Pkgnames") - print_result(dups, "Duplicate PKGBUILDs") - print_result(invalid_archs, "Invalid Archs") - print_result(missing_deps, "Missing Dependencies") - print_result(missing_makedeps, "Missing Makedepends") - print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") - print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") - print_result(circular_deps, "Circular Dependencies") - print_result(dbonly, "Packages found in db, but not in tree") - print_result(absonly,"Packages found in tree, but not in db") - print_subheading("Summary") - print "Missing PKGBUILDs: ", len(missing_pkgbuilds) - print "Invalid PKGBUILDs: ", len(invalid_pkgbuilds) - print "Mismatching PKGBUILD names: ", len(mismatches) - print "Duplicate PKGBUILDs: ", len(dups) - print "Invalid archs: ", len(invalid_archs) - print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) - print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) - print "Circular dependencies: ", len(circular_deps) - print "In db, but not in tree: ", len(dbonly) - print "In tree, but not in db: ", len(absonly) - print "" - -def print_usage(): - print "" - print "Usage: ./check_packages.py [OPTION]" - print "" - print "Options:" - print " --abs-tree= Check the specified tree(s) (default : /var/abs)" - print " --repos= Check the specified repos (default : core,extra)" - print " --arch= Check the specified arch (default : i686)" - print " --repo-dir= Check the dbs at the specified path (default : /srv/ftp)" - print " -h, --help Show this help and exit" - print "" - print "Examples:" - print "\n Check core and extra in existing abs tree:" - print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686" - print "\n Check community:" - print " ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686" - print "" - -if __name__ == "__main__": - ## Default path to the abs root directory - absroots = ["/var/abs"] - ## Default list of repos to check - repos = ['core', 'extra'] - ## Default arch - arch = "i686" - ## Default repodir - repodir = "/srv/ftp" - - try: - opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", - "arch=", "repo-dir="]) - except getopt.GetoptError: - print_usage() - sys.exit() - if opts != []: - for o, a in opts: - if o in ("--abs-tree"): - absroots = a.split(',') - elif o in ("--repos"): - repos = a.split(",") - elif o in ("--arch"): - arch = a - elif o in ("--repo-dir"): - repodir = a - else: - print_usage() - sys.exit() - if args != []: - print_usage() - sys.exit() - - for absroot in absroots: - if not os.path.isdir(absroot): - print "Error : the abs tree " + absroot + " does not exist" - sys.exit() - for repo in repos: - repopath = absroot + "/" + repo - if not os.path.isdir(repopath): - print("Warning : the repository " + repo + " does not exist in " + absroot) - - if not os.path.isdir(repodir): - print "Error: the repository directory %s does not exist" % repodir - sys.exit() - for repo in repos: - path = os.path.join(repodir,repo,'os',arch,repo + DBEXT) - if not os.path.isfile(path): - print "Error : repo DB %s : File not found" % path - sys.exit() - if not tarfile.is_tarfile(path): - print "Error : Cant open repo DB %s, not a valid tar file" % path - sys.exit() - # repos which need to be loaded - loadrepos = set([]) - for repo in repos: - loadrepos = loadrepos | set(get_repo_hierarchy(repo)) - - print_heading("Integrity Check " + arch + " of " + ",".join(repos)) - print("\nPerforming integrity checks...") - - print("==> parsing pkgbuilds") - parse_pkgbuilds(loadrepos,arch) - - # fill provisions - for name,pkg in packages.iteritems(): - for prov in pkg.provides: - provname=prov.split("=")[0] - if provname not in provisions: - provisions[provname] = [] - provisions[provname].append(pkg) - - # fill repopkgs - for name,pkg in packages.iteritems(): - if pkg.repo in repos: - repopkgs[name] = pkg - - print("==> parsing db files") - dbpkgs = parse_dbs(repos,arch) - - print("==> checking mismatches") - for name,pkg in repopkgs.iteritems(): - pkgdirname = pkg.path.split("/")[-1] - if name != pkgdirname and pkg.base != pkgdirname: - mismatches.append(name + " vs. " + pkg.path) - - print("==> checking archs") - for name,pkg in repopkgs.iteritems(): - archs = verify_archs(name,pkg.repo,pkg.archs) - invalid_archs.extend(archs) - - deph,makedeph = [],[] - - print("==> checking dependencies") - for name,pkg in repopkgs.iteritems(): - (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) - pkgdeps[pkg] = deps - missing_deps.extend(missdeps) - deph.extend(hierarchy) - - print("==> checking makedepends") - for name,pkg in repopkgs.iteritems(): - (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) - makepkgdeps[pkg] = makedeps - missing_makedeps.extend(missdeps) - makedeph.extend(hierarchy) - - print("==> checking hierarchy") - dep_hierarchy = check_hierarchy(deph) - makedep_hierarchy = check_hierarchy(makedeph) - - print("==> checking for circular dependencies") - # make sure pkgdeps is filled for every package - for name,pkg in packages.iteritems(): - if pkg not in pkgdeps: - (deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps) - pkgdeps[pkg] = deps - find_scc(repopkgs.values()) - - print("==> checking for differences between db files and pkgbuilds") - for repo in repos: - for pkg in dbpkgs[repo]: - if not (pkg in repopkgs and repopkgs[pkg].repo == repo): - dbonly.append("%s/%s" % (repo,pkg)) - for name,pkg in repopkgs.iteritems(): - if not name in dbpkgs[pkg.repo]: - absonly.append("%s/%s" % (pkg.repo,name)) - - print_results() diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh deleted file mode 100755 index b857ac8..0000000 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/bash - -# Usage : parse_pkgbuilds.sh arch -# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra - -exit() { return; } - -splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') -variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' "${splitpkg_overrides[@]}") -readonly -a variables splitpkg_overrides - -backup_package_variables() { - for var in "${splitpkg_overrides[@]}"; do - indirect="${var}_backup" - eval "${indirect}=(\"\${$var[@]}\")" - done -} - -restore_package_variables() { - for var in "${splitpkg_overrides[@]}"; do - indirect="${var}_backup" - if [ -n "${!indirect}" ]; then - eval "${var}=(\"\${$indirect[@]}\")" - else - unset "${var}" - fi - done -} - -print_info() { - echo -e "%NAME%\n$pkgname\n" - if [ -n "$epoch" ]; then - echo -e "%VERSION%\n$epoch:$pkgver-$pkgrel\n" - else - echo -e "%VERSION%\n$pkgver-$pkgrel\n" - fi - echo -e "%PATH%\n$dir\n" - - if [ -n "$pkgbase" ]; then - echo -e "%BASE%\n$pkgbase\n" - fi - - if [ -n "$arch" ]; then - echo "%ARCH%" - for i in "${arch[@]}"; do echo "$i"; done - echo "" - fi - if [ -n "$depends" ]; then - echo "%DEPENDS%" - for i in "${depends[@]}"; do - echo "$i" - done - echo "" - fi - if [ -n "$makedepends" ]; then - echo "%MAKEDEPENDS%" - for i in "${makedepends[@]}"; do - echo "$i" - done - echo "" - fi - if [ -n "$conflicts" ]; then - echo "%CONFLICTS%" - for i in "${conflicts[@]}"; do echo "$i"; done - echo "" - fi - if [ -n "$provides" ]; then - echo "%PROVIDES%" - for i in "${provides[@]}"; do echo "$i"; done - echo "" - fi -} - -source_pkgbuild() { - ret=0 - dir=$1 - pkgbuild=$dir/PKGBUILD - for var in "${variables[@]}"; do - unset "${var}" - done - source "$pkgbuild" &>/dev/null || ret=$? - - # ensure $pkgname and $pkgver variables were found - if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then - echo -e "%INVALID%\n$pkgbuild\n" - return 1 - fi - - if [ "${#pkgname[@]}" -gt "1" ]; then - pkgbase=${pkgbase:-${pkgname[0]}} - for pkg in "${pkgname[@]}"; do - if [ "$(type -t "package_${pkg}")" != "function" ]; then - echo -e "%INVALID%\n$pkgbuild\n" - return 1 - else - backup_package_variables - pkgname=$pkg - while IFS= read -r line; do - var=${line%%=*} - var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters - for realvar in "${variables[@]}"; do - if [ "$var" == "$realvar" ]; then - eval $line - break - fi - done - done < <(type "package_${pkg}") - print_info - restore_package_variables - fi - done - else - echo - print_info - fi - - return 0 -} - -find_pkgbuilds() { - #Skip over some dirs - local d="${1##*/}" - if [ "$d" = "CVS" -o "$d" = ".svn" ]; then - return - fi - - if [ -f "$1/PKGBUILD" ]; then - source_pkgbuild "$1" - return - fi - empty=1 - for dir in "$1"/*; do - if [ -d "$dir" ]; then - find_pkgbuilds "$dir" - unset empty - fi - done - if [ -n "$empty" ]; then - echo -e "%MISSING%\n$1\n" - fi -} - -if [ -z "$1" -o -z "$2" ]; then - exit 1 -fi - -CARCH=$1 -shift -for dir in "$@"; do - find_pkgbuilds "$dir" -done - -exit 0 diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer deleted file mode 100755 index 1a05521..0000000 --- a/cron-jobs/devlist-mailer +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -#Dummy helper to send email to arch-dev -# It does nothing if no output - -LIST="arch-dev-public@archlinux.org" -#LIST="aaronmgriffin@gmail.com" -FROM="repomaint@archlinux.org" - -SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" -if [ $# -ge 1 ]; then - SUBJECT="$1 $(date +"%d-%m-%Y")" -fi - -if [ $# -ge 2 ]; then - LIST="$2" -fi - -stdin="$(cat)" -#echo used to strip whitespace for checking for actual data -if [ -n "$(echo $stdin)" ]; then - - echo "Subject: $SUBJECT -To: $LIST -From: $FROM - -$stdin" | /usr/sbin/sendmail -F"$FROM" "$LIST" - -fi diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup deleted file mode 100755 index 4a2b418..0000000 --- a/cron-jobs/ftpdir-cleanup +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/../config" -. "$(dirname "$(readlink -e "$0")")/../db-functions" - -clean_pkg() { - local pkg - local target - - if ! "${CLEANUP_DRYRUN}"; then - for pkg in "$@"; do - if [ -h "$pkg" ]; then - rm -f "$pkg" "$pkg.sig" - else - mv_acl "$pkg" "$CLEANUP_DESTDIR/${pkg##*/}" - if [ -e "$pkg.sig" ]; then - mv_acl "$pkg.sig" "$CLEANUP_DESTDIR/${pkg##*/}.sig" - fi - touch "${CLEANUP_DESTDIR}/${pkg##*/}" - fi - done - fi -} - -"${CLEANUP_DRYRUN}" && warning 'dry run mode is active' - -for repo in "${PKGREPOS[@]}"; do - for arch in "${ARCHES[@]}"; do - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then - continue - fi - # get a list of actual available package files - find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}" - # get a list of package files defined in the repo db - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" - - missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) - if [ ${#missing_pkgs[@]} -ge 1 ]; then - error "Missing packages in [%s] (%s)..." "${repo}" "${arch}" - for missing_pkg in "${missing_pkgs[@]}"; do - msg2 '%s' "${missing_pkg}" - done - fi - - old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) - if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old packages from [%s] (%s)..." "${repo}" "${arch}" - for old_pkg in "${old_pkgs[@]}"; do - msg2 '%s' "${old_pkg}" - clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" - done - fi - done -done - -# get a list of all available packages in the pacakge pool -find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" -# create a list of packages in our db -cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" - -old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) -if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old packages from package pool..." - for old_pkg in "${old_pkgs[@]}"; do - msg2 '%s' "${old_pkg}" - clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" - done -fi - -old_pkgs=($(find "${CLEANUP_DESTDIR}" -type f -name "*${PKGEXT}" -mtime +"${CLEANUP_KEEP}" -printf '%f\n')) -if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old packages from the cleanup directory..." - for old_pkg in "${old_pkgs[@]}"; do - msg2 '%s' "${old_pkg}" - if ! "${CLEANUP_DRYRUN}"; then - rm -f "${CLEANUP_DESTDIR}/${old_pkg}" - rm -f "${CLEANUP_DESTDIR}/${old_pkg}.sig" - fi - done -fi - -for repo in "${PKGREPOS[@]}"; do - for arch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${arch}" - done -done - -script_unlock diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check deleted file mode 100755 index 7459380..0000000 --- a/cron-jobs/integrity-check +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -dirname="$(dirname "$(readlink -e "$0")")" - -. "${dirname}/../config" -. "${dirname}/../db-functions" - -script_lock - -if [ $# -ne 1 ]; then - die "usage: %s " "${0##*/}" -fi -mailto=$1 - -check() { - "${dirname}"/check_archlinux/check_packages.py \ - --repos="${repos}" \ - --abs-tree="/srv/abs/rsync/${arch},/srv/abs/rsync/any" \ - --repo-dir="${FTP_BASE}" \ - --arch="${arch}" \ - 2>&1 | "${dirname}"/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" -} - -repos='core,extra,community' -arch='i686' -check - -repos='core,extra,community,multilib' -arch='x86_64' -check - -script_unlock diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents deleted file mode 100755 index 2eb0978..0000000 --- a/cron-jobs/make_repo_torrents +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# Copyright (C) 2014 Joseph Graham -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# This script finds any updated packages and calls -# `make_indivudual_torrent' for each of them. - -username=$( id -un ) - -case "${username}" in - repo | root ) - true - ;; - * ) - echo "This script must be run as repo user or root user." - echo "ByeBye!" - exit 1 - ;; -esac - -# pacman doesn't support multiple different packages of the same name, -# so it's OK to just stuff all the torrents into a single directory. -script_directory="$(dirname "$(readlink -e "$0")")/.." -. "$(dirname "$(readlink -e "$0")")/../config" -public_location="$FTP_BASE/" -torrent_location="$FTP_BASE/torrents/" - -cd "${torrent_location}" - -# Find any directories that might have packages in then -find "${public_location}" -name 'os' -type 'd' | -while read dir -do - # Find any packages - find "${dir}" -name '*\.pkg\.tar\.xz' | - while read pkg - do - pkg_name="${pkg##*/}" - - if [[ -h "${pkg}" ]] # check if it's a symbolic link - then - # We get the target of the symlink - pkg=$( readlink -f "${pkg}" ) - fi - - # If a .torrent file does not already exist for this package, we call - # `make_individual_torrent' to make it. - if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] - then - "$script_directory/make_individual_torrent" "${pkg}" "${public_location}" - fi - done -done - -if [[ "${username}" == root ]] -then - chown repo * -fi diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check deleted file mode 100755 index 239f042..0000000 --- a/cron-jobs/repo-sanity-check +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Solves issue165... on the old roundup install. From the database -# backups, the title was "Older/deprecated packages never leave the -# repo", I don't know how the body of the issue is stored in the DB, -# but the title says enough, I think. - -. "$(dirname "$(readlink -e "$0")")/../config" -. "$(dirname "$(readlink -e "$0")")/../db-functions" - -# Traverse all repos -for _repo in "${PKGREPOS[@]}"; do - msg "Cleaning up [%s]" "${_repo}" - - # Find all pkgnames on this repo's abs - on_abs=($( - find "${SVNREPO}/${_repo}" -name PKGBUILD | \ - while read pkgbuild; do - source "${pkgbuild}" >/dev/null 2>&1 - # cleanup to save memory - unset build package source md5sums pkgdesc pkgver pkgrel epoch \ - url license arch depends makedepends optdepends options \ - >/dev/null 2>&1 - - # also cleanup package functions - for _pkg in "${pkgname[@]}"; do - unset "package_${pkg}" >/dev/null 2>&1 - done - - # this fills the on_abs array - echo "${pkgname[@]}" - done - )) - - # quit if abs is empty - if [ ${#on_abs[*]} -eq 0 ]; then - warning "[%s]'s ABS tree is empty, skipping" "${_repo}" - break - fi - - # Find all pkgnames on repos - on_repo=($( - find "${FTP_BASE}/${_repo}" -name "*.pkg.tar.?z" \ - -printf "%f\n" | sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" - )) - - # Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 \ - <(printf '%s\n' "${on_abs[@]}" | sort -u) \ - <(printf '%s\n' "${on_repo[@]}" | sort -u) )) - - # Remove them from databases, ftpdir-cleanup will take care of the rest - find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \ - -exec repo-remove {} "${remove[@]}" \; >/dev/null 2>&1 - - msg2 "Removed the following packages:" - plain '%s' "${remove[@]}" -done diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs deleted file mode 100755 index c12a128..0000000 --- a/cron-jobs/sourceballs +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash - -dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../config" -. "${dirname}/../db-functions" -pushd "${WORKDIR}" >/dev/null - -script_lock - -for repo in "${PKGREPOS[@]}"; do - for arch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${arch}" || exit 1 - done -done - -#adjust the nice level to run at a lower priority -renice +10 -p $$ > /dev/null - -# Create a readable file for each repo with the following format -# - [ ] -for repo in "${PKGREPOS[@]}"; do - for arch in "${ARCHES[@]}"; do - # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then - continue - fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ - | awk '/^%NAME%/ { getline b }; - /^%BASE%/ { getline b }; - /^%VERSION%/ { getline v }; - /^%LICENSE%/,/^$/ { - if ( !/^%LICENSE%/ ) { l=l" "$0 } - }; - /^%ARCH%/ { - getline a; - printf "%s %s %s %s\n", b, v, a, l; - l=""; - }' - done | sort -u > "${WORKDIR}/db-${repo}" -done - -for repo in "${PKGREPOS[@]}"; do - for arch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${arch}" - done -done - -# Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" - -# Check for all packages if we need to build a source package -for repo in "${PKGREPOS[@]}"; do - newpkgs=() - failedpkgs=() - while read line; do - pkginfo=("${line}") - pkgbase=${pkginfo[0]} - pkgver=${pkginfo[1]} - pkgarch=${pkginfo[2]} - pkglicense=("${pkginfo[@]:3}") - - # Should this package be skipped? - if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then - continue - fi - # Commenting out, we'll sourceball everything - # Check if the license or .force file does not enforce creating a source package -# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then -# continue -# fi - # Store the expected file name of the source package - echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" - - # Build the source package if its not already there - if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then - # Check if we had failed before - if in_array "${pkgbase}-${pkgver}${SRCEXT}" "${failedpkgs[@]}"; then - continue - fi - - # Get the sources from xbs - mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - cp -a "$(xbs releasepath "${pkgbase}" "${repo}" "${pkgarch}")" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 - if [ $? -ge 1 ]; then - failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") - continue - fi - - # Build the actual source package - pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - SRCPKGDEST=. makepkg --nocolor --allsource --ignorearch --skippgpcheck >"${WORKDIR}/${pkgbase}.log" 2>&1 - if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" - # Avoid creating the same source package for every arch - echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" - newpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") - else - failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") - cat "${WORKDIR}/${pkgbase}.log" >> "${WORKDIR}/makepkg-fail.log" - fi - popd >/dev/null - fi - done < "${WORKDIR}/db-${repo}" - - if [ ${#newpkgs[@]} -ge 1 ]; then - msg "Adding source packages for [%s]..." "${repo}" - for new_pkg in "${newpkgs[@]}"; do - msg2 '%s' "${new_pkg}" - done - fi - if [ ${#failedpkgs[@]} -ge 1 ]; then - msg "Failed to create source packages for [%s]..." "${repo}" - for failed_pkg in "${failedpkgs[@]}"; do - msg2 '%s' "${failed_pkg}" - done - fi -done - -# Cleanup old source packages -cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" -old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) - -if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old source packages..." - "${SOURCE_CLEANUP_DRYRUN}" && warning 'dry run mode is active' - for old_pkg in "${old_pkgs[@]}"; do - msg2 '%s' "${old_pkg}" - if ! "${SOURCE_CLEANUP_DRYRUN}"; then - mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" - touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" - fi - done -fi - -old_pkgs=($(find "${SOURCE_CLEANUP_DESTDIR}" -type f -name "*${SRCEXT}" -mtime +"${SOURCE_CLEANUP_KEEP}" -printf '%f\n')) -if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old source packages from the cleanup directory..." - for old_pkg in "${old_pkgs[@]}"; do - msg2 '%s' "${old_pkg}" - "${SOURCE_CLEANUP_DRYRUN}" || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" - done -fi - -if [ -f "${WORKDIR}/makepkg-fail.log" ]; then - msg "Log of failed packages" - cat "${WORKDIR}/makepkg-fail.log" -fi - -script_unlock diff --git a/cron-jobs/sourceballs.force b/cron-jobs/sourceballs.force deleted file mode 100644 index badf15d..0000000 --- a/cron-jobs/sourceballs.force +++ /dev/null @@ -1,4 +0,0 @@ -faad2 -wxgtk -wxpython -glhack diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip deleted file mode 100644 index 14d6f4b..0000000 --- a/cron-jobs/sourceballs.skip +++ /dev/null @@ -1,29 +0,0 @@ -nexuiz-data -torcs-data -tremulous-data -ufoai-data -frogatto-data -vdrift-data -naev-data -btanks-data -wesnoth-data -texlive-bin -texlive-bibtexextra -texlive-core -texlive-fontsextra -texlive-formatsextra -texlive-games -texlive-genericextra -texlive-htmlxml -texlive-humanities -texlive-langcjk -texlive-langcyrillic -texlive-langextra -texlive-langgreek -texlive-latexextra -texlive-music -texlive-pictures -texlive-plainextra -texlive-pstricks -texlive-publishers -texlive-science diff --git a/db-check-nonfree b/db-check-nonfree deleted file mode 100755 index 37b7cf6..0000000 --- a/db-check-nonfree +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -ge 1 ]; then - error "Calling %s with a specific repository is not supported" "${0##*/}" - exit 1 -fi - -# TODO: this might lock too much (architectures) -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${pkgarch}" || exit 1 - done -done - -msg "Check nonfree in repo:" -nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u)) -for repo in "${ARCHREPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - msg2 "%s %s" "$repo" "$pkgarch" - if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then - continue - fi - unset dbpkgs - unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in "${dbpkgs[@]}"; do - if in_array "${pkgname}" "${nonfree[@]}"; then - cleanpkgs+=("${pkgname}") - fi - done - if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: %s" "${cleanpkgs[*]}" - arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" - fi - done -done - -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${pkgarch}" - done -done diff --git a/db-check-package-libraries.py b/db-check-package-libraries.py deleted file mode 100755 index bc2349b..0000000 --- a/db-check-package-libraries.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Check which libraries are provided or required by a package, store -this in a database, update and list broken packages. - -Dependencies: - -- Python 3.2 or later with SQLite 3 support - -- ``bsdtar`` - -- ``readelf`` -""" - - -import os.path -import re -import sqlite3 -import subprocess -import tempfile - - -#: Regexp matching an interesting dynamic entry. -_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" - "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") - - -def make_db(path): - """Make a new, empty, library database at *path*.""" - con = sqlite3.connect(path) - con.executescript(""" -create table provided( - library varchar not null, - package varchar not null -); -create table used( - library varchar not null, - package varchar not null -); -""") - con.close() - - -def begin(database): - """Connect to *database* and start a transaction.""" - con = sqlite3.connect(database) - con.execute("begin exclusive") - return con - - -def add_provided(con, package, libraries): - """Write that *package* provides *libraries*.""" - for library in libraries: - con.execute("insert into provided (package, library) values (?,?)", - (package, library)) - - -def add_used(con, package, libraries): - """Write that *package* uses *libraries*.""" - for library in libraries: - con.execute("insert into used (package, library) values (?,?)", - (package, library)) - - -def remove_package(con, package): - """Remove all entries for a package.""" - con.execute("delete from provided where package=?", (package,)) - con.execute("delete from used where package=?", (package,)) - - -def add_package(con, package): - """Add entries from a named *package*.""" - # Extract to a temporary directory. This could be done more - # efficiently, since there is no need to store more than one file - # at once. - with tempfile.TemporaryDirectory() as temp: - tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) - tar.communicate() - with open(os.path.join(temp, ".PKGINFO")) as pkginfo: - for line in pkginfo: - if line.startswith("pkgname ="): - pkgname = line[len("pkgname ="):].strip() - break - # Don't list previously removed libraries. - remove_package(con, pkgname) - provided = set() - used = set() - # Search for ELFs. - for dirname, dirnames, filenames in os.walk(temp): - assert dirnames is not None # unused, avoid pylint warning - for file_name in filenames: - path = os.path.join(dirname, file_name) - with open(path, "rb") as file_object: - if file_object.read(4) != b"\177ELF": - continue - readelf = subprocess.Popen(("readelf", "-d", path), - stdout=subprocess.PIPE) - for line in readelf.communicate()[0].split(b"\n"): - match = _DYNAMIC.match(line.decode("ascii")) - if match: - if match.group(1) == "SONAME": - provided.add(match.group(2)) - elif match.group(1) == "NEEDED": - used.add(match.group(2)) - else: - raise AssertionError("unknown entry type " - + match.group(1)) - add_provided(con, pkgname, provided) - add_used(con, pkgname, used) - - -def init(arguments): - """Initialize.""" - make_db(arguments.database) - - -def add(arguments): - """Add packages.""" - con = begin(arguments.database) - for package in arguments.packages: - add_package(con, package) - con.commit() - con.close() - - -def remove(arguments): - """Remove packages.""" - con = begin(arguments.database) - for package in arguments.packages: - remove_package(con, package) - con.commit() - con.close() - - -def check(arguments): - """List broken packages.""" - con = begin(arguments.database) - available = set(row[0] for row - in con.execute("select library from provided")) - for package, library in con.execute("select package, library from used"): - if library not in available: - print(package, "needs", library) - con.close() - - -def main(): - """Get arguments and run the command.""" - from argparse import ArgumentParser - parser = ArgumentParser(prog="check-package-libraries.py", - description="Check packages for " - "provided/needed libraries") - parser.add_argument("-d", "--database", type=str, - help="Database file to use", - default="package-libraries.sqlite") - subparsers = parser.add_subparsers() - subparser = subparsers.add_parser(name="init", - help="initialize the database") - subparser.set_defaults(command=init) - subparser = subparsers.add_parser(name="add", - help="add packages to database") - subparser.add_argument("packages", nargs="+", type=str, - help="package files to add") - subparser.set_defaults(command=add) - subparser = subparsers.add_parser(name="remove", - help="remove packages from database") - subparser.add_argument("packages", nargs="+", type=str, - help="package names to remove") - subparser.set_defaults(command=remove) - subparser = subparsers.add_parser(name="check", - help="list broken packages") - subparser.set_defaults(command=check) - arguments = parser.parse_args() - arguments.command(arguments) - - -if __name__ == "__main__": - main() diff --git a/db-cleanup b/db-cleanup deleted file mode 100755 index ffa2601..0000000 --- a/db-cleanup +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Syncs pools against themselves using database contents as filter to cleanup -# them up -# License: GPLv3 - -# Principles -# * Get repos dbs contents -# * Make them a include list -# * Rsync pools against themselves removing excluded files -# * Instant cleanup! - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -EXTRAFLAGS=() -"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) - -filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") -trap "rm -f -- $(printf %q "$filter")" EXIT - -for _repo in "${PKGREPOS[@]}"; do - for _arch in "${ARCHES[@]}"; do - msg "Getting %s-%s database" "${_repo}" "${_arch}" - - dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" - - if [ ! -r "${dbfile}" ]; then - warning "Not found" - continue - fi - - # Echo the contents into a filter file - bsdtar tf "${dbfile}" | \ - cut -d'/' -f1 | \ - sort -u | \ - sed "s|$|*|" >> "$filter" - - done -done - -msg "Removing old files:" - -for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do - msg2 '%s' "${POOL}" - - rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ - --include-from="$filter" \ - --exclude="*" \ - "${FTP_BASE}/${POOL}/" \ - "${FTP_BASE}/${POOL}/" -done - -msg "Removing dead symlinks:" -actions=(-print) -"${CLEANUP_DRYRUN}" || actions+=(-delete) -find -L "${FTP_BASE}/" -type l "${actions[@]}" diff --git a/db-functions b/db-functions deleted file mode 100644 index d76aa41..0000000 --- a/db-functions +++ /dev/null @@ -1,524 +0,0 @@ -#!/hint/bash - -# Some PKGBUILDs need CARCH to be set -CARCH="x86_64" - -# Useful functions -UMASK="" -set_umask () { - [ "$UMASK" == "" ] && UMASK="$(umask)" - export UMASK - umask 002 -} - -restore_umask () { - umask "$UMASK" >/dev/null -} - -# just like mv -f, but we touch the file and then copy the content so -# default ACLs in the target dir will be applied -mv_acl() { - rm -f "$2" - touch "$2" - cat "$1" >"$2" || return 1 - rm -f "$1" -} - -# set up general environment -WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -LOCKS=() -REPO_MODIFIED=0 - -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW -if [[ -t 2 ]]; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" -fi -readonly ALL_OFF BOLD BLUE GREEN RED YELLOW - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> WARNING:${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> ERROR${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -## -# usage : in_array( $needle, $haystack ) -# return : 0 - found -# 1 - not found -## -in_array() { - local needle=$1; shift - [[ -z $1 ]] && return 1 # Not Found - local item - for item in "$@"; do - [[ $item = "$needle" ]] && return 0 # Found - done - return 1 # Not Found -} - -## -# usage : get_full_version( $epoch, $pkgver, $pkgrel ) -# return : full version spec, including epoch (if necessary), pkgver, pkgrel -## -get_full_version() { - if [[ $1 -eq 0 ]]; then - # zero epoch case, don't include it in version - echo "$2-$3" - else - echo "$1:$2-$3" - fi -} - -script_lock() { - local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" - if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" - error "Script %s is already locked by %s." "${0##*/}" "$_owner" - exit 1 - else - set_umask - return 0 - fi -} - -script_unlock() { - local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" - if [ ! -d "$LOCKDIR" ]; then - warning "Script %s was not locked!" "${0##*/}" - restore_umask - return 1 - else - rmdir "$LOCKDIR" - restore_umask - return 0 - fi -} - -cleanup() { - local l - local repo - local arch - - trap - EXIT INT QUIT TERM - for l in "${LOCKS[@]}"; do - repo=${l%.*} - arch=${l#*.} - if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then - msg "Removing left over lock from [%s] (%s)" "${repo}" "${arch}" - repo_unlock "$repo" "$arch" - fi - done - if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then - msg "Removing left over lock from %s" "${0##*/}" - script_unlock - fi - rm -rf "$WORKDIR" - - if (( REPO_MODIFIED )); then - date +%s > "${FTP_BASE}/lastupdate" - date -u +%s > "${FTP_BASE}/lastsync" - fi - - [ "$1" ] && exit "$1" -} - -abort() { - msg 'Aborting...' - cleanup 0 -} - -die() { - error "$@" - cleanup 1 -} - -trap abort INT QUIT TERM HUP -trap cleanup EXIT - - -#repo_lock [timeout] -repo_lock () { - local LOCKDIR="$TMPDIR/.repolock.$1.$2" - local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" - local FILESLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${FILESEXT}.lck" - local _count - local _trial - local _timeout - local _lockblock - local _owner - - # This is the lock file used by repo-add and repo-remove - if [ -f "${DBLOCKFILE}" ]; then - error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$DBLOCKFILE")" - return 1 - fi - if [ -f "${FILESLOCKFILE}" ]; then - error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$FILESLOCKFILE")" - return 1 - fi - - if [ $# -eq 2 ]; then - _lockblock=true - _trial=0 - elif [ $# -eq 3 ]; then - _lockblock=false - _timeout=$3 - let _trial=$_timeout/$LOCK_DELAY - fi - - _count=0 - while [ "$_count" -le "$_trial" ] || "$_lockblock" ; do - if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" - warning "Repo [%s] (%s) is already locked by %s." "${1}" "${2}" "$_owner" - msg2 "Retrying in %d seconds..." "$LOCK_DELAY" - else - LOCKS+=("$1.$2") - set_umask - return 0 - fi - sleep "$LOCK_DELAY" - let _count=$_count+1 - done - - error "Repo [%s] (%s) is already locked by %s. Giving up!" "${1}" "${2}" "$_owner" - return 1 -} - -repo_unlock () { #repo_unlock - local LOCKDIR="$TMPDIR/.repolock.$1.$2" - if [ ! -d "$LOCKDIR" ]; then - warning "Repo lock [%s] (%s) was not locked!" "${1}" "${2}" - restore_umask - return 1 - else - rmdir "$LOCKDIR" - restore_umask - return 0 - fi -} - -# usage: _grep_pkginfo pkgfile pattern -_grep_pkginfo() { - local _ret - - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" - echo "${_ret#${2} = }" -} - - -# Get the package base or name as fallback -getpkgbase() { - local _base - - _base="$(_grep_pkginfo "$1" "pkgbase")" - if [ -z "$_base" ]; then - getpkgname "$1" - else - echo "$_base" - fi -} - -issplitpkg() { - local _base - - _base="$(_grep_pkginfo "$1" "pkgbase")" - if [ -z "$_base" ]; then - return 1 - else - return 0 - fi -} - -# Get the package name -getpkgname() { - local _name - - _name="$(_grep_pkginfo "$1" "pkgname")" - if [ -z "$_name" ]; then - error "Package '%s' has no pkgname in the PKGINFO. Fail!" "$1" - exit 1 - fi - - echo "$_name" -} - -# Get the pkgver-pkgrel of this package -getpkgver() { - local _ver - - _ver="$(_grep_pkginfo "$1" "pkgver")" - if [ -z "$_ver" ]; then - error "Package '%s' has no pkgver in the PKGINFO. Fail!" "$1" - exit 1 - fi - - echo "$_ver" -} - -getpkgarch() { - local _ver - - _ver="$(_grep_pkginfo "$1" "arch")" - if [ -z "$_ver" ]; then - error "Package '%s' has no arch in the PKGINFO. Fail!" "$1" - exit 1 - fi - - echo "$_ver" -} - -getpkgfile() { - if [[ ${#} -ne 1 ]]; then - error 'No canonical package found!' - exit 1 - elif [ ! -f "${1}" ]; then - error "Package %s not found!" "${1}" - exit 1 - elif "${REQUIRE_SIGNATURE}" && [ ! -f "${1}.sig" ]; then - error "Package signature %s not found!" "${1}.sig" - exit 1 - fi - - echo "${1}" -} - -getpkgfiles() { - local f - if [ ! -z "$(printf '%s\n' "${@%\.*}" | sort | uniq -D)" ]; then - error 'Duplicate packages found!' - exit 1 - fi - - for f in "${@}"; do - if [ ! -f "${f}" ]; then - error "Package %s not found!" "${f}" - exit 1 - elif "${REQUIRE_SIGNATURE}" && [ ! -f "${f}.sig" ]; then - error "Package signature %s not found!" "${f}.sig" - exit 1 - fi - done - - echo "${@}" -} - -check_pkgfile() { - local pkgfile=$1 - - local pkgname="$(getpkgname "${pkgfile}")" - [ $? -ge 1 ] && return 1 - local pkgver="$(getpkgver "${pkgfile}")" - [ $? -ge 1 ] && return 1 - local pkgarch="$(getpkgarch "${pkgfile}")" - [ $? -ge 1 ] && return 1 - - in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 - - if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then - return 0 - else - return 1 - fi -} - -check_pkgxbs() { - local pkgfile="${1}" - local _pkgbase="$(getpkgbase "${pkgfile}")" - [ $? -ge 1 ] && return 1 - local _pkgname="$(getpkgname "${pkgfile}")" - [ $? -ge 1 ] && return 1 - local _pkgver="$(getpkgver "${pkgfile}")" - [ $? -ge 1 ] && return 1 - local _pkgarch="$(getpkgarch "${pkgfile}")" - [ $? -ge 1 ] && return 1 - local repo="${2}" - - in_array "${repo}" "${PKGREPOS[@]}" || return 1 - - local xbsver="$(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; get_full_version "${_pkgname}")" - [ "${xbsver}" == "${_pkgver}" ] || return 1 - - local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) - in_array "${_pkgname}" "${xbsnames[@]}" || return 1 - - return 0 -} - -check_splitpkgs() { - local repo="${1}" - shift - local pkgfiles=("${@}") - local pkgfile - local pkgdir - local xbsname - - mkdir -p "${WORKDIR}/check_splitpkgs/" - pushd "${WORKDIR}/check_splitpkgs" >/dev/null - - for pkgfile in "${pkgfiles[@]}"; do - issplitpkg "${pkgfile}" || continue - local _pkgbase="$(getpkgbase "${pkgfile}")" - msg2 "Checking %s" "$_pkgbase" - local _pkgname="$(getpkgname "${pkgfile}")" - local _pkgarch="$(getpkgarch "${pkgfile}")" - mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" - echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" - - local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) - printf '%s\n' "${xbsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" - done - popd >/dev/null - - for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do - [ ! -d "${pkgdir}" ] && continue - sort -u "${pkgdir}/staging" -o "${pkgdir}/staging" - sort -u "${pkgdir}/xbs" -o "${pkgdir}/xbs" - if [ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/xbs")" ]; then - return 1 - fi - done - - return 0 -} - -check_pkgrepos() { - local pkgfile=$1 - - local pkgname="$(getpkgname "${pkgfile}")" - [ $? -ge 1 ] && return 1 - local pkgver="$(getpkgver "${pkgfile}")" - [ $? -ge 1 ] && return 1 - local pkgarch="$(getpkgarch "${pkgfile}")" - [ $? -ge 1 ] && return 1 - - [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 - - local repo - local arch - for repo in "${PKGREPOS[@]}"; do - for arch in "${ARCHES[@]}"; do - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}.sig" ] && return 1 - done - done - - return 0 -} - -#usage: chk_license ${license[@]}" -chk_license() { - local l - for l in "${@}"; do - in_array "${l}" "${ALLOWED_LICENSES[@]}" && return 0 - done - - return 1 -} - -check_repo_permission() { - local repo=$1 - - [ ${#PKGREPOS[@]} -eq 0 ] && return 1 - [ -z "${PKGPOOL}" ] && return 1 - - in_array "${repo}" "${PKGREPOS[@]}" || return 1 - - [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 - - local arch - for arch in "${ARCHES[@]}"; do - local dir="${FTP_BASE}/${repo}/os/${arch}/" - [ -w "${dir}" ] || return 1 - [ -f "${dir}${repo}"${DBEXT} -a ! -w "${dir}${repo}"${DBEXT} ] && return 1 - [ -f "${dir}${repo}"${FILESEXT} -a ! -w "${dir}${repo}"${FILESEXT} ] && return 1 - done - - return 0 -} - -set_repo_permission() { - local repo=$1 - local arch=$2 - local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" - local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" - - if [ -w "${dbfile}" ]; then - local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") - chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "${dbfile}" "$group" - chgrp "$group" "${filesfile}" || error "Could not change group of %s to %s" "${filesfile}" "$group" - chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "${dbfile}" - chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "${filesfile}" - else - error "You don't have permission to change %s" "${dbfile}" - fi -} - -arch_repo_add() { - local repo=$1 - local arch=$2 - local pkgs=("${@:3}") - - printf -v pkgs_str -- '%q ' "${pkgs[@]}" - # package files might be relative to repo dir - pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ - || error 'repo-add %q %s' "${repo}${DBEXT}" "${pkgs_str% }" - /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ - || error 'repo-add -f %q %s' "${repo}${FILESEXT}" "${pkgs_str% }" - popd >/dev/null - set_repo_permission "${repo}" "${arch}" - - REPO_MODIFIED=1 -} - -arch_repo_remove() { - local repo=$1 - local arch=$2 - local pkgs=("${@:3}") - local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" - local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" - - if [ ! -f "${dbfile}" ]; then - error "No database found at '%s'" "${dbfile}" - return 1 - fi - printf -v pkgs_str -- '%q ' "${pkgs[@]}" - /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ - || error 'repo-remove %q %s' "${dbfile}" "${pkgs_str% }" - /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ - || error 'repo-remove %q %s' "${filesfile}" "${pkgs_str% }" - set_repo_permission "${repo}" "${arch}" - - REPO_MODIFIED=1 -} diff --git a/db-libremessages b/db-libremessages deleted file mode 100755 index 37df149..0000000 --- a/db-libremessages +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2006-2010 Pacman Development Team -# Copyright (c) 2002-2006 by Judd Vinet -# Copyright (c) 2005 by Aurelien Foret -# Copyright (c) 2006 by Miklos Vajna -# Copyright (c) 2005 by Christian Hamar -# Copyright (c) 2006 by Alex Smith -# Copyright (c) 2006 by Andras Voroskoi -# Copyright (c) 2011 by Joshua Haase -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# gettext initialization -export TEXTDOMAIN='libretools' -export TEXTDOMAINDIR='/usr/share/locale' - -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW - -if tput setaf 0 &>/dev/null; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" - PURPLE="${ALL_OFF}$(tput setaf 5)" -else - ALL_OFF="\033[1;0m" - BOLD="\033[1;1m" - BLUE="${BOLD}\033[1;34m" - GREEN="${BOLD}\033[1;32m" - RED="${BOLD}\033[1;31m" - YELLOW="${BOLD}\033[1;33m" - PURPLE="${BOLD}\033[1;30;40m" -fi - -stdnull() { - local action=$1; - eval "${action} >/dev/null 2>&1" -} - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -fatal_error() { - local mesg=$1; shift - error "$mesg" "$@" - - exit 1 -} diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages deleted file mode 100755 index 095e1e6..0000000 --- a/db-list-unsigned-packages +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -set -e - -# Output a list of repo/package-name-and-version pairs representing -# unsigned packages available for architecture $1 and specified for -# architecture $2 (usually $1 or any, default is to list all). - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -lt 1 ]; then - msg "usage: %s " "${0##*/}" - exit 1 -fi - -arch=$1 -shift - -for repo in "${PKGREPOS[@]}" -do - db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" -done diff --git a/db-list-unsigned-packages.py b/db-list-unsigned-packages.py deleted file mode 100755 index 80cff51..0000000 --- a/db-list-unsigned-packages.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Output a list of repo/package-name-and-version pairs representing -unsigned packages in the database at standard input of repo named in -the first argument and specified for architectures listed in the -following arguments (usually the one of the database or any, default -is to list all). - -If the --keyset argument is passed, print the key fingerprint of every -signed package. -""" - - -import base64 -import subprocess -import sys -import tarfile - - -def main(): - """Do the job.""" - check_keys = False - if "--keyset" in sys.argv: - sys.argv.remove("--keyset") - check_keys = True - repo = sys.argv[1] - pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) - packages = [] - keys = [] - with tarfile.open(fileobj=sys.stdin.buffer) as archive: - for entry in archive: - if entry.name.endswith("/desc"): - content = archive.extractfile(entry) - skip = False - is_arch = False - key = None - for line in content: - if is_arch: - is_arch = False - if pkgarches and line.strip() not in pkgarches: - skip = True # different architecture - break - if line == b"%PGPSIG%\n": - skip = True # signed - key = b"" - if check_keys: - continue - else: - break - if line == b"%ARCH%\n": - is_arch = True - continue - if key is not None: - if line.strip(): - key += line.strip() - else: - break - if check_keys and key: - key_binary = base64.b64decode(key) - keys.append(key_binary) - packages.append(repo + "/" + entry.name[:-5]) - if skip: - continue - print(repo + "/" + entry.name[:-5]) - if check_keys and keys: - # We have collected all signed package names in packages and - # all keys in keys. Let's now ask gpg to list all signatures - # and find which keys made them. - packets = subprocess.check_output(("gpg", "--list-packets"), - input=b"".join(keys)) - i = 0 - for line in packets.decode("latin1").split("\n"): - if line.startswith(":signature packet:"): - keyid = line[line.index("keyid ") + len("keyid "):] - print(packages[i], keyid) - i += 1 - - -if __name__ == "__main__": - main() diff --git a/db-move b/db-move deleted file mode 100755 index 275a11a..0000000 --- a/db-move +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -lt 3 ]; then - msg "usage: %s ..." "${0##*/}" - exit 1 -fi - -args=("${@}") -repo_from="${args[0]}" -repo_to="${args[1]}" -ftppath_from="${FTP_BASE}/${repo_from}/os/" -ftppath_to="${FTP_BASE}/${repo_to}/os/" - -if ! check_repo_permission "$repo_to" || ! check_repo_permission "$repo_from"; then - die "You don't have permission to move packages from %s to %s" "${repo_from}" "${repo_to}" -fi - -# TODO: this might lock too much (architectures) -for pkgarch in "${ARCHES[@]}"; do - repo_lock "${repo_to}" "${pkgarch}" || exit 1 - repo_lock "${repo_from}" "${pkgarch}" || exit 1 -done - -# First loop is to check that all necessary files exist -for pkgbase in "${args[@]:2}"; do - for pkgarch in "${ARCHES[@]}" 'any'; do - xbsrepo_from="$(xbs releasepath "${pkgbase}" "${repo_from}" "${pkgarch}")" - if [ -r "${xbsrepo_from}/PKGBUILD" ]; then - pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) - if [ ${#pkgnames[@]} -lt 1 ]; then - die "Could not read pkgname" - fi - - pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") - if [ -z "${pkgver}" ]; then - die "Could not read pkgver" - fi - - if [ "${pkgarch}" == 'any' ]; then - tarches=("${ARCHES[@]}") - else - tarches=("${pkgarch}") - fi - - for pkgname in "${pkgnames[@]}"; do - for tarch in "${tarches[@]}"; do - getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} >/dev/null - done - done - continue 2 - fi - done - die "%s not found in %s" "${pkgbase}" "${repo_from}" -done - -msg "Moving packages from [%s] to [%s]..." "${repo_from}" "${repo_to}" - -declare -A add_pkgs -declare -A remove_pkgs -for pkgbase in "${args[@]:2}"; do - # move the package in xbs - arches=($(xbs move "${repo_from}" "${repo_to}" "${pkgbase}")) - # move the package in ftp - for pkgarch in "${arches[@]}"; do - xbsrepo_to="$(xbs releasepath "$pkgbase" "$repo_to" "$pkgarch")" - if true; then # to add an indent level to make merging easier - if [ "${pkgarch}" == 'any' ]; then - tarches=("${ARCHES[@]}") - else - tarches=("${pkgarch}") - fi - msg2 '%s (%s)' "${pkgbase}" "${tarches[*]}" - pkgnames=($(. "${xbsrepo_to}/PKGBUILD"; echo "${pkgname[@]}")) - pkgver=$(. "${xbsrepo_to}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") - - for pkgname in "${pkgnames[@]}"; do - for tarch in "${tarches[@]}"; do - pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}) - pkgfile="${pkgpath##*/}" - - ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" - if [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig" ]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" - fi - add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " - remove_pkgs[${tarch}]+="${pkgname} " - done - done - fi - done -done - -for tarch in "${ARCHES[@]}"; do - if [ -n "${add_pkgs[${tarch}]}" ]; then - arch_repo_add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} - arch_repo_remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} - fi -done - -for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${repo_from}" "${pkgarch}" - repo_unlock "${repo_to}" "${pkgarch}" -done diff --git a/db-remove b/db-remove deleted file mode 100755 index dcbe4b4..0000000 --- a/db-remove +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -lt 3 ]; then - msg "usage: %s ..." "${0##*/}" - exit 1 -fi - -repo="$1" -arch="$2" -pkgbases=("${@:3}") - -if ! check_repo_permission "$repo"; then - die "You don't have permission to remove packages from %s" "${repo}" -fi - -if [ "$arch" == "any" ]; then - tarches=("${ARCHES[@]}") -else - tarches=("$arch") -fi - -for tarch in "${tarches[@]}"; do - repo_lock "$repo" "$tarch" || exit 1 -done - -remove_pkgs=() -for pkgbase in "${pkgbases[@]}"; do - msg "Removing %s from [%s]..." "$pkgbase" "$repo" - - path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" - if [ -d "$path" ]; then - remove_pkgs+=($(. "$path/PKGBUILD"; echo "${pkgname[@]}")) - xbs unrelease "$pkgbase" "$repo" "$arch" - else - warning "%s not found in %s for %s" \ - "$pkgbase" "$(xbs name)" "$repo-$arch" - warning "Removing only %s from the repo" "$pkgbase" - warning "If it was a split package you have to remove the others yourself!" - remove_pkgs+=("$pkgbase") - fi -done - -for tarch in "${tarches[@]}"; do - arch_repo_remove "${repo}" "${tarch}" "${remove_pkgs[@]}" - repo_unlock "$repo" "$tarch" -done diff --git a/db-repo-add b/db-repo-add deleted file mode 100755 index 4611bdf..0000000 --- a/db-repo-add +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -lt 3 ]; then - msg "usage: %s ..." "${0##*/}" - exit 1 -fi - -repo="$1" -arch="$2" -pkgfiles=("${@:3}") - -ftppath="$FTP_BASE/$repo/os" - -if ! check_repo_permission "$repo"; then - die "You don't have permission to add packages to %s" "${repo}" -fi - -if [ "$arch" == "any" ]; then - tarches=("${ARCHES[@]}") -else - tarches=("$arch") -fi - -for tarch in "${tarches[@]}"; do - repo_lock "$repo" "$tarch" || exit 1 -done - -for tarch in "${tarches[@]}"; do - for pkgfile in "${pkgfiles[@]}"; do - if [[ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ]]; then - die "Package file %s not found in %s" "${pkgfile##*/}" "${FTP_BASE}/${repo}/os/${arch}/" - else - msg "Adding %s to [%s]..." "$pkgfile" "$repo" - fi - done - arch_repo_add "${repo}" "${tarch}" "${pkgfiles[@]}" - repo_unlock "$repo" "$tarch" -done diff --git a/db-repo-remove b/db-repo-remove deleted file mode 100755 index aadc4ce..0000000 --- a/db-repo-remove +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -lt 3 ]; then - msg "usage: %s ..." "${0##*/}" - exit 1 -fi - -repo="$1" -arch="$2" -pkgnames=("${@:3}") - -ftppath="$FTP_BASE/$repo/os" - -if ! check_repo_permission "$repo"; then - die "You don't have permission to remove packages from %s" "${repo}" -fi - -if [ "$arch" == "any" ]; then - tarches=("${ARCHES[@]}") -else - tarches=("$arch") -fi - -for tarch in "${tarches[@]}"; do - repo_lock "$repo" "$tarch" || exit 1 -done - -for tarch in "${tarches[@]}"; do - for pkgname in "${pkgnames[@]}"; do - msg "Removing %s from [%s]..." "$pkgname" "$repo" - done - arch_repo_remove "${repo}" "${tarch}" "${pkgnames[@]}" - repo_unlock "$repo" "$tarch" -done diff --git a/db-sync b/db-sync deleted file mode 100755 index 2194fe6..0000000 --- a/db-sync +++ /dev/null @@ -1,208 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics - -# Run as `V=true db-sync` to get verbose output -VERBOSE=${V} -extra=() -${VERBOSE} && extra+=(-v) - -WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT - -# Returns contents of a repo -get_repos() { - # Exclude everything but db files - rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" -} - -get_repo_content() { - # Return all contents - bsdtar tf "${1}" | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${WORKDIR}/${1}/os/${2}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "%d packages in blacklist" ${#blacklist[@]} - - test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in "${ARCHREPOS[@]}"; do - for _arch in "${ARCHARCHES[@]}"; do - msg "Processing %s-%s" "${_repo}-${_arch}" - - db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} - files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - # Get db contents - db=($(get_repo_content "${db_file}")) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - # IMPORTANT: the . in the sed command is needed because an empty - # whitelist would consist of a single * allowing any package to - # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" - - msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync "${extra[@]}" --no-motd -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from="/tmp/${_repo}-${_arch}.whitelist" \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - "${WORKDIR}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists, check for single *s just in case - cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist - - msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local pkgpool - for pkgpool in "${ARCHPKGPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ - "${FTP_BASE}/${pkgpool}/" - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local srcpool - for srcpool in "${ARCHSRCPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ - "${FTP_BASE}/${srcpool}/" - done - - date -u +%s > "${FTP_BASE}/lastsync" - - # Cleanup - unset blacklist whitelists _arch _repo repo_file -} - -trap_exit() { - local signal=$1; shift - echo - error "$@" - trap -- "$signal" - kill "-$signal" "$$" -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-sync.conf" -source "$(dirname "$(readlink -e "$0")")/db-libremessages" - -# Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do - test -z "${!var}" && fatal_error "Empty %s" "${var}" -done - -# From makepkg -set -E -for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" -done -trap 'trap_exit INT "Aborted by user! Exiting..."' INT -trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR - -init diff --git a/db-sync.conf b/db-sync.conf deleted file mode 100644 index f7748c3..0000000 --- a/db-sync.conf +++ /dev/null @@ -1,11 +0,0 @@ -#mirror="mirrors.kernel.org" -mirror="mirrors.niyawe.de" - -## mirrors without sources folder -#mirror="mirror.nl.leaseweb.net" -#mirror="mirror.one.com" -#mirror="mirror.us.leaseweb.net" -#mirror="mirror.bytemark.co.uk" -#mirror="mirror.de.leaseweb.net" - -mirrorpath="archlinux" diff --git a/db-update b/db-update deleted file mode 100755 index 4830791..0000000 --- a/db-update +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -ge 1 ]; then - warning "Calling %s with a specific repository is no longer supported" "${0##*/}" - exit 1 -fi - -# Find repos with packages to release -staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)) -if [ $? -ge 1 ]; then - die "Could not read %s" "${STAGING}" -fi - -repos=() -for staging_repo in "${staging_repos[@]##*/}"; do - if in_array "${staging_repo}" "${PKGREPOS[@]}"; then - repos+=("${staging_repo}") - fi -done - -# TODO: this might lock too much (architectures) -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${pkgarch}" || exit 1 - done -done - -# check if packages are valid -for repo in "${repos[@]}"; do - if ! check_repo_permission "${repo}"; then - die "You don't have permission to update packages in %s" "${repo}" - fi - pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) - if [ $? -eq 0 ]; then - for pkg in "${pkgs[@]}"; do - if [ -h "${pkg}" ]; then - die "Package %s is a symbolic link" "${repo}/${pkg##*/}" - fi - if ! check_pkgfile "${pkg}"; then - die "Package %s is not consistent with its meta data" "${repo}/${pkg##*/}" - fi - if ! check_pkgrepos "${pkg}"; then - die "Package %s already exists in another repository" "${repo}/${pkg##*/}" - fi - done - # This is fucking obnoxious - #if ! check_splitpkgs ${repo} "${pkgs[@]}"; then - # die "Missing split packages for %s" "${repo}" - #fi - else - die "Could not read %s" "${STAGING}" - fi -done - -dirs=() -for repo in "${repos[@]}"; do - msg "Updating [%s]..." "${repo}" - any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) - for pkgarch in "${ARCHES[@]}"; do - add_dirs=() - add_pkgs=() - arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-"${pkgarch}"${PKGEXT} 2>/dev/null)) - for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do - pkgfile="${pkg##*/}" - msg2 "%s (%s)" "${pkgfile}" "${pkgarch}" - # any packages might have been moved by the previous run - if [ -f "${pkg}" ]; then - mv "${pkg}" "$FTP_BASE/${PKGPOOL}" - fi - ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" - # also move signatures - if [ -f "${pkg}.sig" ]; then - mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" - fi - if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" - fi - add_dirs+=("${STAGING}/abslibre/$(getpkgarch "$FTP_BASE/$PKGPOOL/$pkgfile")/$repo/$(getpkgbase "$FTP_BASE/$PKGPOOL/$pkgfile")") - add_pkgs+=("${pkgfile}") - done - for add_dir in "${add_dirs[@]}"; do - (cd "${add_dir}" && xbs release-server "${repo}" "${pkgarch}") || - error 'cd %q && xbs release-server %q %q' "${add_dir}" "${repo}" "${pkgarch}" - done - if [ ${#add_pkgs[@]} -ge 1 ]; then - arch_repo_add "${repo}" "${pkgarch}" "${add_pkgs[@]}" - fi - dirs+=("${add_dirs[@]}") - done -done - -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${pkgarch}" - done -done - -cd "${STAGING}" - -# Remove left over XBS files -rm -rf -- "${dirs[@]}" -dirname -z -- "${dirs[@]}" | - xargs -0 realpath -zm --relative-to="${STAGING}" -- | - xargs -0 rmdir -p -- 2>/dev/null - -# Stage generated source files -while read -r file; do - pub="${FTP_BASE}/${file}" - if [[ -f "$pub" ]]; then - warning "file already exists: %s" "${file}" - else - mkdir -p -- "${pub%/*}" - mv -vn "$file" "$pub" - fi -done < <(find other sources -type f 2>/dev/null) diff --git a/etc b/etc new file mode 120000 index 0000000..c4675b8 --- /dev/null +++ b/etc @@ -0,0 +1 @@ +src/etc \ No newline at end of file diff --git a/extra/coadde/TODO b/extra/coadde/TODO new file mode 100644 index 0000000..9dd4b52 --- /dev/null +++ b/extra/coadde/TODO @@ -0,0 +1,10 @@ +* Test Suite for clean_repo.py + + - Review all repo + - Remove all blacklisted packages + - Get pending list right + - Extract licenses all right + +* Fix db-move + + - Make it use abslibre diff --git a/extra/coadde/any-to-ours b/extra/coadde/any-to-ours new file mode 100755 index 0000000..7ca2878 --- /dev/null +++ b/extra/coadde/any-to-ours @@ -0,0 +1,123 @@ +#!/bin/bash +# Releases 'any' packages from Arch-derivations arches to ours + +trap_exit() { + echo + error "$@" + exit 1 +} + +source "$(dirname "$(readlink -e "$0")")/config_platforms" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +for 'platform' in "${DERIVATIVE_PLATFORMS[@]}"; do + + source "$(dirname "$(readlink -e "$0")")/config_${platform}" + + # Traverse all Arch repos + for '_repo' in "${MAIN_REPOS[@]}"; do + msg "Processing %s..." "${_repo}" + + # Find 'any' packages + # This is hardcoded but it could release other arches... + PKGS=($(find "${REPO_DIR}/${_repo}/os/${ARCHES[0]}/" \ + -iname '*-any.pkg.tar.?z' \ + -printf "%f ")) + + if [ ${#PKGS[@]} -eq 0 ]; then + msg2 "No '%s' packages here" any + continue + fi + + for '_arch' in "${ARCHES[@]}"; do + msg2 "Syncing %s..." "${_arch}" + + # Sync 'any' only and extract the synced packages + SYNCED=($( + rsync -av \ + --include='*-any.pkg.tar.?z' \ + --include='*-any.pkg.tar.?z.sig' \ + --exclude='*' \ + "${REPO_DIR}/${_repo}/os/${ARCHES[0]}/" \ + "${REPO_DIR}/${_repo}/os/${_arch}/" 2>&1 | \ + grep 'any\.pkg\.tar\..z$' | \ + cut -d ' ' -f 1 )) + + if [ "${#SYNCED[@]}" -eq 0 ]; then + msg2 "Already synced (or error happened)" + continue + fi + + msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" + + msg2 "Adding to db..." + + pushd "${REPO_DIR}/${_repo}/os/${_arch}/" >/dev/null + + # Add the packages to the db + repo-add "${_repo}${DB_EXT}" "${SYNCED[@]}" + + popd >/dev/null + + # Avoid mixups + unset 'SYNCED' 'PKGS' + done + if [ -n "${MULTILIB_ARCHES}" ]; then + for '_repo_multilib' in "${MULTILIB_REPOS[@]}"; do + msg "Processing %s..." "${_repo}" '(multilib)' + + # Find 'any' packages + # This is hardcoded but it could release other arches... + PKGS_MULTILIB=($(find "${REPO_DIR}/${_repo_multilib}/os/${MULTILIB_ARCHES[0]}/" \ + -iname '*-any.pkg.tar.?z' \ + -printf "%f ")) + + if [ ${#PKGS_MULTILIB[@]} -eq 0 ]; then + msg2 "No '%s' multilib packages here" any + continue + fi + + for '_arch_multilib' in "${MULTILIB_ARCHES[@]}"; do + msg2 "Syncing %s..." "${_arch_multilib}" '(multilib)' + + # Sync 'any' only and extract the synced packages + SYNCED_MULTILIB=($( + rsync -av \ + --include='*-any.pkg.tar.?z' \ + --include='*-any.pkg.tar.?z.sig' \ + --exclude='*' \ + "${REPO_DIR}/${_repo_multilib}/os/${MULTILIB_ARCHES[0]}/" \ + "${REPO_DIR}/${_repo_multilib}/os/${_arch_multilib}/" 2>&1 | \ + grep 'any\.pkg\.tar\..z$' | \ + cut -d ' ' -f 1 )) + + if [ "${#SYNCED_MULTILIB[@]}" -eq 0 ]; then + msg2 "Already synced (or error happened)" + continue + fi + + msg2 "Synced %d packages: %s" "${#SYNCED_MULTILIB[@]}" "${SYNCED_MULTILIB[*]}" + + msg2 "Adding Multilib to db..." + + pushd "${REPO_DIR}/${_repo_multilib}/os/${_arch_multilib}/" >/dev/null + + # Add the packages to the db + repo-add "${_repo_multilib}${DB_EXT}" "${SYNCED_MULTILIB[@]}" + + popd >/dev/null + + # Avoid mixups + unset 'SYNCED_MULTILIB' 'PKGS_MULTILIB' + done + done + fi + done +done diff --git a/extra/coadde/create-repo b/extra/coadde/create-repo new file mode 100755 index 0000000..c977442 --- /dev/null +++ b/extra/coadde/create-repo @@ -0,0 +1,61 @@ +#!/bin/bash +# Creates repository structure + +source "$(dirname "$(readlink -e "$0")")/config_platforms" +source "$(dirname "$(readlink -e "$0")")/db-functions" + +create-repo-usage() { + msg "Usage: %s [platform] repo1 [repo2 ... repoX]" "${0##*/}" + exit 1 +} + +if [ "$#" -eq '0' ]; then + create-repo-usage +fi + +for 'platform' in "${PLATFORMS[@]}"; do + case "$1" in + --platform|-platform|-p) + + # Don't create [--platform], [-platform] or [-p] and [] repos + _repos=($@) && unset _repos[0] _repos[1] + + # Rename plataform name ($2) to easily script usage + _platform_name="${2/\//+}" _platform_name="${_platform_name,,}" + + if [ "${_platform_name}" == "${platform}" ]; then + source "$(dirname "$(readlink -e "$0")")/config_${2}" + + msg "Creating repos... from ${PLATFORM_NAME} platform" + for '_repo' in "${_repos[@]}"; do + msg2 "Creating [%s]" "${_repo_all}" + for '_arch' in "${ARCHES[@]}"; do + mkdir -p "${REPO_DIR}/${_repo}/os/${_arch}" || \ + error "Failed creating %s dir" "${_arch}" "from ${PLATFORM_NAME} platform" + done + done + + msg "Don't forget to add them to the PKG_REPOS array on %s" "$(dirname "$(readlink -e "$0")")/config_${2}" + else + create-repo-usage + fi + ;; + *) + source "$(dirname "$(readlink -e "$0")")/config_${platform}" + + msg "Creating repos... from all platforms" + for '_repo_all' in "${@}"; do + msg2 "Creating [%s]" "${_repo_all}" + for '_arch' in "${ARCHES[@]}"; do + mkdir -p "${REPO_DIR}/${_repo_all}/os/${_arch}" || \ + error "Failed creating %s dir" "${_arch}" "in all platforms" + done + done + + msg "Don't forget to add them to the PKG_REPOS array on %s" "$(dirname "$(readlink -e "$0")")/config_${platform}" + ;; + esac +done + +# if [ -n "${MULTILIB_ARCHES}" ]; then +# fi diff --git a/extra/coadde/createrepos b/extra/coadde/createrepos new file mode 100755 index 0000000..4416196 --- /dev/null +++ b/extra/coadde/createrepos @@ -0,0 +1,36 @@ +#!/bin/bash +# Creates the repo structure defined in config + +source "$(dirname "$(readlink -e "$0")")/config_platforms" +source "$(dirname "$(readlink -e "$0")")/db-functions" + +createrepos-usage() { + msg "Usage: %s [platform1 platform2 ... platformX]" "${0##*/}" + exit 1 +} + +for 'platform' in "${PLATFORMS[@]}"; do + if [ -n $1 ]; then + for '_platform' in "$@"; do + # Rename plataform name ($_platform) to easily script usage + _platform_name="${_platform/\//+}" _platform_name="${_platform_name,,}" + + if [ "${_platform_name}" == "${platform}" ]; then + source "$(dirname "$(readlink -e "$0")")/config_${_platform}" + + mkdir -p -- "${REPO_DIR}"/{"${PKG_POOL}","${SRC_POOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" + + "$(dirname "$(readlink -e "$0")")/create-repo" -p "${_platform}" "${PKG_REPOS[@]}" + else + createrepos-usage + fi + done + else + source "$(dirname "$(readlink -e "$0")")/config_${platform}" + + mkdir -p -- "${REPO_DIR}"/{"${PKG_POOL}","${SRC_POOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" + + "$(dirname "$(readlink -e "$0")")/create-repo" "${PKG_REPOS[@]}" + ;; + fi +done diff --git a/extra/coadde/cron-jobs/check_archlinux/check_packages.py b/extra/coadde/cron-jobs/check_archlinux/check_packages.py new file mode 100755 index 0000000..ac0194f --- /dev/null +++ b/extra/coadde/cron-jobs/check_archlinux/check_packages.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python2 +# +# check_archlinux.py +# +# Original script by Scott Horowitz +# Rewritten by Xavier Chantry +# +# This script currently checks for a number of issues in your ABS tree: +# 1. Directories with missing PKGBUILDS +# 2. Invalid PKGBUILDs (bash syntax error for instance) +# 3. PKGBUILD names that don't match their directory +# 4. Duplicate PKGBUILDs +# 5. Valid arch's in PKGBUILDS +# 6. Missing (make-)dependencies +# 7. Hierarchy of repos (e.g., that a core package doesn't depend on +# a non-core package) +# 8. Circular dependencies + +import os,re,commands,getopt,sys,tarfile +import pdb + +import ctypes +_alpm = ctypes.cdll.LoadLibrary("libalpm.so") + +DBEXT='.db.tar.gz' + +packages = {} # pkgname : PacmanPackage +repopkgs = {} # pkgname : PacmanPackage +provisions = {} # provision : PacmanPackage +pkgdeps,makepkgdeps = {},{} # PacmanPackage : list of the PacmanPackage dependencies +invalid_pkgbuilds = [] +missing_pkgbuilds = [] +dups = [] + +dbonly = [] +absonly = [] + +mismatches = [] +missing_deps = [] +missing_makedeps = [] +invalid_archs = [] +dep_hierarchy = [] +makedep_hierarchy = [] +circular_deps = [] # pkgname>dep1>dep2>...>pkgname +checked_deps = [] + +class PacmanPackage: + def __init__(self): + self.name,self.version = "","" + self.base = "" + self.path,self.repo = "","" + self.deps,self.makedeps = [],[] + self.provides,self.conflicts = [],[] + self.archs = [] + +class Depend: + def __init__(self,name,version,mod): + self.name = name + self.version = version + self.mod = mod + +def parse_pkgbuilds(repos,arch): + for absroot in absroots: + for repo in repos: + cmd = os.path.dirname(os.path.realpath(sys.argv[0])) + '/parse_pkgbuilds.sh ' + cmd += arch + ' ' + absroot + '/' + repo + (status,output) = commands.getstatusoutput(cmd) + if status != 0: + print "Error : failed to run '%s'" % cmd + sys.exit() + parse_data(repo,output) + +def parse_data(repo,data): + attrname = None + + for line in data.split('\n'): + if line.startswith('%'): + attrname = line.strip('%').lower() + elif line.strip() == '': + attrname = None + elif attrname == "invalid": + if repo in repos: + invalid_pkgbuilds.append(line) + elif attrname == "missing": + if repo in repos: + missing_pkgbuilds.append(line) + elif attrname == "name": + pkg = PacmanPackage() + pkg.name = line + pkg.repo = repo + dup = None + if pkg.name in packages: + dup = packages[pkg.name] + else: + packages[pkg.name] = pkg + elif attrname == "base": + pkg.base = line + elif attrname == "version": + pkg.version = line + elif attrname == "path": + pkg.path = line + if dup != None and (pkg.repo in repos or dup.repo in repos): + dups.append(pkg.path + " vs. " + dup.path) + elif attrname == "arch": + pkg.archs.append(line) + elif attrname == "depends": + pkg.deps.append(line) + elif attrname == "makedepends": + pkg.makedeps.append(line) + elif attrname == "conflicts": + pkg.conflicts.append(line) + elif attrname == "provides": + pkg.provides.append(line) + +def parse_dbs(repos,arch): + dbpkgs = {} + for repo in repos: + pkgs = set([]) + db = tarfile.open(os.path.join(repodir,repo,'os',arch,repo + DBEXT)) + for line in db.getnames(): + if not '/' in line: + pkgs.add(line.rsplit('-',2)[0]) + dbpkgs[repo] = pkgs + return(dbpkgs) + +def splitdep(dep): + name = dep + version = "" + mod = "" + for char in (">=", "<=", "=", ">", "<"): + pos = dep.find(char) + if pos > -1: + name = dep[:pos] + version = dep[pos:].replace(char, "") + mod = char + break + return Depend(name,version,mod) + +def splitprov(prov): + name = prov + version = "" + pos = prov.find("=") + if pos > -1: + name = prov[:pos] + version = prov[pos:].replace("=", "") + return (name,version) + +def vercmp(v1,mod,v2): + """ + >>> vercmp("1.0", "<=", "2.0") + True + >>> vercmp("1:1.0", ">", "2.0") + True + >>> vercmp("1.0.2", ">=", "2.1.0") + False + """ + s1 = ctypes.c_char_p(v1) + s2 = ctypes.c_char_p(v2) + res = _alpm.alpm_pkg_vercmp(s1,s2) + if res == 0: + return (mod.find("=") > -1) + elif res < 0: + return (mod.find("<") > -1) + elif res > 0: + return (mod.find(">") > -1) + return False + + +def depcmp(name,version,dep): + if name != dep.name: + return False + if dep.version == "" or dep.mod == "": + return True + if version == "": + return False + return vercmp(version,dep.mod,dep.version) + +def provcmp(pkg,dep): + for prov in pkg.provides: + (provname,provver) = splitprov(prov) + if depcmp(provname,provver,dep): + return True + return False + +def verify_dep(dep): + dep = splitdep(dep) + if dep.name in packages: + pkg = packages[dep.name] + if depcmp(pkg.name,pkg.version,dep): + return [pkg] + if dep.name in provisions: + provlist = provisions[dep.name] + results = [] + for prov in provlist: + if provcmp(prov,dep): + results.append(prov) + return results + return [] + +def verify_deps(name,repo,deps): + pkg_deps = [] + missdeps = [] + hierarchy = [] + for dep in deps: + pkglist = verify_dep(dep) + if pkglist == []: + missdeps.append(repo + "/" + name + " --> '" + dep + "'") + else: + valid_repos = get_repo_hierarchy(repo) + pkgdep = None + for pkg in pkglist: + if pkg.repo in valid_repos: + pkgdep = pkg + break + if not pkgdep: + pkgdep = pkglist[0] + hierarchy.append((repo,name,pkgdep)) + + pkg_deps.append(pkgdep) + + return (pkg_deps,missdeps,hierarchy) + +def compute_deplist(pkg): + list = [] + stack = [pkg] + while stack != []: + dep = stack.pop() + if dep in pkgdeps: + for dep2 in pkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + if dep in makepkgdeps: + for dep2 in makepkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + return list + +def check_hierarchy(deph): + hierarchy = [] + for (repo,name,pkgdep) in deph: + deplist = compute_deplist(pkgdep) + valid_repos = get_repo_hierarchy(repo) + extdeps = [] + for dep in deplist: + if dep.repo not in valid_repos: + extdeps.append(dep.name) + string = repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name + " (" + string += "%s extra (make)deps to pull" % len(extdeps) + if 0 < len(extdeps) < 10: + string += " : " + ' '.join(extdeps) + string += ")" + hierarchy.append(string) + return hierarchy + +def get_repo_hierarchy(repo): + repo_hierarchy = {'core': ['core'], \ + 'extra': ['core', 'extra'], \ + 'community': ['core', 'extra', 'community'], \ + 'multilib': ['core', 'extra', 'community', 'multilib'] } + if repo in repo_hierarchy: + return repo_hierarchy[repo] + else: + return ['core','extra','community'] + +def verify_archs(name,repo,archs): + valid_archs = ['any', 'i686', 'x86_64'] + invalid_archs = [] + for arch in archs: + if arch not in valid_archs: + invalid_archs.append(repo + "/" + name + " --> " + arch) + return invalid_archs + +def find_scc(packages): + # reset all variables + global index,S,pkgindex,pkglowlink + index = 0 + S = [] + pkgindex = {} + pkglowlink = {} + cycles = [] + for pkg in packages: + tarjan(pkg) + +def tarjan(pkg): + global index,S,pkgindex,pkglowlink,cycles + pkgindex[pkg] = index + pkglowlink[pkg] = index + index += 1 + checked_deps.append(pkg) + S.append(pkg) + deps = [] + if pkg in pkgdeps: + deps = pkgdeps[pkg] + for dep in deps: + if dep not in pkgindex: + tarjan(dep) + pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep]) + elif dep in S: + pkglowlink[pkg] = min(pkglowlink[pkg],pkgindex[dep]) + if pkglowlink[pkg] == pkgindex[pkg]: + dep = S.pop() + if pkg == dep: + return + path = pkg.name + while pkg != dep: + path = dep.repo + "/" + dep.name + ">" + path + dep = S.pop() + path = dep.name + ">" + path + if pkg.repo in repos: + circular_deps.append(path) + +def print_heading(heading): + print "" + print "=" * (len(heading) + 4) + print "= " + heading + " =" + print "=" * (len(heading) + 4) + +def print_subheading(subheading): + print "" + print subheading + print "-" * (len(subheading) + 2) + +def print_missdeps(pkgname,missdeps) : + for d in missdeps: + print pkgname + " : " + d + +def print_result(list, subheading): + if len(list) > 0: + list.sort() + print_subheading(subheading) + for item in list: + print item + +def print_results(): + print_result(missing_pkgbuilds, "Missing PKGBUILDs") + print_result(invalid_pkgbuilds, "Invalid PKGBUILDs") + print_result(mismatches, "Mismatched Pkgnames") + print_result(dups, "Duplicate PKGBUILDs") + print_result(invalid_archs, "Invalid Archs") + print_result(missing_deps, "Missing Dependencies") + print_result(missing_makedeps, "Missing Makedepends") + print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") + print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") + print_result(circular_deps, "Circular Dependencies") + print_result(dbonly, "Packages found in db, but not in tree") + print_result(absonly,"Packages found in tree, but not in db") + print_subheading("Summary") + print "Missing PKGBUILDs: ", len(missing_pkgbuilds) + print "Invalid PKGBUILDs: ", len(invalid_pkgbuilds) + print "Mismatching PKGBUILD names: ", len(mismatches) + print "Duplicate PKGBUILDs: ", len(dups) + print "Invalid archs: ", len(invalid_archs) + print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) + print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) + print "Circular dependencies: ", len(circular_deps) + print "In db, but not in tree: ", len(dbonly) + print "In tree, but not in db: ", len(absonly) + print "" + +def print_usage(): + print "" + print "Usage: ./check_packages.py [OPTION]" + print "" + print "Options:" + print " --abs-tree= Check the specified tree(s) (default : /var/abs)" + print " --repos= Check the specified repos (default : core,extra)" + print " --arch= Check the specified arch (default : i686)" + print " --repo-dir= Check the dbs at the specified path (default : /srv/ftp)" + print " -h, --help Show this help and exit" + print "" + print "Examples:" + print "\n Check core and extra in existing abs tree:" + print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686" + print "\n Check community:" + print " ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686" + print "" + +if __name__ == "__main__": + ## Default path to the abs root directory + absroots = ["/var/abs"] + ## Default list of repos to check + repos = ['core', 'extra'] + ## Default arch + arch = "i686" + ## Default repodir + repodir = "/srv/ftp" + + try: + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", + "arch=", "repo-dir="]) + except getopt.GetoptError: + print_usage() + sys.exit() + if opts != []: + for o, a in opts: + if o in ("--abs-tree"): + absroots = a.split(',') + elif o in ("--repos"): + repos = a.split(",") + elif o in ("--arch"): + arch = a + elif o in ("--repo-dir"): + repodir = a + else: + print_usage() + sys.exit() + if args != []: + print_usage() + sys.exit() + + for absroot in absroots: + if not os.path.isdir(absroot): + print "Error : the abs tree " + absroot + " does not exist" + sys.exit() + for repo in repos: + repopath = absroot + "/" + repo + if not os.path.isdir(repopath): + print("Warning : the repository " + repo + " does not exist in " + absroot) + + if not os.path.isdir(repodir): + print "Error: the repository directory %s does not exist" % repodir + sys.exit() + for repo in repos: + path = os.path.join(repodir,repo,'os',arch,repo + DBEXT) + if not os.path.isfile(path): + print "Error : repo DB %s : File not found" % path + sys.exit() + if not tarfile.is_tarfile(path): + print "Error : Cant open repo DB %s, not a valid tar file" % path + sys.exit() + # repos which need to be loaded + loadrepos = set([]) + for repo in repos: + loadrepos = loadrepos | set(get_repo_hierarchy(repo)) + + print_heading("Integrity Check " + arch + " of " + ",".join(repos)) + print("\nPerforming integrity checks...") + + print("==> parsing pkgbuilds") + parse_pkgbuilds(loadrepos,arch) + + # fill provisions + for name,pkg in packages.iteritems(): + for prov in pkg.provides: + provname=prov.split("=")[0] + if provname not in provisions: + provisions[provname] = [] + provisions[provname].append(pkg) + + # fill repopkgs + for name,pkg in packages.iteritems(): + if pkg.repo in repos: + repopkgs[name] = pkg + + print("==> parsing db files") + dbpkgs = parse_dbs(repos,arch) + + print("==> checking mismatches") + for name,pkg in repopkgs.iteritems(): + pkgdirname = pkg.path.split("/")[-1] + if name != pkgdirname and pkg.base != pkgdirname: + mismatches.append(name + " vs. " + pkg.path) + + print("==> checking archs") + for name,pkg in repopkgs.iteritems(): + archs = verify_archs(name,pkg.repo,pkg.archs) + invalid_archs.extend(archs) + + deph,makedeph = [],[] + + print("==> checking dependencies") + for name,pkg in repopkgs.iteritems(): + (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps + missing_deps.extend(missdeps) + deph.extend(hierarchy) + + print("==> checking makedepends") + for name,pkg in repopkgs.iteritems(): + (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) + makepkgdeps[pkg] = makedeps + missing_makedeps.extend(missdeps) + makedeph.extend(hierarchy) + + print("==> checking hierarchy") + dep_hierarchy = check_hierarchy(deph) + makedep_hierarchy = check_hierarchy(makedeph) + + print("==> checking for circular dependencies") + # make sure pkgdeps is filled for every package + for name,pkg in packages.iteritems(): + if pkg not in pkgdeps: + (deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps + find_scc(repopkgs.values()) + + print("==> checking for differences between db files and pkgbuilds") + for repo in repos: + for pkg in dbpkgs[repo]: + if not (pkg in repopkgs and repopkgs[pkg].repo == repo): + dbonly.append("%s/%s" % (repo,pkg)) + for name,pkg in repopkgs.iteritems(): + if not name in dbpkgs[pkg.repo]: + absonly.append("%s/%s" % (pkg.repo,name)) + + print_results() diff --git a/extra/coadde/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/extra/coadde/cron-jobs/check_archlinux/parse_pkgbuilds.sh new file mode 100755 index 0000000..b857ac8 --- /dev/null +++ b/extra/coadde/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# Usage : parse_pkgbuilds.sh arch +# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra + +exit() { return; } + +splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') +variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' "${splitpkg_overrides[@]}") +readonly -a variables splitpkg_overrides + +backup_package_variables() { + for var in "${splitpkg_overrides[@]}"; do + indirect="${var}_backup" + eval "${indirect}=(\"\${$var[@]}\")" + done +} + +restore_package_variables() { + for var in "${splitpkg_overrides[@]}"; do + indirect="${var}_backup" + if [ -n "${!indirect}" ]; then + eval "${var}=(\"\${$indirect[@]}\")" + else + unset "${var}" + fi + done +} + +print_info() { + echo -e "%NAME%\n$pkgname\n" + if [ -n "$epoch" ]; then + echo -e "%VERSION%\n$epoch:$pkgver-$pkgrel\n" + else + echo -e "%VERSION%\n$pkgver-$pkgrel\n" + fi + echo -e "%PATH%\n$dir\n" + + if [ -n "$pkgbase" ]; then + echo -e "%BASE%\n$pkgbase\n" + fi + + if [ -n "$arch" ]; then + echo "%ARCH%" + for i in "${arch[@]}"; do echo "$i"; done + echo "" + fi + if [ -n "$depends" ]; then + echo "%DEPENDS%" + for i in "${depends[@]}"; do + echo "$i" + done + echo "" + fi + if [ -n "$makedepends" ]; then + echo "%MAKEDEPENDS%" + for i in "${makedepends[@]}"; do + echo "$i" + done + echo "" + fi + if [ -n "$conflicts" ]; then + echo "%CONFLICTS%" + for i in "${conflicts[@]}"; do echo "$i"; done + echo "" + fi + if [ -n "$provides" ]; then + echo "%PROVIDES%" + for i in "${provides[@]}"; do echo "$i"; done + echo "" + fi +} + +source_pkgbuild() { + ret=0 + dir=$1 + pkgbuild=$dir/PKGBUILD + for var in "${variables[@]}"; do + unset "${var}" + done + source "$pkgbuild" &>/dev/null || ret=$? + + # ensure $pkgname and $pkgver variables were found + if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + fi + + if [ "${#pkgname[@]}" -gt "1" ]; then + pkgbase=${pkgbase:-${pkgname[0]}} + for pkg in "${pkgname[@]}"; do + if [ "$(type -t "package_${pkg}")" != "function" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + else + backup_package_variables + pkgname=$pkg + while IFS= read -r line; do + var=${line%%=*} + var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters + for realvar in "${variables[@]}"; do + if [ "$var" == "$realvar" ]; then + eval $line + break + fi + done + done < <(type "package_${pkg}") + print_info + restore_package_variables + fi + done + else + echo + print_info + fi + + return 0 +} + +find_pkgbuilds() { + #Skip over some dirs + local d="${1##*/}" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi + + if [ -f "$1/PKGBUILD" ]; then + source_pkgbuild "$1" + return + fi + empty=1 + for dir in "$1"/*; do + if [ -d "$dir" ]; then + find_pkgbuilds "$dir" + unset empty + fi + done + if [ -n "$empty" ]; then + echo -e "%MISSING%\n$1\n" + fi +} + +if [ -z "$1" -o -z "$2" ]; then + exit 1 +fi + +CARCH=$1 +shift +for dir in "$@"; do + find_pkgbuilds "$dir" +done + +exit 0 diff --git a/extra/coadde/cron-jobs/devlist-mailer b/extra/coadde/cron-jobs/devlist-mailer new file mode 100755 index 0000000..1a05521 --- /dev/null +++ b/extra/coadde/cron-jobs/devlist-mailer @@ -0,0 +1,28 @@ +#!/bin/bash +#Dummy helper to send email to arch-dev +# It does nothing if no output + +LIST="arch-dev-public@archlinux.org" +#LIST="aaronmgriffin@gmail.com" +FROM="repomaint@archlinux.org" + +SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" +if [ $# -ge 1 ]; then + SUBJECT="$1 $(date +"%d-%m-%Y")" +fi + +if [ $# -ge 2 ]; then + LIST="$2" +fi + +stdin="$(cat)" +#echo used to strip whitespace for checking for actual data +if [ -n "$(echo $stdin)" ]; then + + echo "Subject: $SUBJECT +To: $LIST +From: $FROM + +$stdin" | /usr/sbin/sendmail -F"$FROM" "$LIST" + +fi diff --git a/extra/coadde/cron-jobs/ftpdir-cleanup b/extra/coadde/cron-jobs/ftpdir-cleanup new file mode 100755 index 0000000..4a2b418 --- /dev/null +++ b/extra/coadde/cron-jobs/ftpdir-cleanup @@ -0,0 +1,88 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" + +clean_pkg() { + local pkg + local target + + if ! "${CLEANUP_DRYRUN}"; then + for pkg in "$@"; do + if [ -h "$pkg" ]; then + rm -f "$pkg" "$pkg.sig" + else + mv_acl "$pkg" "$CLEANUP_DESTDIR/${pkg##*/}" + if [ -e "$pkg.sig" ]; then + mv_acl "$pkg.sig" "$CLEANUP_DESTDIR/${pkg##*/}.sig" + fi + touch "${CLEANUP_DESTDIR}/${pkg##*/}" + fi + done + fi +} + +"${CLEANUP_DRYRUN}" && warning 'dry run mode is active' + +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + continue + fi + # get a list of actual available package files + find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}" + # get a list of package files defined in the repo db + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" + + missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#missing_pkgs[@]} -ge 1 ]; then + error "Missing packages in [%s] (%s)..." "${repo}" "${arch}" + for missing_pkg in "${missing_pkgs[@]}"; do + msg2 '%s' "${missing_pkg}" + done + fi + + old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from [%s] (%s)..." "${repo}" "${arch}" + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" + done + fi + done +done + +# get a list of all available packages in the pacakge pool +find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" +# create a list of packages in our db +cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" + +old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from package pool..." + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" + done +fi + +old_pkgs=($(find "${CLEANUP_DESTDIR}" -type f -name "*${PKGEXT}" -mtime +"${CLEANUP_KEEP}" -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from the cleanup directory..." + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + if ! "${CLEANUP_DRYRUN}"; then + rm -f "${CLEANUP_DESTDIR}/${old_pkg}" + rm -f "${CLEANUP_DESTDIR}/${old_pkg}.sig" + fi + done +fi + +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${arch}" + done +done + +script_unlock diff --git a/extra/coadde/cron-jobs/integrity-check b/extra/coadde/cron-jobs/integrity-check new file mode 100755 index 0000000..7459380 --- /dev/null +++ b/extra/coadde/cron-jobs/integrity-check @@ -0,0 +1,32 @@ +#!/bin/bash + +dirname="$(dirname "$(readlink -e "$0")")" + +. "${dirname}/../config" +. "${dirname}/../db-functions" + +script_lock + +if [ $# -ne 1 ]; then + die "usage: %s " "${0##*/}" +fi +mailto=$1 + +check() { + "${dirname}"/check_archlinux/check_packages.py \ + --repos="${repos}" \ + --abs-tree="/srv/abs/rsync/${arch},/srv/abs/rsync/any" \ + --repo-dir="${FTP_BASE}" \ + --arch="${arch}" \ + 2>&1 | "${dirname}"/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" +} + +repos='core,extra,community' +arch='i686' +check + +repos='core,extra,community,multilib' +arch='x86_64' +check + +script_unlock diff --git a/extra/coadde/cron-jobs/make_repo_torrents b/extra/coadde/cron-jobs/make_repo_torrents new file mode 100755 index 0000000..2eb0978 --- /dev/null +++ b/extra/coadde/cron-jobs/make_repo_torrents @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright (C) 2014 Joseph Graham +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script finds any updated packages and calls +# `make_indivudual_torrent' for each of them. + +username=$( id -un ) + +case "${username}" in + repo | root ) + true + ;; + * ) + echo "This script must be run as repo user or root user." + echo "ByeBye!" + exit 1 + ;; +esac + +# pacman doesn't support multiple different packages of the same name, +# so it's OK to just stuff all the torrents into a single directory. +script_directory="$(dirname "$(readlink -e "$0")")/.." +. "$(dirname "$(readlink -e "$0")")/../config" +public_location="$FTP_BASE/" +torrent_location="$FTP_BASE/torrents/" + +cd "${torrent_location}" + +# Find any directories that might have packages in then +find "${public_location}" -name 'os' -type 'd' | +while read dir +do + # Find any packages + find "${dir}" -name '*\.pkg\.tar\.xz' | + while read pkg + do + pkg_name="${pkg##*/}" + + if [[ -h "${pkg}" ]] # check if it's a symbolic link + then + # We get the target of the symlink + pkg=$( readlink -f "${pkg}" ) + fi + + # If a .torrent file does not already exist for this package, we call + # `make_individual_torrent' to make it. + if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] + then + "$script_directory/make_individual_torrent" "${pkg}" "${public_location}" + fi + done +done + +if [[ "${username}" == root ]] +then + chown repo * +fi diff --git a/extra/coadde/cron-jobs/repo-sanity-check b/extra/coadde/cron-jobs/repo-sanity-check new file mode 100755 index 0000000..239f042 --- /dev/null +++ b/extra/coadde/cron-jobs/repo-sanity-check @@ -0,0 +1,57 @@ +#!/bin/bash +# Solves issue165... on the old roundup install. From the database +# backups, the title was "Older/deprecated packages never leave the +# repo", I don't know how the body of the issue is stored in the DB, +# but the title says enough, I think. + +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" + +# Traverse all repos +for _repo in "${PKGREPOS[@]}"; do + msg "Cleaning up [%s]" "${_repo}" + + # Find all pkgnames on this repo's abs + on_abs=($( + find "${SVNREPO}/${_repo}" -name PKGBUILD | \ + while read pkgbuild; do + source "${pkgbuild}" >/dev/null 2>&1 + # cleanup to save memory + unset build package source md5sums pkgdesc pkgver pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + + # also cleanup package functions + for _pkg in "${pkgname[@]}"; do + unset "package_${pkg}" >/dev/null 2>&1 + done + + # this fills the on_abs array + echo "${pkgname[@]}" + done + )) + + # quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[%s]'s ABS tree is empty, skipping" "${_repo}" + break + fi + + # Find all pkgnames on repos + on_repo=($( + find "${FTP_BASE}/${_repo}" -name "*.pkg.tar.?z" \ + -printf "%f\n" | sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + )) + + # Compares them, whatever is on repos but not on abs should be removed + remove=($(comm -13 \ + <(printf '%s\n' "${on_abs[@]}" | sort -u) \ + <(printf '%s\n' "${on_repo[@]}" | sort -u) )) + + # Remove them from databases, ftpdir-cleanup will take care of the rest + find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \ + -exec repo-remove {} "${remove[@]}" \; >/dev/null 2>&1 + + msg2 "Removed the following packages:" + plain '%s' "${remove[@]}" +done diff --git a/extra/coadde/cron-jobs/sourceballs b/extra/coadde/cron-jobs/sourceballs new file mode 100755 index 0000000..c12a128 --- /dev/null +++ b/extra/coadde/cron-jobs/sourceballs @@ -0,0 +1,151 @@ +#!/bin/bash + +dirname="$(dirname "$(readlink -e "$0")")" +. "${dirname}/../config" +. "${dirname}/../db-functions" +pushd "${WORKDIR}" >/dev/null + +script_lock + +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 + done +done + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a readable file for each repo with the following format +# - [ ] +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + # Repo does not exist; skip it + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + continue + fi + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + | awk '/^%NAME%/ { getline b }; + /^%BASE%/ { getline b }; + /^%VERSION%/ { getline v }; + /^%LICENSE%/,/^$/ { + if ( !/^%LICENSE%/ ) { l=l" "$0 } + }; + /^%ARCH%/ { + getline a; + printf "%s %s %s %s\n", b, v, a, l; + l=""; + }' + done | sort -u > "${WORKDIR}/db-${repo}" +done + +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${arch}" + done +done + +# Create a list of all available source package file names +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Check for all packages if we need to build a source package +for repo in "${PKGREPOS[@]}"; do + newpkgs=() + failedpkgs=() + while read line; do + pkginfo=("${line}") + pkgbase=${pkginfo[0]} + pkgver=${pkginfo[1]} + pkgarch=${pkginfo[2]} + pkglicense=("${pkginfo[@]:3}") + + # Should this package be skipped? + if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then + continue + fi + # Commenting out, we'll sourceball everything + # Check if the license or .force file does not enforce creating a source package +# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then +# continue +# fi + # Store the expected file name of the source package + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Build the source package if its not already there + if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then + # Check if we had failed before + if in_array "${pkgbase}-${pkgver}${SRCEXT}" "${failedpkgs[@]}"; then + continue + fi + + # Get the sources from xbs + mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" + cp -a "$(xbs releasepath "${pkgbase}" "${repo}" "${pkgarch}")" \ + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + if [ $? -ge 1 ]; then + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") + continue + fi + + # Build the actual source package + pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + SRCPKGDEST=. makepkg --nocolor --allsource --ignorearch --skippgpcheck >"${WORKDIR}/${pkgbase}.log" 2>&1 + if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then + mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" + # Avoid creating the same source package for every arch + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" + newpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") + else + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") + cat "${WORKDIR}/${pkgbase}.log" >> "${WORKDIR}/makepkg-fail.log" + fi + popd >/dev/null + fi + done < "${WORKDIR}/db-${repo}" + + if [ ${#newpkgs[@]} -ge 1 ]; then + msg "Adding source packages for [%s]..." "${repo}" + for new_pkg in "${newpkgs[@]}"; do + msg2 '%s' "${new_pkg}" + done + fi + if [ ${#failedpkgs[@]} -ge 1 ]; then + msg "Failed to create source packages for [%s]..." "${repo}" + for failed_pkg in "${failedpkgs[@]}"; do + msg2 '%s' "${failed_pkg}" + done + fi +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + "${SOURCE_CLEANUP_DRYRUN}" && warning 'dry run mode is active' + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + if ! "${SOURCE_CLEANUP_DRYRUN}"; then + mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find "${SOURCE_CLEANUP_DESTDIR}" -type f -name "*${SRCEXT}" -mtime +"${SOURCE_CLEANUP_KEEP}" -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + "${SOURCE_CLEANUP_DRYRUN}" || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +if [ -f "${WORKDIR}/makepkg-fail.log" ]; then + msg "Log of failed packages" + cat "${WORKDIR}/makepkg-fail.log" +fi + +script_unlock diff --git a/extra/coadde/cron-jobs/sourceballs.force b/extra/coadde/cron-jobs/sourceballs.force new file mode 100644 index 0000000..badf15d --- /dev/null +++ b/extra/coadde/cron-jobs/sourceballs.force @@ -0,0 +1,4 @@ +faad2 +wxgtk +wxpython +glhack diff --git a/extra/coadde/cron-jobs/sourceballs.skip b/extra/coadde/cron-jobs/sourceballs.skip new file mode 100644 index 0000000..14d6f4b --- /dev/null +++ b/extra/coadde/cron-jobs/sourceballs.skip @@ -0,0 +1,29 @@ +nexuiz-data +torcs-data +tremulous-data +ufoai-data +frogatto-data +vdrift-data +naev-data +btanks-data +wesnoth-data +texlive-bin +texlive-bibtexextra +texlive-core +texlive-fontsextra +texlive-formatsextra +texlive-games +texlive-genericextra +texlive-htmlxml +texlive-humanities +texlive-langcjk +texlive-langcyrillic +texlive-langextra +texlive-langgreek +texlive-latexextra +texlive-music +texlive-pictures +texlive-plainextra +texlive-pstricks +texlive-publishers +texlive-science diff --git a/extra/coadde/db-check-nonfree b/extra/coadde/db-check-nonfree new file mode 100755 index 0000000..37b7cf6 --- /dev/null +++ b/extra/coadde/db-check-nonfree @@ -0,0 +1,46 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -ge 1 ]; then + error "Calling %s with a specific repository is not supported" "${0##*/}" + exit 1 +fi + +# TODO: this might lock too much (architectures) +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${pkgarch}" || exit 1 + done +done + +msg "Check nonfree in repo:" +nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u)) +for repo in "${ARCHREPOS[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + msg2 "%s %s" "$repo" "$pkgarch" + if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then + continue + fi + unset dbpkgs + unset cleanpkgs + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in "${dbpkgs[@]}"; do + if in_array "${pkgname}" "${nonfree[@]}"; then + cleanpkgs+=("${pkgname}") + fi + done + if [ ${#cleanpkgs[@]} -ge 1 ]; then + msg2 "Nonfree: %s" "${cleanpkgs[*]}" + arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" + fi + done +done + +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${pkgarch}" + done +done diff --git a/extra/coadde/db-check-package-libraries.py b/extra/coadde/db-check-package-libraries.py new file mode 100755 index 0000000..bc2349b --- /dev/null +++ b/extra/coadde/db-check-package-libraries.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Check which libraries are provided or required by a package, store +this in a database, update and list broken packages. + +Dependencies: + +- Python 3.2 or later with SQLite 3 support + +- ``bsdtar`` + +- ``readelf`` +""" + + +import os.path +import re +import sqlite3 +import subprocess +import tempfile + + +#: Regexp matching an interesting dynamic entry. +_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" + "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") + + +def make_db(path): + """Make a new, empty, library database at *path*.""" + con = sqlite3.connect(path) + con.executescript(""" +create table provided( + library varchar not null, + package varchar not null +); +create table used( + library varchar not null, + package varchar not null +); +""") + con.close() + + +def begin(database): + """Connect to *database* and start a transaction.""" + con = sqlite3.connect(database) + con.execute("begin exclusive") + return con + + +def add_provided(con, package, libraries): + """Write that *package* provides *libraries*.""" + for library in libraries: + con.execute("insert into provided (package, library) values (?,?)", + (package, library)) + + +def add_used(con, package, libraries): + """Write that *package* uses *libraries*.""" + for library in libraries: + con.execute("insert into used (package, library) values (?,?)", + (package, library)) + + +def remove_package(con, package): + """Remove all entries for a package.""" + con.execute("delete from provided where package=?", (package,)) + con.execute("delete from used where package=?", (package,)) + + +def add_package(con, package): + """Add entries from a named *package*.""" + # Extract to a temporary directory. This could be done more + # efficiently, since there is no need to store more than one file + # at once. + with tempfile.TemporaryDirectory() as temp: + tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) + tar.communicate() + with open(os.path.join(temp, ".PKGINFO")) as pkginfo: + for line in pkginfo: + if line.startswith("pkgname ="): + pkgname = line[len("pkgname ="):].strip() + break + # Don't list previously removed libraries. + remove_package(con, pkgname) + provided = set() + used = set() + # Search for ELFs. + for dirname, dirnames, filenames in os.walk(temp): + assert dirnames is not None # unused, avoid pylint warning + for file_name in filenames: + path = os.path.join(dirname, file_name) + with open(path, "rb") as file_object: + if file_object.read(4) != b"\177ELF": + continue + readelf = subprocess.Popen(("readelf", "-d", path), + stdout=subprocess.PIPE) + for line in readelf.communicate()[0].split(b"\n"): + match = _DYNAMIC.match(line.decode("ascii")) + if match: + if match.group(1) == "SONAME": + provided.add(match.group(2)) + elif match.group(1) == "NEEDED": + used.add(match.group(2)) + else: + raise AssertionError("unknown entry type " + + match.group(1)) + add_provided(con, pkgname, provided) + add_used(con, pkgname, used) + + +def init(arguments): + """Initialize.""" + make_db(arguments.database) + + +def add(arguments): + """Add packages.""" + con = begin(arguments.database) + for package in arguments.packages: + add_package(con, package) + con.commit() + con.close() + + +def remove(arguments): + """Remove packages.""" + con = begin(arguments.database) + for package in arguments.packages: + remove_package(con, package) + con.commit() + con.close() + + +def check(arguments): + """List broken packages.""" + con = begin(arguments.database) + available = set(row[0] for row + in con.execute("select library from provided")) + for package, library in con.execute("select package, library from used"): + if library not in available: + print(package, "needs", library) + con.close() + + +def main(): + """Get arguments and run the command.""" + from argparse import ArgumentParser + parser = ArgumentParser(prog="check-package-libraries.py", + description="Check packages for " + "provided/needed libraries") + parser.add_argument("-d", "--database", type=str, + help="Database file to use", + default="package-libraries.sqlite") + subparsers = parser.add_subparsers() + subparser = subparsers.add_parser(name="init", + help="initialize the database") + subparser.set_defaults(command=init) + subparser = subparsers.add_parser(name="add", + help="add packages to database") + subparser.add_argument("packages", nargs="+", type=str, + help="package files to add") + subparser.set_defaults(command=add) + subparser = subparsers.add_parser(name="remove", + help="remove packages from database") + subparser.add_argument("packages", nargs="+", type=str, + help="package names to remove") + subparser.set_defaults(command=remove) + subparser = subparsers.add_parser(name="check", + help="list broken packages") + subparser.set_defaults(command=check) + arguments = parser.parse_args() + arguments.command(arguments) + + +if __name__ == "__main__": + main() diff --git a/extra/coadde/db-cleanup b/extra/coadde/db-cleanup new file mode 100755 index 0000000..ffa2601 --- /dev/null +++ b/extra/coadde/db-cleanup @@ -0,0 +1,69 @@ +#!/bin/bash +# Syncs pools against themselves using database contents as filter to cleanup +# them up +# License: GPLv3 + +# Principles +# * Get repos dbs contents +# * Make them a include list +# * Rsync pools against themselves removing excluded files +# * Instant cleanup! + +trap_exit() { + echo + error "$@" + exit 1 +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +EXTRAFLAGS=() +"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) + +filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") +trap "rm -f -- $(printf %q "$filter")" EXIT + +for _repo in "${PKGREPOS[@]}"; do + for _arch in "${ARCHES[@]}"; do + msg "Getting %s-%s database" "${_repo}" "${_arch}" + + dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" + + if [ ! -r "${dbfile}" ]; then + warning "Not found" + continue + fi + + # Echo the contents into a filter file + bsdtar tf "${dbfile}" | \ + cut -d'/' -f1 | \ + sort -u | \ + sed "s|$|*|" >> "$filter" + + done +done + +msg "Removing old files:" + +for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do + msg2 '%s' "${POOL}" + + rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ + --include-from="$filter" \ + --exclude="*" \ + "${FTP_BASE}/${POOL}/" \ + "${FTP_BASE}/${POOL}/" +done + +msg "Removing dead symlinks:" +actions=(-print) +"${CLEANUP_DRYRUN}" || actions+=(-delete) +find -L "${FTP_BASE}/" -type l "${actions[@]}" diff --git a/extra/coadde/db-functions b/extra/coadde/db-functions new file mode 100644 index 0000000..d76aa41 --- /dev/null +++ b/extra/coadde/db-functions @@ -0,0 +1,524 @@ +#!/hint/bash + +# Some PKGBUILDs need CARCH to be set +CARCH="x86_64" + +# Useful functions +UMASK="" +set_umask () { + [ "$UMASK" == "" ] && UMASK="$(umask)" + export UMASK + umask 002 +} + +restore_umask () { + umask "$UMASK" >/dev/null +} + +# just like mv -f, but we touch the file and then copy the content so +# default ACLs in the target dir will be applied +mv_acl() { + rm -f "$2" + touch "$2" + cat "$1" >"$2" || return 1 + rm -f "$1" +} + +# set up general environment +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +LOCKS=() +REPO_MODIFIED=0 + +# check if messages are to be printed using color +unset ALL_OFF BOLD BLUE GREEN RED YELLOW +if [[ -t 2 ]]; then + ALL_OFF="$(tput sgr0)" + BOLD="$(tput bold)" + BLUE="${BOLD}$(tput setaf 4)" + GREEN="${BOLD}$(tput setaf 2)" + RED="${BOLD}$(tput setaf 1)" + YELLOW="${BOLD}$(tput setaf 3)" +fi +readonly ALL_OFF BOLD BLUE GREEN RED YELLOW + +plain() { + local mesg=$1; shift + printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" +} + +msg() { + local mesg=$1; shift + printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" +} + +msg2() { + local mesg=$1; shift + printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" +} + +warning() { + local mesg=$1; shift + printf "${YELLOW}==> WARNING:${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "${RED}==> ERROR${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +## +# usage : in_array( $needle, $haystack ) +# return : 0 - found +# 1 - not found +## +in_array() { + local needle=$1; shift + [[ -z $1 ]] && return 1 # Not Found + local item + for item in "$@"; do + [[ $item = "$needle" ]] && return 0 # Found + done + return 1 # Not Found +} + +## +# usage : get_full_version( $epoch, $pkgver, $pkgrel ) +# return : full version spec, including epoch (if necessary), pkgver, pkgrel +## +get_full_version() { + if [[ $1 -eq 0 ]]; then + # zero epoch case, don't include it in version + echo "$2-$3" + else + echo "$1:$2-$3" + fi +} + +script_lock() { + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" + if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then + local _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" + error "Script %s is already locked by %s." "${0##*/}" "$_owner" + exit 1 + else + set_umask + return 0 + fi +} + +script_unlock() { + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" + if [ ! -d "$LOCKDIR" ]; then + warning "Script %s was not locked!" "${0##*/}" + restore_umask + return 1 + else + rmdir "$LOCKDIR" + restore_umask + return 0 + fi +} + +cleanup() { + local l + local repo + local arch + + trap - EXIT INT QUIT TERM + for l in "${LOCKS[@]}"; do + repo=${l%.*} + arch=${l#*.} + if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then + msg "Removing left over lock from [%s] (%s)" "${repo}" "${arch}" + repo_unlock "$repo" "$arch" + fi + done + if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then + msg "Removing left over lock from %s" "${0##*/}" + script_unlock + fi + rm -rf "$WORKDIR" + + if (( REPO_MODIFIED )); then + date +%s > "${FTP_BASE}/lastupdate" + date -u +%s > "${FTP_BASE}/lastsync" + fi + + [ "$1" ] && exit "$1" +} + +abort() { + msg 'Aborting...' + cleanup 0 +} + +die() { + error "$@" + cleanup 1 +} + +trap abort INT QUIT TERM HUP +trap cleanup EXIT + + +#repo_lock [timeout] +repo_lock () { + local LOCKDIR="$TMPDIR/.repolock.$1.$2" + local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" + local FILESLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${FILESEXT}.lck" + local _count + local _trial + local _timeout + local _lockblock + local _owner + + # This is the lock file used by repo-add and repo-remove + if [ -f "${DBLOCKFILE}" ]; then + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$DBLOCKFILE")" + return 1 + fi + if [ -f "${FILESLOCKFILE}" ]; then + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$FILESLOCKFILE")" + return 1 + fi + + if [ $# -eq 2 ]; then + _lockblock=true + _trial=0 + elif [ $# -eq 3 ]; then + _lockblock=false + _timeout=$3 + let _trial=$_timeout/$LOCK_DELAY + fi + + _count=0 + while [ "$_count" -le "$_trial" ] || "$_lockblock" ; do + if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then + _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" + warning "Repo [%s] (%s) is already locked by %s." "${1}" "${2}" "$_owner" + msg2 "Retrying in %d seconds..." "$LOCK_DELAY" + else + LOCKS+=("$1.$2") + set_umask + return 0 + fi + sleep "$LOCK_DELAY" + let _count=$_count+1 + done + + error "Repo [%s] (%s) is already locked by %s. Giving up!" "${1}" "${2}" "$_owner" + return 1 +} + +repo_unlock () { #repo_unlock + local LOCKDIR="$TMPDIR/.repolock.$1.$2" + if [ ! -d "$LOCKDIR" ]; then + warning "Repo lock [%s] (%s) was not locked!" "${1}" "${2}" + restore_umask + return 1 + else + rmdir "$LOCKDIR" + restore_umask + return 0 + fi +} + +# usage: _grep_pkginfo pkgfile pattern +_grep_pkginfo() { + local _ret + + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" + echo "${_ret#${2} = }" +} + + +# Get the package base or name as fallback +getpkgbase() { + local _base + + _base="$(_grep_pkginfo "$1" "pkgbase")" + if [ -z "$_base" ]; then + getpkgname "$1" + else + echo "$_base" + fi +} + +issplitpkg() { + local _base + + _base="$(_grep_pkginfo "$1" "pkgbase")" + if [ -z "$_base" ]; then + return 1 + else + return 0 + fi +} + +# Get the package name +getpkgname() { + local _name + + _name="$(_grep_pkginfo "$1" "pkgname")" + if [ -z "$_name" ]; then + error "Package '%s' has no pkgname in the PKGINFO. Fail!" "$1" + exit 1 + fi + + echo "$_name" +} + +# Get the pkgver-pkgrel of this package +getpkgver() { + local _ver + + _ver="$(_grep_pkginfo "$1" "pkgver")" + if [ -z "$_ver" ]; then + error "Package '%s' has no pkgver in the PKGINFO. Fail!" "$1" + exit 1 + fi + + echo "$_ver" +} + +getpkgarch() { + local _ver + + _ver="$(_grep_pkginfo "$1" "arch")" + if [ -z "$_ver" ]; then + error "Package '%s' has no arch in the PKGINFO. Fail!" "$1" + exit 1 + fi + + echo "$_ver" +} + +getpkgfile() { + if [[ ${#} -ne 1 ]]; then + error 'No canonical package found!' + exit 1 + elif [ ! -f "${1}" ]; then + error "Package %s not found!" "${1}" + exit 1 + elif "${REQUIRE_SIGNATURE}" && [ ! -f "${1}.sig" ]; then + error "Package signature %s not found!" "${1}.sig" + exit 1 + fi + + echo "${1}" +} + +getpkgfiles() { + local f + if [ ! -z "$(printf '%s\n' "${@%\.*}" | sort | uniq -D)" ]; then + error 'Duplicate packages found!' + exit 1 + fi + + for f in "${@}"; do + if [ ! -f "${f}" ]; then + error "Package %s not found!" "${f}" + exit 1 + elif "${REQUIRE_SIGNATURE}" && [ ! -f "${f}.sig" ]; then + error "Package signature %s not found!" "${f}.sig" + exit 1 + fi + done + + echo "${@}" +} + +check_pkgfile() { + local pkgfile=$1 + + local pkgname="$(getpkgname "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local pkgver="$(getpkgver "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local pkgarch="$(getpkgarch "${pkgfile}")" + [ $? -ge 1 ] && return 1 + + in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 + + if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then + return 0 + else + return 1 + fi +} + +check_pkgxbs() { + local pkgfile="${1}" + local _pkgbase="$(getpkgbase "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local _pkgname="$(getpkgname "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local _pkgver="$(getpkgver "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local _pkgarch="$(getpkgarch "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local repo="${2}" + + in_array "${repo}" "${PKGREPOS[@]}" || return 1 + + local xbsver="$(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; get_full_version "${_pkgname}")" + [ "${xbsver}" == "${_pkgver}" ] || return 1 + + local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) + in_array "${_pkgname}" "${xbsnames[@]}" || return 1 + + return 0 +} + +check_splitpkgs() { + local repo="${1}" + shift + local pkgfiles=("${@}") + local pkgfile + local pkgdir + local xbsname + + mkdir -p "${WORKDIR}/check_splitpkgs/" + pushd "${WORKDIR}/check_splitpkgs" >/dev/null + + for pkgfile in "${pkgfiles[@]}"; do + issplitpkg "${pkgfile}" || continue + local _pkgbase="$(getpkgbase "${pkgfile}")" + msg2 "Checking %s" "$_pkgbase" + local _pkgname="$(getpkgname "${pkgfile}")" + local _pkgarch="$(getpkgarch "${pkgfile}")" + mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" + echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" + + local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) + printf '%s\n' "${xbsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" + done + popd >/dev/null + + for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do + [ ! -d "${pkgdir}" ] && continue + sort -u "${pkgdir}/staging" -o "${pkgdir}/staging" + sort -u "${pkgdir}/xbs" -o "${pkgdir}/xbs" + if [ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/xbs")" ]; then + return 1 + fi + done + + return 0 +} + +check_pkgrepos() { + local pkgfile=$1 + + local pkgname="$(getpkgname "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local pkgver="$(getpkgver "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local pkgarch="$(getpkgarch "${pkgfile}")" + [ $? -ge 1 ] && return 1 + + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 + + local repo + local arch + for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}.sig" ] && return 1 + done + done + + return 0 +} + +#usage: chk_license ${license[@]}" +chk_license() { + local l + for l in "${@}"; do + in_array "${l}" "${ALLOWED_LICENSES[@]}" && return 0 + done + + return 1 +} + +check_repo_permission() { + local repo=$1 + + [ ${#PKGREPOS[@]} -eq 0 ] && return 1 + [ -z "${PKGPOOL}" ] && return 1 + + in_array "${repo}" "${PKGREPOS[@]}" || return 1 + + [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 + + local arch + for arch in "${ARCHES[@]}"; do + local dir="${FTP_BASE}/${repo}/os/${arch}/" + [ -w "${dir}" ] || return 1 + [ -f "${dir}${repo}"${DBEXT} -a ! -w "${dir}${repo}"${DBEXT} ] && return 1 + [ -f "${dir}${repo}"${FILESEXT} -a ! -w "${dir}${repo}"${FILESEXT} ] && return 1 + done + + return 0 +} + +set_repo_permission() { + local repo=$1 + local arch=$2 + local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ -w "${dbfile}" ]; then + local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "${dbfile}" "$group" + chgrp "$group" "${filesfile}" || error "Could not change group of %s to %s" "${filesfile}" "$group" + chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "${dbfile}" + chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "${filesfile}" + else + error "You don't have permission to change %s" "${dbfile}" + fi +} + +arch_repo_add() { + local repo=$1 + local arch=$2 + local pkgs=("${@:3}") + + printf -v pkgs_str -- '%q ' "${pkgs[@]}" + # package files might be relative to repo dir + pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null + /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ + || error 'repo-add %q %s' "${repo}${DBEXT}" "${pkgs_str% }" + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ + || error 'repo-add -f %q %s' "${repo}${FILESEXT}" "${pkgs_str% }" + popd >/dev/null + set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 +} + +arch_repo_remove() { + local repo=$1 + local arch=$2 + local pkgs=("${@:3}") + local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${dbfile}" ]; then + error "No database found at '%s'" "${dbfile}" + return 1 + fi + printf -v pkgs_str -- '%q ' "${pkgs[@]}" + /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ + || error 'repo-remove %q %s' "${dbfile}" "${pkgs_str% }" + /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ + || error 'repo-remove %q %s' "${filesfile}" "${pkgs_str% }" + set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 +} diff --git a/extra/coadde/db-libremessages b/extra/coadde/db-libremessages new file mode 100755 index 0000000..37df149 --- /dev/null +++ b/extra/coadde/db-libremessages @@ -0,0 +1,83 @@ +# Copyright (c) 2006-2010 Pacman Development Team +# Copyright (c) 2002-2006 by Judd Vinet +# Copyright (c) 2005 by Aurelien Foret +# Copyright (c) 2006 by Miklos Vajna +# Copyright (c) 2005 by Christian Hamar +# Copyright (c) 2006 by Alex Smith +# Copyright (c) 2006 by Andras Voroskoi +# Copyright (c) 2011 by Joshua Haase +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# gettext initialization +export TEXTDOMAIN='libretools' +export TEXTDOMAINDIR='/usr/share/locale' + +# check if messages are to be printed using color +unset ALL_OFF BOLD BLUE GREEN RED YELLOW + +if tput setaf 0 &>/dev/null; then + ALL_OFF="$(tput sgr0)" + BOLD="$(tput bold)" + BLUE="${BOLD}$(tput setaf 4)" + GREEN="${BOLD}$(tput setaf 2)" + RED="${BOLD}$(tput setaf 1)" + YELLOW="${BOLD}$(tput setaf 3)" + PURPLE="${ALL_OFF}$(tput setaf 5)" +else + ALL_OFF="\033[1;0m" + BOLD="\033[1;1m" + BLUE="${BOLD}\033[1;34m" + GREEN="${BOLD}\033[1;32m" + RED="${BOLD}\033[1;31m" + YELLOW="${BOLD}\033[1;33m" + PURPLE="${BOLD}\033[1;30;40m" +fi + +stdnull() { + local action=$1; + eval "${action} >/dev/null 2>&1" +} + +plain() { + local mesg=$1; shift + printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg() { + local mesg=$1; shift + printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg2() { + local mesg=$1; shift + printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +warning() { + local mesg=$1; shift + printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +fatal_error() { + local mesg=$1; shift + error "$mesg" "$@" + + exit 1 +} diff --git a/extra/coadde/db-list-unsigned-packages b/extra/coadde/db-list-unsigned-packages new file mode 100755 index 0000000..095e1e6 --- /dev/null +++ b/extra/coadde/db-list-unsigned-packages @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -e + +# Output a list of repo/package-name-and-version pairs representing +# unsigned packages available for architecture $1 and specified for +# architecture $2 (usually $1 or any, default is to list all). + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -lt 1 ]; then + msg "usage: %s " "${0##*/}" + exit 1 +fi + +arch=$1 +shift + +for repo in "${PKGREPOS[@]}" +do + db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" +done diff --git a/extra/coadde/db-list-unsigned-packages.py b/extra/coadde/db-list-unsigned-packages.py new file mode 100755 index 0000000..80cff51 --- /dev/null +++ b/extra/coadde/db-list-unsigned-packages.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Output a list of repo/package-name-and-version pairs representing +unsigned packages in the database at standard input of repo named in +the first argument and specified for architectures listed in the +following arguments (usually the one of the database or any, default +is to list all). + +If the --keyset argument is passed, print the key fingerprint of every +signed package. +""" + + +import base64 +import subprocess +import sys +import tarfile + + +def main(): + """Do the job.""" + check_keys = False + if "--keyset" in sys.argv: + sys.argv.remove("--keyset") + check_keys = True + repo = sys.argv[1] + pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) + packages = [] + keys = [] + with tarfile.open(fileobj=sys.stdin.buffer) as archive: + for entry in archive: + if entry.name.endswith("/desc"): + content = archive.extractfile(entry) + skip = False + is_arch = False + key = None + for line in content: + if is_arch: + is_arch = False + if pkgarches and line.strip() not in pkgarches: + skip = True # different architecture + break + if line == b"%PGPSIG%\n": + skip = True # signed + key = b"" + if check_keys: + continue + else: + break + if line == b"%ARCH%\n": + is_arch = True + continue + if key is not None: + if line.strip(): + key += line.strip() + else: + break + if check_keys and key: + key_binary = base64.b64decode(key) + keys.append(key_binary) + packages.append(repo + "/" + entry.name[:-5]) + if skip: + continue + print(repo + "/" + entry.name[:-5]) + if check_keys and keys: + # We have collected all signed package names in packages and + # all keys in keys. Let's now ask gpg to list all signatures + # and find which keys made them. + packets = subprocess.check_output(("gpg", "--list-packets"), + input=b"".join(keys)) + i = 0 + for line in packets.decode("latin1").split("\n"): + if line.startswith(":signature packet:"): + keyid = line[line.index("keyid ") + len("keyid "):] + print(packages[i], keyid) + i += 1 + + +if __name__ == "__main__": + main() diff --git a/extra/coadde/db-move b/extra/coadde/db-move new file mode 100755 index 0000000..275a11a --- /dev/null +++ b/extra/coadde/db-move @@ -0,0 +1,106 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -lt 3 ]; then + msg "usage: %s ..." "${0##*/}" + exit 1 +fi + +args=("${@}") +repo_from="${args[0]}" +repo_to="${args[1]}" +ftppath_from="${FTP_BASE}/${repo_from}/os/" +ftppath_to="${FTP_BASE}/${repo_to}/os/" + +if ! check_repo_permission "$repo_to" || ! check_repo_permission "$repo_from"; then + die "You don't have permission to move packages from %s to %s" "${repo_from}" "${repo_to}" +fi + +# TODO: this might lock too much (architectures) +for pkgarch in "${ARCHES[@]}"; do + repo_lock "${repo_to}" "${pkgarch}" || exit 1 + repo_lock "${repo_from}" "${pkgarch}" || exit 1 +done + +# First loop is to check that all necessary files exist +for pkgbase in "${args[@]:2}"; do + for pkgarch in "${ARCHES[@]}" 'any'; do + xbsrepo_from="$(xbs releasepath "${pkgbase}" "${repo_from}" "${pkgarch}")" + if [ -r "${xbsrepo_from}/PKGBUILD" ]; then + pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) + if [ ${#pkgnames[@]} -lt 1 ]; then + die "Could not read pkgname" + fi + + pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") + if [ -z "${pkgver}" ]; then + die "Could not read pkgver" + fi + + if [ "${pkgarch}" == 'any' ]; then + tarches=("${ARCHES[@]}") + else + tarches=("${pkgarch}") + fi + + for pkgname in "${pkgnames[@]}"; do + for tarch in "${tarches[@]}"; do + getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} >/dev/null + done + done + continue 2 + fi + done + die "%s not found in %s" "${pkgbase}" "${repo_from}" +done + +msg "Moving packages from [%s] to [%s]..." "${repo_from}" "${repo_to}" + +declare -A add_pkgs +declare -A remove_pkgs +for pkgbase in "${args[@]:2}"; do + # move the package in xbs + arches=($(xbs move "${repo_from}" "${repo_to}" "${pkgbase}")) + # move the package in ftp + for pkgarch in "${arches[@]}"; do + xbsrepo_to="$(xbs releasepath "$pkgbase" "$repo_to" "$pkgarch")" + if true; then # to add an indent level to make merging easier + if [ "${pkgarch}" == 'any' ]; then + tarches=("${ARCHES[@]}") + else + tarches=("${pkgarch}") + fi + msg2 '%s (%s)' "${pkgbase}" "${tarches[*]}" + pkgnames=($(. "${xbsrepo_to}/PKGBUILD"; echo "${pkgname[@]}")) + pkgver=$(. "${xbsrepo_to}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") + + for pkgname in "${pkgnames[@]}"; do + for tarch in "${tarches[@]}"; do + pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}) + pkgfile="${pkgpath##*/}" + + ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" + if [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig" ]; then + ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" + fi + add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " + remove_pkgs[${tarch}]+="${pkgname} " + done + done + fi + done +done + +for tarch in "${ARCHES[@]}"; do + if [ -n "${add_pkgs[${tarch}]}" ]; then + arch_repo_add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} + arch_repo_remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} + fi +done + +for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${repo_from}" "${pkgarch}" + repo_unlock "${repo_to}" "${pkgarch}" +done diff --git a/extra/coadde/db-remove b/extra/coadde/db-remove new file mode 100755 index 0000000..dcbe4b4 --- /dev/null +++ b/extra/coadde/db-remove @@ -0,0 +1,49 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -lt 3 ]; then + msg "usage: %s ..." "${0##*/}" + exit 1 +fi + +repo="$1" +arch="$2" +pkgbases=("${@:3}") + +if ! check_repo_permission "$repo"; then + die "You don't have permission to remove packages from %s" "${repo}" +fi + +if [ "$arch" == "any" ]; then + tarches=("${ARCHES[@]}") +else + tarches=("$arch") +fi + +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 +done + +remove_pkgs=() +for pkgbase in "${pkgbases[@]}"; do + msg "Removing %s from [%s]..." "$pkgbase" "$repo" + + path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" + if [ -d "$path" ]; then + remove_pkgs+=($(. "$path/PKGBUILD"; echo "${pkgname[@]}")) + xbs unrelease "$pkgbase" "$repo" "$arch" + else + warning "%s not found in %s for %s" \ + "$pkgbase" "$(xbs name)" "$repo-$arch" + warning "Removing only %s from the repo" "$pkgbase" + warning "If it was a split package you have to remove the others yourself!" + remove_pkgs+=("$pkgbase") + fi +done + +for tarch in "${tarches[@]}"; do + arch_repo_remove "${repo}" "${tarch}" "${remove_pkgs[@]}" + repo_unlock "$repo" "$tarch" +done diff --git a/extra/coadde/db-repo-add b/extra/coadde/db-repo-add new file mode 100755 index 0000000..4611bdf --- /dev/null +++ b/extra/coadde/db-repo-add @@ -0,0 +1,41 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -lt 3 ]; then + msg "usage: %s ..." "${0##*/}" + exit 1 +fi + +repo="$1" +arch="$2" +pkgfiles=("${@:3}") + +ftppath="$FTP_BASE/$repo/os" + +if ! check_repo_permission "$repo"; then + die "You don't have permission to add packages to %s" "${repo}" +fi + +if [ "$arch" == "any" ]; then + tarches=("${ARCHES[@]}") +else + tarches=("$arch") +fi + +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 +done + +for tarch in "${tarches[@]}"; do + for pkgfile in "${pkgfiles[@]}"; do + if [[ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ]]; then + die "Package file %s not found in %s" "${pkgfile##*/}" "${FTP_BASE}/${repo}/os/${arch}/" + else + msg "Adding %s to [%s]..." "$pkgfile" "$repo" + fi + done + arch_repo_add "${repo}" "${tarch}" "${pkgfiles[@]}" + repo_unlock "$repo" "$tarch" +done diff --git a/extra/coadde/db-repo-remove b/extra/coadde/db-repo-remove new file mode 100755 index 0000000..aadc4ce --- /dev/null +++ b/extra/coadde/db-repo-remove @@ -0,0 +1,37 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -lt 3 ]; then + msg "usage: %s ..." "${0##*/}" + exit 1 +fi + +repo="$1" +arch="$2" +pkgnames=("${@:3}") + +ftppath="$FTP_BASE/$repo/os" + +if ! check_repo_permission "$repo"; then + die "You don't have permission to remove packages from %s" "${repo}" +fi + +if [ "$arch" == "any" ]; then + tarches=("${ARCHES[@]}") +else + tarches=("$arch") +fi + +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 +done + +for tarch in "${tarches[@]}"; do + for pkgname in "${pkgnames[@]}"; do + msg "Removing %s from [%s]..." "$pkgname" "$repo" + done + arch_repo_remove "${repo}" "${tarch}" "${pkgnames[@]}" + repo_unlock "$repo" "$tarch" +done diff --git a/extra/coadde/db-sync b/extra/coadde/db-sync new file mode 100755 index 0000000..2194fe6 --- /dev/null +++ b/extra/coadde/db-sync @@ -0,0 +1,208 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * make a tarball of files used for forensics + +# Run as `V=true db-sync` to get verbose output +VERBOSE=${V} +extra=() +${VERBOSE} && extra+=(-v) + +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + +# Returns contents of a repo +get_repos() { + # Exclude everything but db files + rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" +} + +get_repo_content() { + # Return all contents + bsdtar tf "${1}" | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + echo "${WORKDIR}/${1}/os/${2}/${1}" +} + +# Process the databases and get the libre packages +init() { + + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in "${ARCHREPOS[@]}"; do + for _arch in "${ARCHARCHES[@]}"; do + msg "Processing %s-%s" "${_repo}-${_arch}" + + db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} + files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content "${db_file}")) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" + + msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync "${extra[@]}" --no-motd -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + "${WORKDIR}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists, check for single *s just in case + cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist + + msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + local pkgpool + for pkgpool in "${ARCHPKGPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ + "${FTP_BASE}/${pkgpool}/" + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + local srcpool + for srcpool in "${ARCHSRCPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ + "${FTP_BASE}/${srcpool}/" + done + + date -u +%s > "${FTP_BASE}/lastsync" + + # Cleanup + unset blacklist whitelists _arch _repo repo_file +} + +trap_exit() { + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" +} + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-sync.conf" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" + +# Check variables presence +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do + test -z "${!var}" && fatal_error "Empty %s" "${var}" +done + +# From makepkg +set -E +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR + +init diff --git a/extra/coadde/db-sync.conf b/extra/coadde/db-sync.conf new file mode 100644 index 0000000..f7748c3 --- /dev/null +++ b/extra/coadde/db-sync.conf @@ -0,0 +1,11 @@ +#mirror="mirrors.kernel.org" +mirror="mirrors.niyawe.de" + +## mirrors without sources folder +#mirror="mirror.nl.leaseweb.net" +#mirror="mirror.one.com" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.bytemark.co.uk" +#mirror="mirror.de.leaseweb.net" + +mirrorpath="archlinux" diff --git a/extra/coadde/db-update b/extra/coadde/db-update new file mode 100755 index 0000000..4830791 --- /dev/null +++ b/extra/coadde/db-update @@ -0,0 +1,118 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -ge 1 ]; then + warning "Calling %s with a specific repository is no longer supported" "${0##*/}" + exit 1 +fi + +# Find repos with packages to release +staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)) +if [ $? -ge 1 ]; then + die "Could not read %s" "${STAGING}" +fi + +repos=() +for staging_repo in "${staging_repos[@]##*/}"; do + if in_array "${staging_repo}" "${PKGREPOS[@]}"; then + repos+=("${staging_repo}") + fi +done + +# TODO: this might lock too much (architectures) +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${pkgarch}" || exit 1 + done +done + +# check if packages are valid +for repo in "${repos[@]}"; do + if ! check_repo_permission "${repo}"; then + die "You don't have permission to update packages in %s" "${repo}" + fi + pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) + if [ $? -eq 0 ]; then + for pkg in "${pkgs[@]}"; do + if [ -h "${pkg}" ]; then + die "Package %s is a symbolic link" "${repo}/${pkg##*/}" + fi + if ! check_pkgfile "${pkg}"; then + die "Package %s is not consistent with its meta data" "${repo}/${pkg##*/}" + fi + if ! check_pkgrepos "${pkg}"; then + die "Package %s already exists in another repository" "${repo}/${pkg##*/}" + fi + done + # This is fucking obnoxious + #if ! check_splitpkgs ${repo} "${pkgs[@]}"; then + # die "Missing split packages for %s" "${repo}" + #fi + else + die "Could not read %s" "${STAGING}" + fi +done + +dirs=() +for repo in "${repos[@]}"; do + msg "Updating [%s]..." "${repo}" + any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) + for pkgarch in "${ARCHES[@]}"; do + add_dirs=() + add_pkgs=() + arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-"${pkgarch}"${PKGEXT} 2>/dev/null)) + for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do + pkgfile="${pkg##*/}" + msg2 "%s (%s)" "${pkgfile}" "${pkgarch}" + # any packages might have been moved by the previous run + if [ -f "${pkg}" ]; then + mv "${pkg}" "$FTP_BASE/${PKGPOOL}" + fi + ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" + # also move signatures + if [ -f "${pkg}.sig" ]; then + mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" + fi + if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then + ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" + fi + add_dirs+=("${STAGING}/abslibre/$(getpkgarch "$FTP_BASE/$PKGPOOL/$pkgfile")/$repo/$(getpkgbase "$FTP_BASE/$PKGPOOL/$pkgfile")") + add_pkgs+=("${pkgfile}") + done + for add_dir in "${add_dirs[@]}"; do + (cd "${add_dir}" && xbs release-server "${repo}" "${pkgarch}") || + error 'cd %q && xbs release-server %q %q' "${add_dir}" "${repo}" "${pkgarch}" + done + if [ ${#add_pkgs[@]} -ge 1 ]; then + arch_repo_add "${repo}" "${pkgarch}" "${add_pkgs[@]}" + fi + dirs+=("${add_dirs[@]}") + done +done + +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${pkgarch}" + done +done + +cd "${STAGING}" + +# Remove left over XBS files +rm -rf -- "${dirs[@]}" +dirname -z -- "${dirs[@]}" | + xargs -0 realpath -zm --relative-to="${STAGING}" -- | + xargs -0 rmdir -p -- 2>/dev/null + +# Stage generated source files +while read -r file; do + pub="${FTP_BASE}/${file}" + if [[ -f "$pub" ]]; then + warning "file already exists: %s" "${file}" + else + mkdir -p -- "${pub%/*}" + mv -vn "$file" "$pub" + fi +done < <(find other sources -type f 2>/dev/null) diff --git a/extra/coadde/list_nonfree_in_db.py b/extra/coadde/list_nonfree_in_db.py new file mode 100755 index 0000000..a486fa5 --- /dev/null +++ b/extra/coadde/list_nonfree_in_db.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python2 +#-*- encoding: utf-8 -*- +from filter import * +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="nonfree_in_db", + description="Cleans nonfree files on repo",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean", + required=True,) + + args=parser.parse_args() + + if not (args.blacklist_file and args.database): + parser.print_help() + exit(1) + + blacklist=listado(args.blacklist_file) + pkgs=get_pkginfo_from_db(args.database) + + print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) diff --git a/extra/coadde/make_individual_torrent b/extra/coadde/make_individual_torrent new file mode 100755 index 0000000..0a7e778 --- /dev/null +++ b/extra/coadde/make_individual_torrent @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright (C) 2014 Joseph Graham +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script is called by `make_repo_torrents' to make a torrent. It +# depends on `mktorrent'. It takes the following args: +# $1 - path of package +# $2 - public location + +# Comma seperated list of trackers, no spaces +# t67.eu is run by Xylon, hackcoop by fauno & friends +trackers='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' + +# This mirror is put as a webseed. Which mirror we use for a webseed +# doesn't really matter since it's re-written on the client machine by +# pacman2pacman so it won't normally be used anyway. +seed_url='http://repo.parabolagnulinux.org/' + +if [[ -z "${1}" ]] +then + echo "Error. First arg must be the path of the package." + echo 1 +fi + +if [[ -z "${2}" ]] +then + echo "Error. Second arg must be the public location." + echo 1 +fi + +pkg="${1}" +public_location="${2}" + +pkg_name="${pkg##*/}" + +# URL of the actual package for the webseed +webseed="${seed_url}${pkg#${public_location}}" + +mktorrent -a "${trackers}" "${pkg}" -w "${webseed}" >/dev/null || +echo "Error making torrent for \"${pkg}\"" diff --git a/extra/coadde/mkrepo b/extra/coadde/mkrepo new file mode 100755 index 0000000..b11dc0b --- /dev/null +++ b/extra/coadde/mkrepo @@ -0,0 +1,15 @@ +#!/bin/bash +# Author: Nicolás Reynolds +# License: GPLv3+ +# Description: A script to quickly create new [repos] + +source "$(dirname "$(readlink -e "$0")")/config" + +for repo in "$@"; do + echo ":: Creating [$repo]" + for arch in "${ARCHES[@]}"; do + mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" + done +done + +echo ":: All done. Add the repo to the ParabolaWeb admin page." diff --git a/extra/coadde/test/__init__.py b/extra/coadde/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/extra/coadde/test/blacklist_sample b/extra/coadde/test/blacklist_sample new file mode 100644 index 0000000..2a02af6 --- /dev/null +++ b/extra/coadde/test/blacklist_sample @@ -0,0 +1,2 @@ +alex:alex-libre: Aquí va un comentario +gmime22 ::Non free dependencies \ No newline at end of file diff --git a/extra/coadde/test/core.db.tar.gz b/extra/coadde/test/core.db.tar.gz new file mode 100644 index 0000000..5eb2081 Binary files /dev/null and b/extra/coadde/test/core.db.tar.gz differ diff --git a/extra/coadde/test/depends b/extra/coadde/test/depends new file mode 100644 index 0000000..7ff3ad4 --- /dev/null +++ b/extra/coadde/test/depends @@ -0,0 +1,4 @@ +%DEPENDS% +glibc>=2.13 +zlib + diff --git a/extra/coadde/test/desc b/extra/coadde/test/desc new file mode 100644 index 0000000..abba644 --- /dev/null +++ b/extra/coadde/test/desc @@ -0,0 +1,39 @@ +%FILENAME% +binutils-2.21-4-x86_64.pkg.tar.xz + +%NAME% +binutils + +%VERSION% +2.21-4 + +%DESC% +A set of programs to assemble and manipulate binary and object files + +%GROUPS% +base + +%CSIZE% +3412892 + +%ISIZE% +17571840 + +%MD5SUM% +4e666f87c78998f4839f33dc06d2043a + +%URL% +http://www.gnu.org/software/binutils/ + +%LICENSE% +GPL + +%ARCH% +x86_64 + +%BUILDDATE% +1297240369 + +%PACKAGER% +Allan McRae + diff --git a/extra/coadde/test/lib/common.inc b/extra/coadde/test/lib/common.inc new file mode 100644 index 0000000..a2dee10 --- /dev/null +++ b/extra/coadde/test/lib/common.inc @@ -0,0 +1,266 @@ +set -E + +. "$(dirname ${BASH_SOURCE[0]})/../../config" +. "$(dirname ${BASH_SOURCE[0]})/../../db-functions" + +oneTimeSetUp() { + local p + local d + local a + local pkgname + local pkgarch + local pkgversion + local build + pkgdir="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)" + cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" + msg 'Building packages...' + for d in "${pkgdir}"/*; do + pushd $d >/dev/null + pkgname=($(. PKGBUILD; echo ${pkgname[@]})) + pkgarch=($(. PKGBUILD; echo ${arch[@]})) + pkgversion=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + + build=true + for a in ${pkgarch[@]}; do + for p in ${pkgname[@]}; do + [ ! -f ${p}-${pkgversion}-${a}${PKGEXT} ] && build=false + done + done + + if ! ${build}; then + if [ "${pkgarch[0]}" == 'any' ]; then + sudo extra-x86_64-build || die 'extra-x86_64-build failed' + else + for a in ${pkgarch[@]}; do + sudo extra-${a}-build || die "extra-${a}-build failed" + done + fi + for a in ${pkgarch[@]}; do + for p in ${pkgname[@]}; do + cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/$(basename ${d}) + done + done + fi + popd >/dev/null + done +} + +oneTimeTearDown() { + rm -rf "${pkgdir}" +} + +setUp() { + local p + local pkg + local r + local a + + [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" + TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" + #msg "Using ${TMP}" + + PKGREPOS=('core' 'extra' 'testing') + PKGPOOL='pool/packages' + mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} + + for r in ${PKGREPOS[@]}; do + mkdir -p "${TMP}/staging/${r}" + for a in ${ARCHES[@]} any; do + mkdir -p "${TMP}/ftp/${r}/os/${a}" + done + done + mkdir -p "${TMP}/ftp/${PKGPOOL}" + mkdir -p "${TMP}/ftp/${SRCPOOL}" + + msg 'Creating svn repository...' + svnadmin create "${TMP}/svn-packages-repo" + svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + + for p in "${pkgdir}"/*; do + pkg=$(basename $p) + mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} + cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ + svn add -q "${TMP}/svn-packages-copy"/${pkg} + svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" + done + + cat < "$(dirname ${BASH_SOURCE[0]})/../../config.local" + FTP_BASE="${TMP}/ftp" + SVNREPO="file://${TMP}/svn-packages-repo" + PKGREPOS=(${PKGREPOS[@]}) + PKGPOOL="${PKGPOOL}" + CLEANUP_DESTDIR="${TMP}/package-cleanup" + SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" + STAGING="${TMP}/staging" + TMPDIR="${TMP}/tmp" + CLEANUP_DRYRUN=false + SOURCE_CLEANUP_DRYRUN=false + REQUIRE_SIGNATURE=true +eot + . "$(dirname ${BASH_SOURCE[0]})/../../config" +} + +tearDown() { + rm -rf "${TMP}" + rm -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" + echo +} + +releasePackage() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + + pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null + archrelease ${repo}-${arch} >/dev/null 2&>1 + pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + popd >/dev/null + cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ + + if ${REQUIRE_SIGNATURE}; then + # TODO: really sign the packages with a valid key + find "${STAGING}"/${repo}/ -type f \ + -name "*-${pkgver}-${arch}${PKGEXT}" \ + -exec touch {}.sig \; + fi +} + +checkAnyPackageDB() { + local repo=$1 + local pkg=$2 + local arch + local db + + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" + if ${REQUIRE_SIGNATURE}; then + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" + fi + + for arch in i686 x86_64; do + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" + + if ${REQUIRE_SIGNATURE}; then + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" + fi + done + [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" + [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" + + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done + + [ -r "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" + [ -r "${FTP_BASE}/${repo}/os/any/${pkg}.sig" ] && fail "${repo}/os/any/${pkg}.sig should not exist" +} + +checkAnyPackage() { + local repo=$1 + local pkg=$2 + + checkAnyPackageDB $repo $pkg + + local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ + || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" +} + +checkPackageDB() { + local repo=$1 + local pkg=$2 + local arch=$3 + local db + + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" + [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" + + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" + + if ${REQUIRE_SIGNATURE}; then + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" + [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" + + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" + fi + + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done +} + +checkPackage() { + local repo=$1 + local pkg=$2 + local arch=$3 + + checkPackageDB $repo $pkg $arch + + local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ + || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" +} + +checkRemovedPackageDB() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + local db + + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done +} + +checkRemovedPackage() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + + checkRemovedPackageDB $repo $pkgbase $arch + + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ + && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" +} + +checkRemovedAnyPackageDB() { + local repo=$1 + local pkgbase=$2 + local arch + local db + + for db in ${DBEXT} ${FILESEXT}; do + for arch in i686 x86_64; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done + done +} + +checkRemovedAnyPackage() { + local repo=$1 + local pkgbase=$2 + + checkRemovedAnyPackageDB $repo $pkgbase + + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ + && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" +} diff --git a/extra/coadde/test/lib/shunit2 b/extra/coadde/test/lib/shunit2 new file mode 100755 index 0000000..8862ffd --- /dev/null +++ b/extra/coadde/test/lib/shunit2 @@ -0,0 +1,1048 @@ +#! /bin/sh +# $Id: shunit2 335 2011-05-01 20:10:33Z kate.ward@forestent.com $ +# vim:et:ft=sh:sts=2:sw=2 +# +# Copyright 2008 Kate Ward. All Rights Reserved. +# Released under the LGPL (GNU Lesser General Public License) +# +# shUnit2 -- Unit testing framework for Unix shell scripts. +# http://code.google.com/p/shunit2/ +# +# Author: kate.ward@forestent.com (Kate Ward) +# +# shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is +# based on the popular JUnit unit testing framework for Java. + +# return if shunit already loaded +[ -n "${SHUNIT_VERSION:-}" ] && exit 0 + +SHUNIT_VERSION='2.1.6' + +SHUNIT_TRUE=0 +SHUNIT_FALSE=1 +SHUNIT_ERROR=2 + +# enable strict mode by default +SHUNIT_STRICT=${SHUNIT_STRICT:-${SHUNIT_TRUE}} + +_shunit_warn() { echo "shunit2:WARN $@" >&2; } +_shunit_error() { echo "shunit2:ERROR $@" >&2; } +_shunit_fatal() { echo "shunit2:FATAL $@" >&2; exit ${SHUNIT_ERROR}; } + +# specific shell checks +if [ -n "${ZSH_VERSION:-}" ]; then + setopt |grep "^shwordsplit$" >/dev/null + if [ $? -ne ${SHUNIT_TRUE} ]; then + _shunit_fatal 'zsh shwordsplit option is required for proper operation' + fi + if [ -z "${SHUNIT_PARENT:-}" ]; then + _shunit_fatal "zsh does not pass \$0 through properly. please declare \ +\"SHUNIT_PARENT=\$0\" before calling shUnit2" + fi +fi + +# +# constants +# + +__SHUNIT_ASSERT_MSG_PREFIX='ASSERT:' +__SHUNIT_MODE_SOURCED='sourced' +__SHUNIT_MODE_STANDALONE='standalone' +__SHUNIT_PARENT=${SHUNIT_PARENT:-$0} + +# set the constants readonly +shunit_constants_=`set |grep '^__SHUNIT_' |cut -d= -f1` +echo "${shunit_constants_}" |grep '^Binary file' >/dev/null && \ + shunit_constants_=`set |grep -a '^__SHUNIT_' |cut -d= -f1` +for shunit_constant_ in ${shunit_constants_}; do + shunit_ro_opts_='' + case ${ZSH_VERSION:-} in + '') ;; # this isn't zsh + [123].*) ;; # early versions (1.x, 2.x, 3.x) + *) shunit_ro_opts_='-g' ;; # all later versions. declare readonly globally + esac + readonly ${shunit_ro_opts_} ${shunit_constant_} +done +unset shunit_constant_ shunit_constants_ shunit_ro_opts_ + +# variables +__shunit_lineno='' # line number of executed test +__shunit_mode=${__SHUNIT_MODE_SOURCED} # operating mode +__shunit_reportGenerated=${SHUNIT_FALSE} # is report generated +__shunit_script='' # filename of unittest script (standalone mode) +__shunit_skip=${SHUNIT_FALSE} # is skipping enabled +__shunit_suite='' # suite of tests to execute + +# counts of tests +__shunit_testSuccess=${SHUNIT_TRUE} +__shunit_testsTotal=0 +__shunit_testsPassed=0 +__shunit_testsFailed=0 + +# counts of asserts +__shunit_assertsTotal=0 +__shunit_assertsPassed=0 +__shunit_assertsFailed=0 +__shunit_assertsSkipped=0 + +# macros +_SHUNIT_LINENO_='eval __shunit_lineno=""; if [ "${1:-}" = "--lineno" ]; then [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi' + +#----------------------------------------------------------------------------- +# assert functions +# + +# Assert that two values are equal to one another. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertEquals() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertEquals() requires two or three arguments; $# given" + _shunit_error "1: ${1:+$1} 2: ${2:+$2} 3: ${3:+$3}${4:+ 4: $4}" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if [ "${shunit_expected_}" = "${shunit_actual_}" ]; then + _shunit_assertPass + else + failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${shunit_return} +} +_ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' + +# Assert that two values are not equal to one another. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotEquals() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotEquals() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if [ "${shunit_expected_}" != "${shunit_actual_}" ]; then + _shunit_assertPass + else + failSame "${shunit_message_}" "$@" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${shunit_return} +} +_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' + +# Assert that a value is null (i.e. an empty string) +# +# Args: +# message: string: failure message [optional] +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNull() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertNull() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertTrue "${shunit_message_}" "[ -z '$1' ]" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' + +# Assert that a value is not null (i.e. a non-empty string) +# +# Args: +# message: string: failure message [optional] +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotNull() +{ + ${_SHUNIT_LINENO_} + if [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null + _shunit_error "assertNotNull() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_actual_=`_shunit_escapeCharactersInString "${1:-}"` + test -n "${shunit_actual_}" + assertTrue "${shunit_message_}" $? + shunit_return=$? + + unset shunit_actual_ shunit_message_ + return ${shunit_return} +} +_ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' + +# Assert that two values are the same (i.e. equal to one another). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' + +# Assert that two values are not the same (i.e. not equal to one another). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_:-}$1" + shift + fi + assertNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' + +# Assert that a value or shell test condition is true. +# +# In shell, a value of 0 is true and a non-zero value is false. Any integer +# value passed can thereby be tested. +# +# Shell supports much more complicated tests though, and a means to support +# them was needed. As such, this function tests that conditions are true or +# false through evaluation rather than just looking for a true or false. +# +# The following test will succeed: +# assertTrue 0 +# assertTrue "[ 34 -gt 23 ]" +# The folloing test will fail with a message: +# assertTrue 123 +# assertTrue "test failed" "[ -r '/non/existant/file' ]" +# +# Args: +# message: string: failure message [optional] +# condition: string: integer value or shell conditional statement +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertTrue() +{ + ${_SHUNIT_LINENO_} + if [ $# -gt 2 ]; then + _shunit_error "assertTrue() takes one two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # see if condition is an integer, i.e. a return value + shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` + shunit_return=${SHUNIT_TRUE} + if [ -z "${shunit_condition_}" ]; then + # null condition + shunit_return=${SHUNIT_FALSE} + elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] + then + # possible return value. treating 0 as true, and non-zero as false. + [ ${shunit_condition_} -ne 0 ] && shunit_return=${SHUNIT_FALSE} + else + # (hopefully) a condition + ( eval ${shunit_condition_} ) >/dev/null 2>&1 + [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE} + fi + + # record the test + if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ shunit_match_ + return ${shunit_return} +} +_ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' + +# Assert that a value or shell test condition is false. +# +# In shell, a value of 0 is true and a non-zero value is false. Any integer +# value passed can thereby be tested. +# +# Shell supports much more complicated tests though, and a means to support +# them was needed. As such, this function tests that conditions are true or +# false through evaluation rather than just looking for a true or false. +# +# The following test will succeed: +# assertFalse 1 +# assertFalse "[ 'apples' = 'oranges' ]" +# The folloing test will fail with a message: +# assertFalse 0 +# assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" +# +# Args: +# message: string: failure message [optional] +# condition: string: integer value or shell conditional statement +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertFalse() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertFalse() quires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # see if condition is an integer, i.e. a return value + shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` + shunit_return=${SHUNIT_TRUE} + if [ -z "${shunit_condition_}" ]; then + # null condition + shunit_return=${SHUNIT_FALSE} + elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] + then + # possible return value. treating 0 as true, and non-zero as false. + [ ${shunit_condition_} -eq 0 ] && shunit_return=${SHUNIT_FALSE} + else + # (hopefully) a condition + ( eval ${shunit_condition_} ) >/dev/null 2>&1 + [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE} + fi + + # record the test + if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ shunit_match_ + return ${shunit_return} +} +_ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# failure functions +# + +# Records a test failure. +# +# Args: +# message: string: failure message [optional] +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +fail() +{ + ${_SHUNIT_LINENO_} + if [ $# -gt 1 ]; then + _shunit_error "fail() requires zero or one arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 1 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + _shunit_assertFail "${shunit_message_}" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +_FAIL_='eval fail --lineno "${LINENO:-}"' + +# Records a test failure, stating two values were not equal. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotEquals() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotEquals() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>" + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${SHUNIT_FALSE} +} +_FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' + +# Records a test failure, stating two values should have been the same. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +_FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' + +# Records a test failure, stating two values were not equal. +# +# This is functionally equivalent to calling failNotEquals(). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotEquals() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + failNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# skipping functions +# + +# Force remaining assert and fail functions to be "skipped". +# +# This function forces the remaining assert and fail functions to be "skipped", +# i.e. they will have no effect. Each function skipped will be recorded so that +# the total of asserts and fails will not be altered. +# +# Args: +# None +startSkipping() +{ + __shunit_skip=${SHUNIT_TRUE} +} + +# Resume the normal recording behavior of assert and fail calls. +# +# Args: +# None +endSkipping() +{ + __shunit_skip=${SHUNIT_FALSE} +} + +# Returns the state of assert and fail call skipping. +# +# Args: +# None +# Returns: +# boolean: (TRUE/FALSE constant) +isSkipping() +{ + return ${__shunit_skip} +} + +#----------------------------------------------------------------------------- +# suite functions +# + +# Stub. This function should contains all unit test calls to be made. +# +# DEPRECATED (as of 2.1.0) +# +# This function can be optionally overridden by the user in their test suite. +# +# If this function exists, it will be called when shunit2 is sourced. If it +# does not exist, shunit2 will search the parent script for all functions +# beginning with the word 'test', and they will be added dynamically to the +# test suite. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Adds a function name to the list of tests schedule for execution. +# +# This function should only be called from within the suite() function. +# +# Args: +# function: string: name of a function to add to current unit test suite +suite_addTest() +{ + shunit_func_=${1:-} + + __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" + __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1` + + unset shunit_func_ +} + +# Stub. This function will be called once before any tests are run. +# +# Common one-time environment preparation tasks shared by all tests can be +# defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Stub. This function will be called once after all tests are finished. +# +# Common one-time environment cleanup tasks shared by all tests can be defined +# here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Stub. This function will be called before each test is run. +# +# Common environment preparation tasks shared by all tests can be defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#setUp() { :; } + +# Note: see _shunit_mktempFunc() for actual implementation +# Stub. This function will be called after each test is run. +# +# Common environment cleanup tasks shared by all tests can be defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +#------------------------------------------------------------------------------ +# internal shUnit2 functions +# + +# Create a temporary directory to store various run-time files in. +# +# This function is a cross-platform temporary directory creation tool. Not all +# OSes have the mktemp function, so one is included here. +# +# Args: +# None +# Outputs: +# string: the temporary directory that was created +_shunit_mktempDir() +{ + # try the standard mktemp function + ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return + + # the standard mktemp didn't work. doing our own. + if [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then + _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 "${_shunit_file_}" +#! /bin/sh +exit ${SHUNIT_TRUE} +EOF + chmod +x "${_shunit_file_}" + done + + unset _shunit_file_ +} + +# Final cleanup function to leave things as we found them. +# +# Besides removing the temporary directory, this function is in charge of the +# final exit code of the unit test. The exit code is based on how the script +# was ended (e.g. normal exit, or via Ctrl-C). +# +# Args: +# name: string: name of the trap called (specified when trap defined) +_shunit_cleanup() +{ + _shunit_name_=$1 + + case ${_shunit_name_} in + EXIT) _shunit_signal_=0 ;; + INT) _shunit_signal_=2 ;; + TERM) _shunit_signal_=15 ;; + *) + _shunit_warn "unrecognized trap value (${_shunit_name_})" + _shunit_signal_=0 + ;; + esac + + # do our work + rm -fr "${__shunit_tmpDir}" + + # exit for all non-EXIT signals + if [ ${_shunit_name_} != 'EXIT' ]; then + _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" + # disable EXIT trap + trap 0 + # add 128 to signal and exit + exit `expr ${_shunit_signal_} + 128` + elif [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ] ; then + _shunit_assertFail 'Unknown failure encountered running a test' + _shunit_generateReport + exit ${SHUNIT_ERROR} + fi + + unset _shunit_name_ _shunit_signal_ +} + +# The actual running of the tests happens here. +# +# Args: +# None +_shunit_execSuite() +{ + for _shunit_test_ in ${__shunit_suite}; do + __shunit_testSuccess=${SHUNIT_TRUE} + + # disable skipping + endSkipping + + # execute the per-test setup function + setUp + + # execute the test + echo "${_shunit_test_}" + eval ${_shunit_test_} + + # execute the per-test tear-down function + tearDown + + # update stats + if [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then + __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1` + else + __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1` + fi + done + + unset _shunit_test_ +} + +# Generates the user friendly report with appropriate OK/FAILED message. +# +# Args: +# None +# Output: +# string: the report of successful and failed tests, as well as totals. +_shunit_generateReport() +{ + _shunit_ok_=${SHUNIT_TRUE} + + # if no exit code was provided one, determine an appropriate one + [ ${__shunit_testsFailed} -gt 0 \ + -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \ + && _shunit_ok_=${SHUNIT_FALSE} + + echo + if [ ${__shunit_testsTotal} -eq 1 ]; then + echo "Ran ${__shunit_testsTotal} test." + else + echo "Ran ${__shunit_testsTotal} tests." + fi + + _shunit_failures_='' + _shunit_skipped_='' + [ ${__shunit_assertsFailed} -gt 0 ] \ + && _shunit_failures_="failures=${__shunit_assertsFailed}" + [ ${__shunit_assertsSkipped} -gt 0 ] \ + && _shunit_skipped_="skipped=${__shunit_assertsSkipped}" + + if [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then + _shunit_msg_='OK' + [ -n "${_shunit_skipped_}" ] \ + && _shunit_msg_="${_shunit_msg_} (${_shunit_skipped_})" + else + _shunit_msg_="FAILED (${_shunit_failures_}" + [ -n "${_shunit_skipped_}" ] \ + && _shunit_msg_="${_shunit_msg_},${_shunit_skipped_}" + _shunit_msg_="${_shunit_msg_})" + fi + + echo + echo ${_shunit_msg_} + __shunit_reportGenerated=${SHUNIT_TRUE} + + unset _shunit_failures_ _shunit_msg_ _shunit_ok_ _shunit_skipped_ +} + +# Test for whether a function should be skipped. +# +# Args: +# None +# Returns: +# boolean: whether the test should be skipped (TRUE/FALSE constant) +_shunit_shouldSkip() +{ + [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE} + _shunit_assertSkip +} + +# Records a successful test. +# +# Args: +# None +_shunit_assertPass() +{ + __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` +} + +# Records a test failure. +# +# Args: +# message: string: failure message to provide user +_shunit_assertFail() +{ + _shunit_msg_=$1 + + __shunit_testSuccess=${SHUNIT_FALSE} + __shunit_assertsFailed=`expr ${__shunit_assertsFailed} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` + echo "${__SHUNIT_ASSERT_MSG_PREFIX}${_shunit_msg_}" + + unset _shunit_msg_ +} + +# Records a skipped test. +# +# Args: +# None +_shunit_assertSkip() +{ + __shunit_assertsSkipped=`expr ${__shunit_assertsSkipped} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` +} + +# Prepare a script filename for sourcing. +# +# Args: +# script: string: path to a script to source +# Returns: +# string: filename prefixed with ./ (if necessary) +_shunit_prepForSourcing() +{ + _shunit_script_=$1 + case "${_shunit_script_}" in + /*|./*) echo "${_shunit_script_}" ;; + *) echo "./${_shunit_script_}" ;; + esac + unset _shunit_script_ +} + +# Escape a character in a string. +# +# Args: +# c: string: unescaped character +# s: string: to escape character in +# Returns: +# string: with escaped character(s) +_shunit_escapeCharInStr() +{ + [ -n "$2" ] || return # no point in doing work on an empty string + + # Note: using shorter variable names to prevent conflicts with + # _shunit_escapeCharactersInString(). + _shunit_c_=$1 + _shunit_s_=$2 + + + # escape the character + echo ''${_shunit_s_}'' |sed 's/\'${_shunit_c_}'/\\\'${_shunit_c_}'/g' + + unset _shunit_c_ _shunit_s_ +} + +# Escape a character in a string. +# +# Args: +# str: string: to escape characters in +# Returns: +# string: with escaped character(s) +_shunit_escapeCharactersInString() +{ + [ -n "$1" ] || return # no point in doing work on an empty string + + _shunit_str_=$1 + + # Note: using longer variable names to prevent conflicts with + # _shunit_escapeCharInStr(). + for _shunit_char_ in '"' '$' "'" '`'; do + _shunit_str_=`_shunit_escapeCharInStr "${_shunit_char_}" "${_shunit_str_}"` + done + + echo "${_shunit_str_}" + unset _shunit_char_ _shunit_str_ +} + +# Extract list of functions to run tests against. +# +# Args: +# script: string: name of script to extract functions from +# Returns: +# string: of function names +_shunit_extractTestFunctions() +{ + _shunit_script_=$1 + + # extract the lines with test function names, strip of anything besides the + # function name, and output everything on a single line. + _shunit_regex_='^[ ]*(function )*test[A-Za-z0-9_]* *\(\)' + egrep "${_shunit_regex_}" "${_shunit_script_}" \ + |sed 's/^[^A-Za-z0-9_]*//;s/^function //;s/\([A-Za-z0-9_]*\).*/\1/g' \ + |xargs + + unset _shunit_regex_ _shunit_script_ +} + +#------------------------------------------------------------------------------ +# main +# + +# determine the operating mode +if [ $# -eq 0 ]; then + __shunit_script=${__SHUNIT_PARENT} + __shunit_mode=${__SHUNIT_MODE_SOURCED} +else + __shunit_script=$1 + [ -r "${__shunit_script}" ] || \ + _shunit_fatal "unable to read from ${__shunit_script}" + __shunit_mode=${__SHUNIT_MODE_STANDALONE} +fi + +# create a temporary storage location +__shunit_tmpDir=`_shunit_mktempDir` + +# provide a public temporary directory for unit test scripts +# TODO(kward): document this +SHUNIT_TMPDIR="${__shunit_tmpDir}/tmp" +mkdir "${SHUNIT_TMPDIR}" + +# setup traps to clean up after ourselves +trap '_shunit_cleanup EXIT' 0 +trap '_shunit_cleanup INT' 2 +trap '_shunit_cleanup TERM' 15 + +# create phantom functions to work around issues with Cygwin +_shunit_mktempFunc +PATH="${__shunit_tmpDir}:${PATH}" + +# make sure phantom functions are executable. this will bite if /tmp (or the +# current $TMPDIR) points to a path on a partition that was mounted with the +# 'noexec' option. the noexec command was created with _shunit_mktempFunc(). +noexec 2>/dev/null || _shunit_fatal \ + 'please declare TMPDIR with path on partition with exec permission' + +# we must manually source the tests in standalone mode +if [ "${__shunit_mode}" = "${__SHUNIT_MODE_STANDALONE}" ]; then + . "`_shunit_prepForSourcing \"${__shunit_script}\"`" +fi + +# execute the oneTimeSetUp function (if it exists) +oneTimeSetUp + +# execute the suite function defined in the parent test script +# deprecated as of 2.1.0 +suite + +# if no suite function was defined, dynamically build a list of functions +if [ -z "${__shunit_suite}" ]; then + shunit_funcs_=`_shunit_extractTestFunctions "${__shunit_script}"` + for shunit_func_ in ${shunit_funcs_}; do + suite_addTest ${shunit_func_} + done +fi +unset shunit_func_ shunit_funcs_ + +# execute the tests +_shunit_execSuite + +# execute the oneTimeTearDown function (if it exists) +oneTimeTearDown + +# generate the report +_shunit_generateReport + +# that's it folks +[ ${__shunit_testsFailed} -eq 0 ] +exit $? diff --git a/extra/coadde/test/packages/pkg-any-a/PKGBUILD b/extra/coadde/test/packages/pkg-any-a/PKGBUILD new file mode 100644 index 0000000..8749a35 --- /dev/null +++ b/extra/coadde/test/packages/pkg-any-a/PKGBUILD @@ -0,0 +1,12 @@ +pkgname=pkg-any-a +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') + +package() { + install -d -m755 ${pkgdir}/usr/share/${pkgname} + echo 'test' > ${pkgdir}/usr/share/${pkgname}/test +} diff --git a/extra/coadde/test/packages/pkg-any-b/PKGBUILD b/extra/coadde/test/packages/pkg-any-b/PKGBUILD new file mode 100644 index 0000000..e6a0498 --- /dev/null +++ b/extra/coadde/test/packages/pkg-any-b/PKGBUILD @@ -0,0 +1,12 @@ +pkgname=pkg-any-b +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') + +package() { + install -d -m755 ${pkgdir}/usr/share/${pkgname} + echo 'test' > ${pkgdir}/usr/share/${pkgname}/test +} diff --git a/extra/coadde/test/packages/pkg-simple-a/Makefile b/extra/coadde/test/packages/pkg-simple-a/Makefile new file mode 120000 index 0000000..50be211 --- /dev/null +++ b/extra/coadde/test/packages/pkg-simple-a/Makefile @@ -0,0 +1 @@ +../../src/Makefile \ No newline at end of file diff --git a/extra/coadde/test/packages/pkg-simple-a/PKGBUILD b/extra/coadde/test/packages/pkg-simple-a/PKGBUILD new file mode 100644 index 0000000..953ecfa --- /dev/null +++ b/extra/coadde/test/packages/pkg-simple-a/PKGBUILD @@ -0,0 +1,22 @@ +pkgname=pkg-simple-a +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +makedepends=('gcc') +source=('Makefile' 'test.c') +md5sums=('c6cb8dcc86253355fed559416d0c8dcf' + '3c1e4279feb678fd9cabaccdb28e40d0') + +build() { + cd ${srcdir} + make +} + +package() { + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname} +} diff --git a/extra/coadde/test/packages/pkg-simple-a/test.c b/extra/coadde/test/packages/pkg-simple-a/test.c new file mode 120000 index 0000000..ed5b5ac --- /dev/null +++ b/extra/coadde/test/packages/pkg-simple-a/test.c @@ -0,0 +1 @@ +../../src/test.c \ No newline at end of file diff --git a/extra/coadde/test/packages/pkg-simple-b/Makefile b/extra/coadde/test/packages/pkg-simple-b/Makefile new file mode 120000 index 0000000..50be211 --- /dev/null +++ b/extra/coadde/test/packages/pkg-simple-b/Makefile @@ -0,0 +1 @@ +../../src/Makefile \ No newline at end of file diff --git a/extra/coadde/test/packages/pkg-simple-b/PKGBUILD b/extra/coadde/test/packages/pkg-simple-b/PKGBUILD new file mode 100644 index 0000000..95ffd09 --- /dev/null +++ b/extra/coadde/test/packages/pkg-simple-b/PKGBUILD @@ -0,0 +1,22 @@ +pkgname=pkg-simple-b +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +makedepends=('gcc') +source=('Makefile' 'test.c') +md5sums=('c6cb8dcc86253355fed559416d0c8dcf' + '3c1e4279feb678fd9cabaccdb28e40d0') + +build() { + cd ${srcdir} + make +} + +package() { + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname} +} diff --git a/extra/coadde/test/packages/pkg-simple-b/test.c b/extra/coadde/test/packages/pkg-simple-b/test.c new file mode 120000 index 0000000..ed5b5ac --- /dev/null +++ b/extra/coadde/test/packages/pkg-simple-b/test.c @@ -0,0 +1 @@ +../../src/test.c \ No newline at end of file diff --git a/extra/coadde/test/packages/pkg-simple-epoch/Makefile b/extra/coadde/test/packages/pkg-simple-epoch/Makefile new file mode 120000 index 0000000..50be211 --- /dev/null +++ b/extra/coadde/test/packages/pkg-simple-epoch/Makefile @@ -0,0 +1 @@ +../../src/Makefile \ No newline at end of file diff --git a/extra/coadde/test/packages/pkg-simple-epoch/PKGBUILD b/extra/coadde/test/packages/pkg-simple-epoch/PKGBUILD new file mode 100644 index 0000000..eebe2bd --- /dev/null +++ b/extra/coadde/test/packages/pkg-simple-epoch/PKGBUILD @@ -0,0 +1,23 @@ +pkgname=pkg-simple-epoch +pkgver=1 +pkgrel=1 +epoch=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +makedepends=('gcc') +source=('Makefile' 'test.c') +md5sums=('c6cb8dcc86253355fed559416d0c8dcf' + '3c1e4279feb678fd9cabaccdb28e40d0') + +build() { + cd ${srcdir} + make +} + +package() { + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname} +} diff --git a/extra/coadde/test/packages/pkg-simple-epoch/test.c b/extra/coadde/test/packages/pkg-simple-epoch/test.c new file mode 120000 index 0000000..ed5b5ac --- /dev/null +++ b/extra/coadde/test/packages/pkg-simple-epoch/test.c @@ -0,0 +1 @@ +../../src/test.c \ No newline at end of file diff --git a/extra/coadde/test/packages/pkg-split-a/Makefile b/extra/coadde/test/packages/pkg-split-a/Makefile new file mode 120000 index 0000000..50be211 --- /dev/null +++ b/extra/coadde/test/packages/pkg-split-a/Makefile @@ -0,0 +1 @@ +../../src/Makefile \ No newline at end of file diff --git a/extra/coadde/test/packages/pkg-split-a/PKGBUILD b/extra/coadde/test/packages/pkg-split-a/PKGBUILD new file mode 100644 index 0000000..e941976 --- /dev/null +++ b/extra/coadde/test/packages/pkg-split-a/PKGBUILD @@ -0,0 +1,28 @@ +pkgbase=pkg-split-a +pkgname=('pkg-split-a1' 'pkg-split-a2') +pkgver=1 +pkgrel=1 +pkgdesc="A split package called ${pkgbase}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +makedepends=('gcc') +source=('Makefile' 'test.c') +md5sums=('c6cb8dcc86253355fed559416d0c8dcf' + '3c1e4279feb678fd9cabaccdb28e40d0') + +build() { + cd ${srcdir} + make +} + +package_pkg-split-a1() { + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} +} + +package_pkg-split-a2() { + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} +} diff --git a/extra/coadde/test/packages/pkg-split-a/test.c b/extra/coadde/test/packages/pkg-split-a/test.c new file mode 120000 index 0000000..ed5b5ac --- /dev/null +++ b/extra/coadde/test/packages/pkg-split-a/test.c @@ -0,0 +1 @@ +../../src/test.c \ No newline at end of file diff --git a/extra/coadde/test/packages/pkg-split-b/Makefile b/extra/coadde/test/packages/pkg-split-b/Makefile new file mode 120000 index 0000000..50be211 --- /dev/null +++ b/extra/coadde/test/packages/pkg-split-b/Makefile @@ -0,0 +1 @@ +../../src/Makefile \ No newline at end of file diff --git a/extra/coadde/test/packages/pkg-split-b/PKGBUILD b/extra/coadde/test/packages/pkg-split-b/PKGBUILD new file mode 100644 index 0000000..6ddbc45 --- /dev/null +++ b/extra/coadde/test/packages/pkg-split-b/PKGBUILD @@ -0,0 +1,29 @@ +pkgbase=pkg-split-b +pkgname=('pkg-split-b1' 'pkg-split-b2') +pkgver=1 +pkgrel=1 +pkgdesc="A split package called ${pkgbase}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') + +depends=('glibc') +makedepends=('gcc') +source=('Makefile' 'test.c') +md5sums=('c6cb8dcc86253355fed559416d0c8dcf' + '3c1e4279feb678fd9cabaccdb28e40d0') + +build() { + cd ${srcdir} + make +} + +package_pkg-split-b1() { + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} +} + +package_pkg-split-b2() { + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} +} diff --git a/extra/coadde/test/packages/pkg-split-b/test.c b/extra/coadde/test/packages/pkg-split-b/test.c new file mode 120000 index 0000000..ed5b5ac --- /dev/null +++ b/extra/coadde/test/packages/pkg-split-b/test.c @@ -0,0 +1 @@ +../../src/test.c \ No newline at end of file diff --git a/extra/coadde/test/rsync_output_sample b/extra/coadde/test/rsync_output_sample new file mode 100644 index 0000000..72d9cd0 --- /dev/null +++ b/extra/coadde/test/rsync_output_sample @@ -0,0 +1,14 @@ +dr-xr-sr-x 4096 2010/09/11 11:37:10 . +-rw-r--r-- 11 2011/02/08 00:00:01 lastsync +drwxrwxr-x 15 2010/09/11 11:28:50 community-staging +drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os +drwxrwxr-x 8192 2011/02/07 17:00:01 community-staging/os/i686 +lrwxrwxrwx 52 2010/12/23 16:51:01 community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz -> ../../../pool/community/alex-2.3.4-1-i686.pkg.tar.xz +lrwxrwxrwx 27 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db -> community-staging.db.tar.gz +-rw-rw-r-- 2237 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db.tar.gz +-rw-rw-r-- 3209 2011/02/07 14:00:13 community-staging/os/i686/community-staging.db.tar.gz.old +drwxrwxr-x 15 2009/07/22 15:07:56 community +drwxrwxr-x 40 2009/08/04 15:57:42 community/os +drwxrwsr-x 36864 2011/02/03 05:00:01 community/os/any +-rw-rw-r-- 303336 2010/07/16 10:06:28 community/os/any/any2dvd-0.34-4-any.pkg.tar.xz +-rw-rw-r-- 221664 2010/03/28 15:55:48 community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz diff --git a/extra/coadde/test/runTest b/extra/coadde/test/runTest new file mode 100755 index 0000000..b8713d8 --- /dev/null +++ b/extra/coadde/test/runTest @@ -0,0 +1,15 @@ +#!/bin/bash + +. "$(dirname ${BASH_SOURCE[0]})/lib/common.inc" + +for t in "$(dirname ${BASH_SOURCE[0]})/test.d/"*.sh; do + l=$(basename ${t} .sh) + if [ -x ${t} ]; then + msg "Running test '${l}'" + ${t} + [ $? -ne 0 ] && die "Test '${l}' failed" + echo -e "\n\n\n" + else + warning "Skipping test ${l}" + fi +done diff --git a/extra/coadde/test/src/Makefile b/extra/coadde/test/src/Makefile new file mode 100644 index 0000000..105b730 --- /dev/null +++ b/extra/coadde/test/src/Makefile @@ -0,0 +1,5 @@ +all: + gcc $(CFLAGS) -o test test.c + +install: + install -D -m755 test $(DESTDIR)/usr/bin/$(DESTBIN) diff --git a/extra/coadde/test/src/test.c b/extra/coadde/test/src/test.c new file mode 100644 index 0000000..a661689 --- /dev/null +++ b/extra/coadde/test/src/test.c @@ -0,0 +1,7 @@ +#include +#include + +int main(void) { + printf("Arch is the best!\n"); + return EXIT_SUCCESS; +} diff --git a/extra/coadde/test/test.d/create-filelists.sh b/extra/coadde/test/test.d/create-filelists.sh new file mode 100755 index 0000000..49734c4 --- /dev/null +++ b/extra/coadde/test/test.d/create-filelists.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testCreateSimpleFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then + fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}" + fi + done + done +} + +testCreateAnyFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then + fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}" + fi + done + done +} + +testCreateSplitFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local pkgname + local pkgnames + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + for pkgbase in ${pkgs[@]}; do + pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) + for pkgname in ${pkgnames[@]}; do + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then + fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}" + fi + done + done + done +} + + +testCleanupFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + for arch in ${arches[@]}; do + ../db-remove extra ${arch} pkg-simple-a + done + + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then + fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}" + fi + if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-a"; then + fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}" + fi + done + +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/db-move.sh b/extra/coadde/test/test.d/db-move.sh new file mode 100755 index 0000000..9d7c1f6 --- /dev/null +++ b/extra/coadde/test/test.d/db-move.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testMoveSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update + + ../db-move testing extra pkg-simple-a + + for arch in ${arches[@]}; do + checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + + checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} + done +} + +testMoveMultiplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update + + ../db-move testing extra pkg-simple-a pkg-simple-b + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +testMoveEpochPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update + + ../db-move testing extra pkg-simple-epoch + + for arch in ${arches[@]}; do + checkPackage extra pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} + done +} + +testMoveAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} any + done + + ../db-update + ../db-move testing extra pkg-any-a + + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz + checkRemovedAnyPackage testing pkg-any-a + checkAnyPackage testing pkg-any-b-1-1-any.pkg.tar.xz +} + +testMoveSplitPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update + ../db-move testing extra pkg-split-a + + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/pkg-split-a"/*-${arch}${PKGEXT}; do + checkPackage extra $(basename ${pkg}) ${arch} + done + done + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/pkg-split-b"/*-${arch}${PKGEXT}; do + checkPackage testing $(basename ${pkg}) ${arch} + done + done + + checkRemovedAnyPackage testing pkg-split-a +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/db-remove.sh b/extra/coadde/test/test.d/db-remove.sh new file mode 100755 index 0000000..416e693 --- /dev/null +++ b/extra/coadde/test/test.d/db-remove.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testRemovePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + ../db-remove extra ${arch} ${pkgbase} + done + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkRemovedPackage extra ${pkgbase} ${arch} + done + done +} + +testRemoveMultiplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for arch in ${arches[@]}; do + ../db-remove extra ${arch} ${pkgs[@]} + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkRemovedPackage extra ${pkgbase} ${arch} + done + done +} + +testRemoveAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + ../db-remove extra any ${pkgbase} + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedAnyPackage extra ${pkgbase} + done +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/db-repo-add.sh b/extra/coadde/test/test.d/db-repo-add.sh new file mode 100755 index 0000000..8603104 --- /dev/null +++ b/extra/coadde/test/test.d/db-repo-add.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testAddSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" + touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" + ../db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz + done + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +testAddMultiplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for arch in ${arches[@]}; do + add_pkgs=() + for pkgbase in ${pkgs[@]}; do + cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" + touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" + add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz + done + ../db-repo-add extra ${arch} ${add_pkgs[@]} + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/db-repo-remove.sh b/extra/coadde/test/test.d/db-repo-remove.sh new file mode 100755 index 0000000..315d63d --- /dev/null +++ b/extra/coadde/test/test.d/db-repo-remove.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testRemovePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + ../db-repo-remove extra ${arch} ${pkgbase} + done + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkRemovedPackageDB extra ${pkgbase} ${arch} + done + done +} + +testRemoveMultiplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for arch in ${arches[@]}; do + ../db-repo-remove extra ${arch} ${pkgs[@]} + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkRemovedPackageDB extra ${pkgbase} ${arch} + done + done +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/db-update.sh b/extra/coadde/test/test.d/db-update.sh new file mode 100755 index 0000000..e38c328 --- /dev/null +++ b/extra/coadde/test/test.d/db-update.sh @@ -0,0 +1,165 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testAddSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +testAddSingleSimplePackage() { + releasePackage extra 'pkg-simple-a' 'i686' + ../db-update + checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' +} + +testAddSingleEpochPackage() { + releasePackage extra 'pkg-simple-epoch' 'i686' + ../db-update + checkPackage extra 'pkg-simple-epoch-1:1-1-i686.pkg.tar.xz' 'i686' +} + +testAddAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz + done +} + +testAddSplitPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do + checkPackage extra $(basename ${pkg}) ${arch} + done + done + done +} + +testUpdateAnyPackage() { + releasePackage extra pkg-any-a any + ../db-update + + pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo extra-i686-build >/dev/null 2>&1 + mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" + popd >/dev/null + + releasePackage extra pkg-any-a any + ../db-update + + checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any + + rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" +} + +testUpdateAnyPackageToDifferentRepositoriesAtOnce() { + releasePackage extra pkg-any-a any + + pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo extra-i686-build >/dev/null 2>&1 + mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" + popd >/dev/null + + releasePackage testing pkg-any-a any + + ../db-update + + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any + + rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" +} + +testUpdateSameAnyPackageToSameRepository() { + releasePackage extra pkg-any-a any + ../db-update + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + + releasePackage extra pkg-any-a any + ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to the same repository should fail'; return 1) +} + +testUpdateSameAnyPackageToDifferentRepositories() { + releasePackage extra pkg-any-a any + ../db-update + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + + releasePackage testing pkg-any-a any + ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) + + local arch + for arch in i686 x86_64; do + ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" + done +} + + +testAddIncompleteSplitPackage() { + local arches=('i686' 'x86_64') + local repo='extra' + local pkgbase='pkg-split-a' + local arch + + for arch in ${arches[@]}; do + releasePackage ${repo} ${pkgbase} ${arch} + done + + # remove a split package to make db-update fail + rm "${STAGING}"/extra/${pkgbase}1-* + + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a split package is missing!" + + for arch in ${arches[@]}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + done +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/ftpdir-cleanup.sh b/extra/coadde/test/test.d/ftpdir-cleanup.sh new file mode 100755 index 0000000..20026b4 --- /dev/null +++ b/extra/coadde/test/test.d/ftpdir-cleanup.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testCleanupSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for arch in ${arches[@]}; do + ../db-remove extra ${arch} pkg-simple-a + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + for arch in ${arches[@]}; do + local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" + checkRemovedPackage extra 'pkg-simple-a' ${arch} + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" + + local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" + checkPackage extra ${pkg2} ${arch} + done +} + +testCleanupEpochPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for arch in ${arches[@]}; do + ../db-remove extra ${arch} pkg-simple-epoch + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + for arch in ${arches[@]}; do + local pkg1="pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz" + checkRemovedPackage extra 'pkg-simple-epoch' ${arch} + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" + done +} + +testCleanupAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch='any' + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update + ../db-remove extra any pkg-any-a + ../cron-jobs/ftpdir-cleanup >/dev/null + + local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + checkRemovedAnyPackage extra 'pkg-any-a' + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" + + local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" + checkAnyPackage extra ${pkg2} +} + +testCleanupSplitPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for arch in ${arches[@]}; do + ../db-remove extra ${arch} ${pkgs[0]} + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}${PKGEXT}; do + checkRemovedPackage extra ${pkgs[0]} ${arch} + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg}" ] && fail "${PKGPOOL}/${pkg} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" + done + + for pkg in "${pkgdir}/${pkgs[1]}"/*-${arch}${PKGEXT}; do + checkPackage extra $(basename ${pkg}) ${arch} + done + done +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/packages.sh b/extra/coadde/test/test.d/packages.sh new file mode 100755 index 0000000..488cb15 --- /dev/null +++ b/extra/coadde/test/test.d/packages.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testPackages() { + # TODO: namcap -r sodepends fails with i686 packages + find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e sodepends,pkgnameindesc {} + || fail 'namcap failed' +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/pool-transition.sh b/extra/coadde/test/test.d/pool-transition.sh new file mode 100755 index 0000000..5873f00 --- /dev/null +++ b/extra/coadde/test/test.d/pool-transition.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testMovePackagesWithoutPool() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') + local pkgbase + local arch + local pkg + local old + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update + + # transform two packages to old style layout + for arch in ${arches[@]}; do + for old in 0 2; do + for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}${PKGEXT}; do + pkg=$(basename $pkg) + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" + done + done + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + ../db-move testing extra ${pkgs[@]} + + ../cron-jobs/ftpdir-cleanup >/dev/null + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do + checkPackage extra $(basename ${pkg}) ${arch} + done + checkRemovedPackage testing ${pkgbase} ${arch} + done + done +} + +testUpdateAnyPackageWithoutPool() { + local pkgname='pkg-any-a' + local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + local pkg2='pkg-any-a-1-2-any.pkg.tar.xz' + local arch + + + releasePackage extra pkg-any-a any + ../db-update + # transform two packages to old style layout + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" "${FTP_BASE}/extra/os/any" + for arch in i686 x86_64; do + ln -sf "../any/${pkg1}" "${FTP_BASE}/extra/os/${arch}" + done + + pushd "${TMP}/svn-packages-copy/${pkgname}/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo extra-i686-build >/dev/null 2>&1 + mv "${pkg2}" "${pkgdir}/${pkgname}/" + popd >/dev/null + + releasePackage extra ${pkgname} any + ../db-update + rm -f "${pkgdir}/${pkgname}/${pkg2}" + + ../cron-jobs/ftpdir-cleanup >/dev/null + + checkAnyPackage extra "${pkg2}" + + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" + for arch in any i686 x86_64; do + [ -f "${FTP_BASE}/extra/os/${arch}/${pkg1}" ] && fail "extra/os/${arch}/${pkg1} found" + done +} + +testMoveAnyPackagesWithoutPool() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch + local pkg + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} any + done + + ../db-update + + # transform a package to old style layout + for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do + pkg=$(basename $pkg) + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" + for arch in i686 x86_64; do + ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" + done + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + ../db-move testing extra ${pkgs[@]} + + ../cron-jobs/ftpdir-cleanup >/dev/null + + for pkgbase in ${pkgs[@]}; do + for pkg in "${pkgdir}/${pkgbase}"/*-any${PKGEXT}; do + checkAnyPackage extra $(basename ${pkg}) + done + checkRemovedAnyPackage testing ${pkgbase} + done + + for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do + pkg=$(basename $pkg) + for arch in any i686 x86_64; do + [ -f "${FTP_BASE}/testing/os/${arch}/${pkg}" ] && fail "testing/os/${arch}/${pkg} found" + done + done +} + +testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { + local pkg + local arch + + releasePackage extra pkg-any-a any + ../db-update + + # transform a package to old style layout + for pkg in "${pkgdir}/pkg-any-a"/*-any${PKGEXT}; do + pkg=$(basename $pkg) + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" + for arch in i686 x86_64; do + ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" + done + done + + releasePackage testing pkg-any-a any + ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) + + for arch in i686 x86_64; do + ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q pkg-any-a) \ + && fail "pkg-any-a should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" + done +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/signed-packages.sh b/extra/coadde/test/test.d/signed-packages.sh new file mode 100755 index 0000000..5d6f4ff --- /dev/null +++ b/extra/coadde/test/test.d/signed-packages.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testAddUnsignedPackage() { + releasePackage extra 'pkg-simple-a' 'i686' + # remove any signature + rm "${STAGING}"/extra/*.sig + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/sourceballs.sh b/extra/coadde/test/test.d/sourceballs.sh new file mode 100755 index 0000000..fdcf08c --- /dev/null +++ b/extra/coadde/test/test.d/sourceballs.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testSourceballs() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + ../cron-jobs/sourceballs + for pkgbase in ${pkgs[@]}; do + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + done +} + +testAnySourceballs() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + ../db-update + + ../cron-jobs/sourceballs + for pkgbase in ${pkgs[@]}; do + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + done +} + +testSplitSourceballs() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + ../cron-jobs/sourceballs + for pkgbase in ${pkgs[@]}; do + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + done +} + +testSourceballsCleanup() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + ../cron-jobs/sourceballs + + for arch in ${arches[@]}; do + ../db-remove extra ${arch} pkg-simple-a + done + + ../cron-jobs/sourceballs + [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" + [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test.d/testing2x.sh b/extra/coadde/test/test.d/testing2x.sh new file mode 100755 index 0000000..eda6cd6 --- /dev/null +++ b/extra/coadde/test/test.d/testing2x.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testTesting2xAnyPackage() { + releasePackage core pkg-any-a any + ../db-update + + pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + sudo extra-i686-build >/dev/null 2>&1 + mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" + popd >/dev/null + + releasePackage testing pkg-any-a any + ../db-update + rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" + + ../testing2x pkg-any-a + + checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz any + checkRemovedAnyPackage testing pkg-any-a +} + +. "${curdir}/../lib/shunit2" diff --git a/extra/coadde/test/test_filter.py b/extra/coadde/test/test_filter.py new file mode 100644 index 0000000..d8006f9 --- /dev/null +++ b/extra/coadde/test/test_filter.py @@ -0,0 +1,196 @@ +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class pkginfo_from_file_KnownValues(unittest.TestCase): + # (filename, name, version, release, arch) + # filename is location + known=( + ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), + ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), + ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), + ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), + ) + + def generate_results(self, example_tuple, attr): + location, name, version, release, arch = example_tuple + return pkginfo_from_filename(location)[attr], locals()[attr] + + def testReturnPackageObject(self): + for i in self.known: + location, name, version, release, arch = i + self.assertIsInstance(pkginfo_from_filename(location),Package) + + def testNames(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +class pkginfo_from_file_BadInput(unittest.TestCase): + bad=("community-testing/os/i686/community-testing.db", + "community-testing/os/i686/community-testing.db.tar.gz", + "community-testing/os/i686/community-testing.db.tar.gz.old", + "community-testing/os/i686/community-testing.files", + "community-testing/os/i686/community-testing.files.tar.gz", + "community-testing/os/x86_64") + + def testBadInput(self): + for i in self.bad: + self.assertRaises(NonValidFile,pkginfo_from_filename,i) + +class pkginfoFromRsyncOutput(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + try: + output_file = open("rsync_output_sample") + rsync_out= output_file.read() + output_file.close() + except IOError: print("There is no rsync_output_sample file") + + pkglist = pkginfo_from_rsync_output(rsync_out) + + def testOutputArePackages(self): + if not self.pkglist: + self.fail("not pkglist:" + str(self.pkglist)) + for pkg in self.pkglist: + self.assertIsInstance(pkg,Package) + + def testPackageInfo(self): + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,self.example_package_list) + +class generateRsyncBlacklist(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=rsyncBlacklist_from_blacklist(self.example_package_list, + listado("blacklist_sample"), + False) + b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] + self.assertEqual(a,b) + +class pkginfo_from_descKnownValues(unittest.TestCase): + pkgsample=Package() + pkgsample.package_info={"name" : "binutils", + "version" : "2.21", + "release" : "4", + "arch" : "x86_64", + "license" : "GPL", + "location": "binutils-2.21-4-x86_64.pkg.tar.xz", + "depends" : False,} + fsock=open("desc") + pkggen=pkginfo_from_desc(fsock.read()) + fsock.close() + def testPkginfoFromDesc(self): + if self.pkggen is None: + self.fail("return value is None") + self.assertEqual(self.pkgsample,self.pkggen) + +class pkginfo_from_db(unittest.TestCase): + archdb = os.path.join("./workdir") + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "acl", + "version" : "2.2.49", + "release" : "2", + "arch" : "x86_64", + "license" : ("LGPL",), + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz", + "depends" : ("attr>=2.4.41"),} + example_package_list[1].package_info={ "name" : "glibc", + "version" : "2.13", + "release" : "4", + "arch" : "x86_64", + "license" : ("GPL","LGPL"), + "location": "glibc-2.13-4-x86_64.pkg.tar.xz", + "depends" : ("linux-api-headers>=2.6.37","tzdata",),} + example_package_list[2].package_info={ "name" : "", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "", + "depends" : False,} + + +if __name__ == "__main__": + unittest.main() + diff --git a/extra/coadde/xbslibre b/extra/coadde/xbslibre new file mode 100755 index 0000000..e40865a --- /dev/null +++ b/extra/coadde/xbslibre @@ -0,0 +1,93 @@ +#!/bin/bash + +set -e + +# Steps +# * Sync bs derivation (abs) +# * Download blacklist.txt +# * Sync bs (abslibre) from bs derivation (abs) excluding from blacklist +# * Create repo.abs.tar.gz tarballs + +source "$(dirname "$(readlink -e "$0")")/config_platforms" + +for 'platform' in "${DERIVATIVE_PLATFORMS[@]}"; do + + source "$(dirname "$(readlink -e "$0")")/config_${platform}" + + function sync_bs_derivation() { + for 'ARCH' in 'any' "${ARCHES[@]}"; do + rsync "${SYNC_ARGS}" "${SYNC_DERIVATION_SERVER}::${BS_DERIVATION_NAME}/${ARCH}" "${BS_DERIVATION_DIR}/${ARCH}" || return $? + done + + # fix some permissions + find "${BS_DERIVATION_DIR}" -type d -print0 | xargs -0 chmod 755 + find "${BS_DERIVATION_DIR}" -type f -print0 | xargs -0 chmod 644 + } + + function get_blacklist() { + printf ":: Updating blacklist...\t" + cat "${BLACKLIST_FILE}" | cut -d':' -f1 | sort -u | \ + sed "s/^/**\//" > "${BLACKLIST_TMP}" || { + printf "[FAILED]\n" + return 1 + } + + # Prevent using an empty blacklist + [ $(wc -l "${BLACKLIST_TMP}" | cut -d " " -f1) -eq 0 ] && return 1 + + printf "[OK]\n" + } + + function sync_bs() { + + # Clone BS git repo + if [ -d "${BS_GIT_TMP}"/.git ]; then + pushd "${BS_GIT_TMP}" >/dev/null 2>&1 + git pull + popd >/dev/null 2>&1 + else + git clone "${BS_GIT}" "${BS_GIT_TMP}" + fi + + # Sync from BS_DERIVATION and then sync from BS + printf ":: Syncing ${BS_MAIN_NAME}...\t" + (rsync "${SYNC_ARGS}" --delete-excluded \ + --exclude-from="${BLACKLIST_TMP}" \ + "${BS_DERIVATION_DIR}" \ + "${BS_MAIN_DIR}" \ + && + for 'ARCH' in "${ARCHES[@]}"; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ "${BS_GIT_TMP}" "${BS_MAIN_DIR}/${ARCH}/"; done) || { + printf "[FAILED]\n" + return 1 + } + + # fix some permissions + find "${BS_MAIN_DIR}" -type d -print0 | xargs -0 chmod 755 + find "${BS_MAIN_DIR}" -type f -print0 | xargs -0 chmod 644 + + printf "[OK]\n" + } + + # Create .abs.tar.gz tarballs + create_tarballs() { + for 'repo' in "${BS_MAIN_DIR}/${ARCHES[@]}/"*; do + baserepo="${repo##*/}" + arch="$(basename $(dirname ${repo}))" + + # Remove the old one + mkdir -p "${REPO_DIR}/${baserepo}/os/${arch}" + rm -fv "${REPO_DIR}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" + # Create a new one joining arch and any + # Remove the first part of the path (it could be $repo but any isn't hit) + bsdtar -czf "${REPO_DIR}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" \ + -s ":${BS_MAIN_DIR}/[a-z0-9_]\+/[a-z]\+::" \ + "${repo}"/* "${BS_MAIN_DIR}/any/${baserepo}"/* + + done + } + + sync_bs_derivation + get_blacklist + sync_bs + create_tarballs +done diff --git a/list_nonfree_in_db.py b/list_nonfree_in_db.py deleted file mode 100755 index a486fa5..0000000 --- a/list_nonfree_in_db.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python2 -#-*- encoding: utf-8 -*- -from filter import * -import argparse - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="nonfree_in_db", - description="Cleans nonfree files on repo",) - - parser.add_argument("-k", "--blacklist-file", type=str, - help="File containing blacklisted names", - required=True,) - - parser.add_argument("-b", "--database", type=str, - help="dabatase to clean", - required=True,) - - args=parser.parse_args() - - if not (args.blacklist_file and args.database): - parser.print_help() - exit(1) - - blacklist=listado(args.blacklist_file) - pkgs=get_pkginfo_from_db(args.database) - - print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) diff --git a/make_individual_torrent b/make_individual_torrent deleted file mode 100755 index 0a7e778..0000000 --- a/make_individual_torrent +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# Copyright (C) 2014 Joseph Graham -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# This script is called by `make_repo_torrents' to make a torrent. It -# depends on `mktorrent'. It takes the following args: -# $1 - path of package -# $2 - public location - -# Comma seperated list of trackers, no spaces -# t67.eu is run by Xylon, hackcoop by fauno & friends -trackers='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' - -# This mirror is put as a webseed. Which mirror we use for a webseed -# doesn't really matter since it's re-written on the client machine by -# pacman2pacman so it won't normally be used anyway. -seed_url='http://repo.parabolagnulinux.org/' - -if [[ -z "${1}" ]] -then - echo "Error. First arg must be the path of the package." - echo 1 -fi - -if [[ -z "${2}" ]] -then - echo "Error. Second arg must be the public location." - echo 1 -fi - -pkg="${1}" -public_location="${2}" - -pkg_name="${pkg##*/}" - -# URL of the actual package for the webseed -webseed="${seed_url}${pkg#${public_location}}" - -mktorrent -a "${trackers}" "${pkg}" -w "${webseed}" >/dev/null || -echo "Error making torrent for \"${pkg}\"" diff --git a/mkrepo b/mkrepo deleted file mode 100755 index b11dc0b..0000000 --- a/mkrepo +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Author: Nicolás Reynolds -# License: GPLv3+ -# Description: A script to quickly create new [repos] - -source "$(dirname "$(readlink -e "$0")")/config" - -for repo in "$@"; do - echo ":: Creating [$repo]" - for arch in "${ARCHES[@]}"; do - mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" - done -done - -echo ":: All done. Add the repo to the ParabolaWeb admin page." diff --git a/share b/share new file mode 120000 index 0000000..bf9c6fb --- /dev/null +++ b/share @@ -0,0 +1 @@ +src/share \ No newline at end of file diff --git a/src/etc/dbscripts.cfg b/src/etc/dbscripts.cfg new file mode 100644 index 0000000..4ef4353 --- /dev/null +++ b/src/etc/dbscripts.cfg @@ -0,0 +1,6 @@ +#!/hint/bash + +# Parabola Platforms +MAIN_PLATFORMS=('gnu+linux') # 'gnu+hurd' +DERIVATIVE_PLATFORMS=('abslibre_gnu+linux' 'abslibre_gnu+linux_arm') # 'abslibre_gnu+hurd' +PLATFORMS=("${MAIN_PLATFORMS[@]}" "${DERIVATIVE_PLATFORMS[@]}") diff --git a/src/etc/dbscripts.d/gnu+hurd_add.cfg b/src/etc/dbscripts.d/gnu+hurd_add.cfg new file mode 100644 index 0000000..ae05228 --- /dev/null +++ b/src/etc/dbscripts.d/gnu+hurd_add.cfg @@ -0,0 +1,67 @@ +#!/hint/bash + +# Parabola Platform +PLATFORM_NAME='GNU/Hurd' +PLATFORM='gnu+hurd' + +# Multilib Support Archictectures +MULTILIB_ARCHES=() + +# Archictectures +ARCHES=('i686' "${MULTILIB_ARCHES[@]}") + +# Main Repositories from Arch GNU/Hurd +MAIN_REPOS=('core' 'extra' 'testing' 'staging-core' 'staging-extra') +# Libre Repositories from Parabola GNU/Hurd +LIBRE_MAIN_REPOS=('libre' 'libre-testing') + +# Multilib Repositories from Arch GNU/Hurd +MULTILIB_REPOS=('multilib' 'multilib-testing') +# Libre Multilib Repositories from Parabola GNU/Hurd +LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') + +# Extra Repositories +EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') + +# Community Repositories +COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') + +# Platform Repositories +PLATFORM_REPOS=() + +# Remote Repositories +PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") +PKG_MULTILIB_REPOS=("${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}") + +# Directories where packages are shared between repos +# *relative to REPO_DIR* +PKG_POOL="${PKG_POOL_DIR}/abslibre_${PLATFORM}" + +# Directories where sources are stored +SRC_POOL="${SRC_POOL_DIR}/abslibre_${PLATFORM}" + +# Build System and Repository scripts +BS_MAIN_NAME="abslibre_${PLATFORM}" +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_DERIVATION_NAME="abs_${PLATFORM}" +BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" +BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" +# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" +BLACKLIST_NAME="blacklist_${PLATFORM}.txt" +BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" +BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" +SYNC_DERIVATION_SERVER='rsync.archhurd.org' + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/abslibre_${PLATFORM}" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/src/etc/dbscripts.d/gnu+hurd_native.cfg b/src/etc/dbscripts.d/gnu+hurd_native.cfg new file mode 100644 index 0000000..61fefc0 --- /dev/null +++ b/src/etc/dbscripts.d/gnu+hurd_native.cfg @@ -0,0 +1,57 @@ +#!/hint/bash + +# Parabola Platform +PLATFORM_NAME='GNU/Hurd' +PLATFORM='gnu+hurd' + +# Multilib Support Archictectures +MULTILIB_ARCHES=() + +# Archictectures +ARCHES=('i686' "${MULTILIB_ARCHES[@]}") + +# Main Repositories +MAIN_REPOS=('core' 'extra' 'testing') + +# Multilib Repositories +MULTILIB_REPOS=('multilib' 'multilib-testing') + +# Extra Repositories +EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') + +# Community Repositories +COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') + +# Platform Repositories +PLATFORM_REPOS=() + +# Remote Repositories +PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") +PKG_MULTILIB_REPOS=("${MULTILIB_REPOS[@]}") + +# Directories where packages are shared between repos +# *relative to REPO_DIR* +PKG_POOL="${PKG_POOL_DIR}/${PLATFORM}" + +# Directories where sources are stored +SRC_POOL="${SRC_POOL_DIR}/${PLATFORM}" + +# Build System and Repository scripts +BS_MAIN_NAME="pbs_${PLATFORM}" +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_GIT="${GIT_BASE}/${BS_MAIN_NAME}/pmr.git" +# BS_GIT='http://projects.parabola.gnu/${BS_MAIN_NAME}/pmr.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/${PLATFORM}" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/src/etc/dbscripts.d/gnu+linux_add.cfg b/src/etc/dbscripts.d/gnu+linux_add.cfg new file mode 100644 index 0000000..b6fe319 --- /dev/null +++ b/src/etc/dbscripts.d/gnu+linux_add.cfg @@ -0,0 +1,67 @@ +#!/hint/bash + +# Parabola Platform +PLATFORM_NAME='GNU/Linux-libre' +PLATFORM='gnu+linux' + +# Multilib Support Archictectures +MULTILIB_ARCHES=('x86_64') + +# Archictectures +ARCHES=('i686' "${MULTILIB_ARCHES[@]}") + +# Main Repositories from Arch GNU/Linux +MAIN_REPOS=('core' 'extra' 'testing' 'community' 'community-testing') +# Libre Repositories from Parabola GNU/Linux-libre +LIBRE_MAIN_REPOS=('libre' 'libre-testing') + +# Multilib Repositories from Arch GNU/Linux +MULTILIB_REPOS=('multilib' 'multilib-testing') +# Libre Multilib Repositories from Parabola GNU/Linux-libre +LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') + +# Extra Repositories +EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') + +# Community Repositories +COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') + +# Platform Repositories +PLATFORM_REPOS=('nonsystemd' 'nonsystemd-testing') + +# Remote Repositories +PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") +PKG_MULTILIB_REPOS=("${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}") + +# Directories where packages are shared between repos +# *relative to REPO_DIR* +PKG_POOL="${PKG_POOL_DIR}/abslibre_${PLATFORM}" + +# Directories where sources are stored +SRC_POOL="${SRC_POOL_DIR}/abslibre_${PLATFORM}" + +# Build System and Repository scripts +BS_MAIN_NAME="abslibre_${PLATFORM}" +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_DERIVATION_NAME="abs_${PLATFORM}" +BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" +BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" +# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" +BLACKLIST_NAME="blacklist_${PLATFORM}.txt" +BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" +BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" +SYNC_DERIVATION_SERVER='rsync.archlinux.org' + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/abslibre_${PLATFORM}" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/src/etc/dbscripts.d/gnu+linux_add_arm.cfg b/src/etc/dbscripts.d/gnu+linux_add_arm.cfg new file mode 100644 index 0000000..8eb226b --- /dev/null +++ b/src/etc/dbscripts.d/gnu+linux_add_arm.cfg @@ -0,0 +1,67 @@ +#!/hint/bash + +# Parabola Platform +PLATFORM_NAME='GNU/Linux-libre' +PLATFORM='gnu+linux' + +# Multilib Support Archictectures +MULTILIB_ARCHES=() + +# Archictectures +ARCHES=('arm' 'armv6h' 'armv7h' 'aarch64' "${MULTILIB_ARCHES[@]}") + +# Main Repositories from Arch GNU/Linux ARM +MAIN_REPOS=('core' 'extra' 'community' 'aur' 'alarm') +# Libre Repositories from Parabola GNU/Linux-libre +LIBRE_MAIN_REPOS=('libre' 'libre-testing') + +# Multilib Repositories from Arch GNU/Linux ARM +MULTILIB_REPOS=('multilib') +# Libre Multilib Repositories from Parabola GNU/Linux-libre +LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') + +# Extra Repositories +EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') + +# Community Repositories +COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') + +# Platform Repositories +PLATFORM_REPOS=('nonsystemd' 'nonsystemd-testing') + +# Remote Repositories +PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") +PKG_MULTILIB_REPOS=("${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}") + +# Directories where packages are shared between repos +# *relative to REPO_DIR* +PKG_POOL="${PKG_POOL_DIR}/abslibre_${PLATFORM}_arm" + +# Directories where sources are stored +SRC_POOL="${SRC_POOL_DIR}/abslibre_${PLATFORM}_arm" + +# Build System and Repository scripts +BS_MAIN_NAME="abslibre_${PLATFORM}_arm" +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_DERIVATION_NAME="abs_${PLATFORM}_arm" +BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" +BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" +# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" +BLACKLIST_NAME="blacklist_${PLATFORM}_arm.txt" +BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" +BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" +SYNC_DERIVATION_SERVER='rsync.archlinuxarm.org' + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/abslibre_${PLATFORM}_arm" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/src/etc/dbscripts.d/gnu+linux_add_legacy.cfg b/src/etc/dbscripts.d/gnu+linux_add_legacy.cfg new file mode 100644 index 0000000..0cd0d8a --- /dev/null +++ b/src/etc/dbscripts.d/gnu+linux_add_legacy.cfg @@ -0,0 +1,67 @@ +#!/hint/bash + +# Parabola Platform +PLATFORM_NAME='GNU/Linux-libre' +PLATFORM='gnu+linux' + +# Multilib Support Archictectures +MULTILIB_ARCHES=('x86_64') + +# Archictectures +ARCHES=('i686' "${MULTILIB_ARCHES[@]}") + +# Main Repositories from Arch GNU/Linux +MAIN_REPOS=('core' 'extra' 'testing' 'community' 'community-testing') +# Libre Repositories from Parabola GNU/Linux-libre +LIBRE_MAIN_REPOS=('libre' 'libre-testing') + +# Multilib Repositories from Arch GNU/Linux +MULTILIB_REPOS=('multilib' 'multilib-testing') +# Libre Multilib Repositories from Parabola GNU/Linux-libre +LIBRE_MULTILIB_REPOS=('libre-multilib' 'libre-multilib-testing') + +# Extra Repositories +EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') + +# Community Repositories +COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') + +# Platform Repositories +PLATFORM_REPOS=('nonsystemd' 'nonsystemd-testing') + +# Remote Repositories +PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${LIBRE_MAIN_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") +PKG_MULTILIB_REPOS=("${LIBRE_MULTILIB_REPOS[@]}" "${MULTILIB_REPOS[@]}") + +# Directories where packages are shared between repos +# *relative to REPO_DIR* +PKG_POOL="pool/parabola" + +# Directories where sources are stored +SRC_POOL="sources/parabola" + +# Build System and Repository scripts +BS_MAIN_NAME="abslibre" +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_DERIVATION_NAME="abs" +BS_DERIVATION_DIR="${SRV_DIR}/${BS_DERIVATION_NAME}" +BS_GIT="${GIT_BASE}/abslibre/${BS_MAIN_NAME}.git" +# BS_GIT='http://projects.parabola.gnu/abslibre/${BS_MAIN_NAME}.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" +BLACKLIST_NAME="blacklist.txt" +BLACKLIST_FILE="${HOME}/blacklist/${BLACKLIST_NAME}.txt" +BLACKLIST_TMP="${TMP_DIR}/${BLACKLIST_NAME}.txt" +SYNC_DERIVATION_SERVER='rsync.archlinux.org' + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/main" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/src/etc/dbscripts.d/gnu+linux_native.cfg b/src/etc/dbscripts.d/gnu+linux_native.cfg new file mode 100644 index 0000000..d0a611a --- /dev/null +++ b/src/etc/dbscripts.d/gnu+linux_native.cfg @@ -0,0 +1,57 @@ +#!/hint/bash + +# Parabola Platform +PLATFORM_NAME='GNU/Linux-libre' +PLATFORM='gnu+linux' + +# Multilib Support Archictectures +MULTILIB_ARCHES=() # 'x86_64' + +# Archictectures +ARCHES=('mips64el' "${MULTILIB_ARCHES[@]}") # 'i686' + +# Main Repositories +MAIN_REPOS=('core' 'extra' 'testing') + +# Multilib Repositories +MULTILIB_REPOS=('multilib' 'multilib-testing') + +# Extra Repositories +EXTRA_REPOS=('nonprism' 'nonprism-testing' 'kernels' 'kernels-testing' 'cross' 'java') + +# Community Repositories +COMMUNITY_REPOS=('pcr' 'pcr-testing' 'pur' '~aurelien' '~brendan' '~coadde' '~drtan' '~emulatorman' '~jorginho' '~lukeshu' '~smv' '~xihh') + +# Platform Repositories +PLATFORM_REPOS=() + +# Remote Repositories +PKG_REPOS=("${EXTRA_REPOS[@]}" "${PLATFORM_REPOS[@]}" "${MAIN_REPOS[@]}" "${COMMUNITY_REPOS[@]}") +PKG_MULTILIB_REPOS=("${MULTILIB_REPOS[@]}") + +# Directories where packages are shared between repos +# *relative to REPO_DIR* +PKG_POOL="${PKG_POOL_DIR}/${PLATFORM}" + +# Directories where sources are stored +SRC_POOL="${SRC_POOL_DIR}/${PLATFORM}" + +# Build System and Repository scripts +BS_MAIN_NAME="pbs_${PLATFORM}" +BS_MAIN_DIR="${SRV_DIR}/${BS_MAIN_NAME}" +BS_GIT="${GIT_BASE}/${BS_MAIN_NAME}/pmr.git" +# BS_GIT='http://projects.parabola.gnu/${BS_MAIN_NAME}/pmr.git' +BS_GIT_TMP="${TMP_DIR}/${BS_MAIN_NAME}" + +# Base Repository +REPO_DIR="${SRV_DIR}/repo/${PLATFORM}" + +CLEANUP_DESTDIR="${REPO_DIR}/old/packages" +CLEANUP_DRYRUN='false' +# Time in days to keep moved packages +CLEANUP_KEEP='30' + +SOURCE_CLEANUP_DESTDIR="${REPO_DIR}/old/sources" +SOURCE_CLEANUP_DRYRUN='true' +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP='30' diff --git a/src/etc/dbscripts.d/main.cfg b/src/etc/dbscripts.d/main.cfg new file mode 100644 index 0000000..c1b9ffa --- /dev/null +++ b/src/etc/dbscripts.d/main.cfg @@ -0,0 +1,36 @@ +#!/hint/bash + +# Parabola Software Development +PSD=('rolling' 'testing' 'stable') + +ROLLING_NAME='rolling' +TESTING_NAME='waterbird-v2' +STABLE_NAME='minicat-v1' + +PKG_POOL_DIR='pools' +SRC_POOL_DIR='sources' + +SRV_DIR='/srv' +TMP_DIR='/tmp' +PKG_POOL_DIR='pools' +SRC_POOL_DIR='sources' +GIT_BASE="${SRV_DIR}/git" +SYNC_ARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' + +REQUIRE_SIGNATURE='true' + +LOCK_DELAY='10' +LOCK_TIMEOUT='300' + +[ -n "${STAGING:-}" ] || STAGING="${HOME}/staging/unknown/staging" +DB_EXT='.db.tar.gz' +FILES_EXT='.files.tar.gz' +PKG_EXT=".pkg.tar.?z" +SRC_EXT='.src.tar.gz' + +MIRROR_LIST='/etc/pacman.d/mirrorlist' +MAKEPKG_CONF='/etc/makepkg.conf' +MAKEPKG_ARCH="$(uname -m)" + +# parabolaweb root +WEB_DIR="${SRV_DIR}/http/www.parabola.nu/web" diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/blacklist_sample b/test/blacklist_sample deleted file mode 100644 index 2a02af6..0000000 --- a/test/blacklist_sample +++ /dev/null @@ -1,2 +0,0 @@ -alex:alex-libre: Aquí va un comentario -gmime22 ::Non free dependencies \ No newline at end of file diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz deleted file mode 100644 index 5eb2081..0000000 Binary files a/test/core.db.tar.gz and /dev/null differ diff --git a/test/depends b/test/depends deleted file mode 100644 index 7ff3ad4..0000000 --- a/test/depends +++ /dev/null @@ -1,4 +0,0 @@ -%DEPENDS% -glibc>=2.13 -zlib - diff --git a/test/desc b/test/desc deleted file mode 100644 index abba644..0000000 --- a/test/desc +++ /dev/null @@ -1,39 +0,0 @@ -%FILENAME% -binutils-2.21-4-x86_64.pkg.tar.xz - -%NAME% -binutils - -%VERSION% -2.21-4 - -%DESC% -A set of programs to assemble and manipulate binary and object files - -%GROUPS% -base - -%CSIZE% -3412892 - -%ISIZE% -17571840 - -%MD5SUM% -4e666f87c78998f4839f33dc06d2043a - -%URL% -http://www.gnu.org/software/binutils/ - -%LICENSE% -GPL - -%ARCH% -x86_64 - -%BUILDDATE% -1297240369 - -%PACKAGER% -Allan McRae - diff --git a/test/lib/common.inc b/test/lib/common.inc deleted file mode 100644 index a2dee10..0000000 --- a/test/lib/common.inc +++ /dev/null @@ -1,266 +0,0 @@ -set -E - -. "$(dirname ${BASH_SOURCE[0]})/../../config" -. "$(dirname ${BASH_SOURCE[0]})/../../db-functions" - -oneTimeSetUp() { - local p - local d - local a - local pkgname - local pkgarch - local pkgversion - local build - pkgdir="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)" - cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" - msg 'Building packages...' - for d in "${pkgdir}"/*; do - pushd $d >/dev/null - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - pkgarch=($(. PKGBUILD; echo ${arch[@]})) - pkgversion=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - - build=true - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - [ ! -f ${p}-${pkgversion}-${a}${PKGEXT} ] && build=false - done - done - - if ! ${build}; then - if [ "${pkgarch[0]}" == 'any' ]; then - sudo extra-x86_64-build || die 'extra-x86_64-build failed' - else - for a in ${pkgarch[@]}; do - sudo extra-${a}-build || die "extra-${a}-build failed" - done - fi - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/$(basename ${d}) - done - done - fi - popd >/dev/null - done -} - -oneTimeTearDown() { - rm -rf "${pkgdir}" -} - -setUp() { - local p - local pkg - local r - local a - - [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" - TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" - #msg "Using ${TMP}" - - PKGREPOS=('core' 'extra' 'testing') - PKGPOOL='pool/packages' - mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} - - for r in ${PKGREPOS[@]}; do - mkdir -p "${TMP}/staging/${r}" - for a in ${ARCHES[@]} any; do - mkdir -p "${TMP}/ftp/${r}/os/${a}" - done - done - mkdir -p "${TMP}/ftp/${PKGPOOL}" - mkdir -p "${TMP}/ftp/${SRCPOOL}" - - msg 'Creating svn repository...' - svnadmin create "${TMP}/svn-packages-repo" - svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" - - for p in "${pkgdir}"/*; do - pkg=$(basename $p) - mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} - cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ - svn add -q "${TMP}/svn-packages-copy"/${pkg} - svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" - done - - cat < "$(dirname ${BASH_SOURCE[0]})/../../config.local" - FTP_BASE="${TMP}/ftp" - SVNREPO="file://${TMP}/svn-packages-repo" - PKGREPOS=(${PKGREPOS[@]}) - PKGPOOL="${PKGPOOL}" - CLEANUP_DESTDIR="${TMP}/package-cleanup" - SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" - STAGING="${TMP}/staging" - TMPDIR="${TMP}/tmp" - CLEANUP_DRYRUN=false - SOURCE_CLEANUP_DRYRUN=false - REQUIRE_SIGNATURE=true -eot - . "$(dirname ${BASH_SOURCE[0]})/../../config" -} - -tearDown() { - rm -rf "${TMP}" - rm -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" - echo -} - -releasePackage() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null - archrelease ${repo}-${arch} >/dev/null 2&>1 - pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - popd >/dev/null - cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ - - if ${REQUIRE_SIGNATURE}; then - # TODO: really sign the packages with a valid key - find "${STAGING}"/${repo}/ -type f \ - -name "*-${pkgver}-${arch}${PKGEXT}" \ - -exec touch {}.sig \; - fi -} - -checkAnyPackageDB() { - local repo=$1 - local pkg=$2 - local arch - local db - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - if ${REQUIRE_SIGNATURE}; then - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" - fi - - for arch in i686 x86_64; do - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ - || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - - if ${REQUIRE_SIGNATURE}; then - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ - || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" - fi - done - [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done - - [ -r "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" - [ -r "${FTP_BASE}/${repo}/os/any/${pkg}.sig" ] && fail "${repo}/os/any/${pkg}.sig should not exist" -} - -checkAnyPackage() { - local repo=$1 - local pkg=$2 - - checkAnyPackageDB $repo $pkg - - local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ - || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" -} - -checkPackageDB() { - local repo=$1 - local pkg=$2 - local arch=$3 - local db - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" - [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ - || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - - if ${REQUIRE_SIGNATURE}; then - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ - || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" - fi - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done -} - -checkPackage() { - local repo=$1 - local pkg=$2 - local arch=$3 - - checkPackageDB $repo $pkg $arch - - local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ - || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" -} - -checkRemovedPackageDB() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - local db - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done -} - -checkRemovedPackage() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - - checkRemovedPackageDB $repo $pkgbase $arch - - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ - && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" -} - -checkRemovedAnyPackageDB() { - local repo=$1 - local pkgbase=$2 - local arch - local db - - for db in ${DBEXT} ${FILESEXT}; do - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done - done -} - -checkRemovedAnyPackage() { - local repo=$1 - local pkgbase=$2 - - checkRemovedAnyPackageDB $repo $pkgbase - - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ - && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" -} diff --git a/test/lib/shunit2 b/test/lib/shunit2 deleted file mode 100755 index 8862ffd..0000000 --- a/test/lib/shunit2 +++ /dev/null @@ -1,1048 +0,0 @@ -#! /bin/sh -# $Id: shunit2 335 2011-05-01 20:10:33Z kate.ward@forestent.com $ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# -# shUnit2 -- Unit testing framework for Unix shell scripts. -# http://code.google.com/p/shunit2/ -# -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is -# based on the popular JUnit unit testing framework for Java. - -# return if shunit already loaded -[ -n "${SHUNIT_VERSION:-}" ] && exit 0 - -SHUNIT_VERSION='2.1.6' - -SHUNIT_TRUE=0 -SHUNIT_FALSE=1 -SHUNIT_ERROR=2 - -# enable strict mode by default -SHUNIT_STRICT=${SHUNIT_STRICT:-${SHUNIT_TRUE}} - -_shunit_warn() { echo "shunit2:WARN $@" >&2; } -_shunit_error() { echo "shunit2:ERROR $@" >&2; } -_shunit_fatal() { echo "shunit2:FATAL $@" >&2; exit ${SHUNIT_ERROR}; } - -# specific shell checks -if [ -n "${ZSH_VERSION:-}" ]; then - setopt |grep "^shwordsplit$" >/dev/null - if [ $? -ne ${SHUNIT_TRUE} ]; then - _shunit_fatal 'zsh shwordsplit option is required for proper operation' - fi - if [ -z "${SHUNIT_PARENT:-}" ]; then - _shunit_fatal "zsh does not pass \$0 through properly. please declare \ -\"SHUNIT_PARENT=\$0\" before calling shUnit2" - fi -fi - -# -# constants -# - -__SHUNIT_ASSERT_MSG_PREFIX='ASSERT:' -__SHUNIT_MODE_SOURCED='sourced' -__SHUNIT_MODE_STANDALONE='standalone' -__SHUNIT_PARENT=${SHUNIT_PARENT:-$0} - -# set the constants readonly -shunit_constants_=`set |grep '^__SHUNIT_' |cut -d= -f1` -echo "${shunit_constants_}" |grep '^Binary file' >/dev/null && \ - shunit_constants_=`set |grep -a '^__SHUNIT_' |cut -d= -f1` -for shunit_constant_ in ${shunit_constants_}; do - shunit_ro_opts_='' - case ${ZSH_VERSION:-} in - '') ;; # this isn't zsh - [123].*) ;; # early versions (1.x, 2.x, 3.x) - *) shunit_ro_opts_='-g' ;; # all later versions. declare readonly globally - esac - readonly ${shunit_ro_opts_} ${shunit_constant_} -done -unset shunit_constant_ shunit_constants_ shunit_ro_opts_ - -# variables -__shunit_lineno='' # line number of executed test -__shunit_mode=${__SHUNIT_MODE_SOURCED} # operating mode -__shunit_reportGenerated=${SHUNIT_FALSE} # is report generated -__shunit_script='' # filename of unittest script (standalone mode) -__shunit_skip=${SHUNIT_FALSE} # is skipping enabled -__shunit_suite='' # suite of tests to execute - -# counts of tests -__shunit_testSuccess=${SHUNIT_TRUE} -__shunit_testsTotal=0 -__shunit_testsPassed=0 -__shunit_testsFailed=0 - -# counts of asserts -__shunit_assertsTotal=0 -__shunit_assertsPassed=0 -__shunit_assertsFailed=0 -__shunit_assertsSkipped=0 - -# macros -_SHUNIT_LINENO_='eval __shunit_lineno=""; if [ "${1:-}" = "--lineno" ]; then [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi' - -#----------------------------------------------------------------------------- -# assert functions -# - -# Assert that two values are equal to one another. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertEquals() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertEquals() requires two or three arguments; $# given" - _shunit_error "1: ${1:+$1} 2: ${2:+$2} 3: ${3:+$3}${4:+ 4: $4}" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - shunit_return=${SHUNIT_TRUE} - if [ "${shunit_expected_}" = "${shunit_actual_}" ]; then - _shunit_assertPass - else - failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" - shunit_return=${SHUNIT_FALSE} - fi - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${shunit_return} -} -_ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' - -# Assert that two values are not equal to one another. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotEquals() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertNotEquals() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - shunit_return=${SHUNIT_TRUE} - if [ "${shunit_expected_}" != "${shunit_actual_}" ]; then - _shunit_assertPass - else - failSame "${shunit_message_}" "$@" - shunit_return=${SHUNIT_FALSE} - fi - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${shunit_return} -} -_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' - -# Assert that a value is null (i.e. an empty string) -# -# Args: -# message: string: failure message [optional] -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNull() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertNull() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - assertTrue "${shunit_message_}" "[ -z '$1' ]" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' - -# Assert that a value is not null (i.e. a non-empty string) -# -# Args: -# message: string: failure message [optional] -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotNull() -{ - ${_SHUNIT_LINENO_} - if [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null - _shunit_error "assertNotNull() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_actual_=`_shunit_escapeCharactersInString "${1:-}"` - test -n "${shunit_actual_}" - assertTrue "${shunit_message_}" $? - shunit_return=$? - - unset shunit_actual_ shunit_message_ - return ${shunit_return} -} -_ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' - -# Assert that two values are the same (i.e. equal to one another). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - assertEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' - -# Assert that two values are not the same (i.e. not equal to one another). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertNotSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_:-}$1" - shift - fi - assertNotEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' - -# Assert that a value or shell test condition is true. -# -# In shell, a value of 0 is true and a non-zero value is false. Any integer -# value passed can thereby be tested. -# -# Shell supports much more complicated tests though, and a means to support -# them was needed. As such, this function tests that conditions are true or -# false through evaluation rather than just looking for a true or false. -# -# The following test will succeed: -# assertTrue 0 -# assertTrue "[ 34 -gt 23 ]" -# The folloing test will fail with a message: -# assertTrue 123 -# assertTrue "test failed" "[ -r '/non/existant/file' ]" -# -# Args: -# message: string: failure message [optional] -# condition: string: integer value or shell conditional statement -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertTrue() -{ - ${_SHUNIT_LINENO_} - if [ $# -gt 2 ]; then - _shunit_error "assertTrue() takes one two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_condition_=$1 - - # see if condition is an integer, i.e. a return value - shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` - shunit_return=${SHUNIT_TRUE} - if [ -z "${shunit_condition_}" ]; then - # null condition - shunit_return=${SHUNIT_FALSE} - elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] - then - # possible return value. treating 0 as true, and non-zero as false. - [ ${shunit_condition_} -ne 0 ] && shunit_return=${SHUNIT_FALSE} - else - # (hopefully) a condition - ( eval ${shunit_condition_} ) >/dev/null 2>&1 - [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE} - fi - - # record the test - if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then - _shunit_assertPass - else - _shunit_assertFail "${shunit_message_}" - fi - - unset shunit_message_ shunit_condition_ shunit_match_ - return ${shunit_return} -} -_ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' - -# Assert that a value or shell test condition is false. -# -# In shell, a value of 0 is true and a non-zero value is false. Any integer -# value passed can thereby be tested. -# -# Shell supports much more complicated tests though, and a means to support -# them was needed. As such, this function tests that conditions are true or -# false through evaluation rather than just looking for a true or false. -# -# The following test will succeed: -# assertFalse 1 -# assertFalse "[ 'apples' = 'oranges' ]" -# The folloing test will fail with a message: -# assertFalse 0 -# assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" -# -# Args: -# message: string: failure message [optional] -# condition: string: integer value or shell conditional statement -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertFalse() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertFalse() quires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_condition_=$1 - - # see if condition is an integer, i.e. a return value - shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` - shunit_return=${SHUNIT_TRUE} - if [ -z "${shunit_condition_}" ]; then - # null condition - shunit_return=${SHUNIT_FALSE} - elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] - then - # possible return value. treating 0 as true, and non-zero as false. - [ ${shunit_condition_} -eq 0 ] && shunit_return=${SHUNIT_FALSE} - else - # (hopefully) a condition - ( eval ${shunit_condition_} ) >/dev/null 2>&1 - [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE} - fi - - # record the test - if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then - _shunit_assertPass - else - _shunit_assertFail "${shunit_message_}" - fi - - unset shunit_message_ shunit_condition_ shunit_match_ - return ${shunit_return} -} -_ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' - -#----------------------------------------------------------------------------- -# failure functions -# - -# Records a test failure. -# -# Args: -# message: string: failure message [optional] -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -fail() -{ - ${_SHUNIT_LINENO_} - if [ $# -gt 1 ]; then - _shunit_error "fail() requires zero or one arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 1 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - - _shunit_assertFail "${shunit_message_}" - - unset shunit_message_ - return ${SHUNIT_FALSE} -} -_FAIL_='eval fail --lineno "${LINENO:-}"' - -# Records a test failure, stating two values were not equal. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failNotEquals() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failNotEquals() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>" - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${SHUNIT_FALSE} -} -_FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' - -# Records a test failure, stating two values should have been the same. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - - _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" - - unset shunit_message_ - return ${SHUNIT_FALSE} -} -_FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' - -# Records a test failure, stating two values were not equal. -# -# This is functionally equivalent to calling failNotEquals(). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failNotSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failNotEquals() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - failNotEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' - -#----------------------------------------------------------------------------- -# skipping functions -# - -# Force remaining assert and fail functions to be "skipped". -# -# This function forces the remaining assert and fail functions to be "skipped", -# i.e. they will have no effect. Each function skipped will be recorded so that -# the total of asserts and fails will not be altered. -# -# Args: -# None -startSkipping() -{ - __shunit_skip=${SHUNIT_TRUE} -} - -# Resume the normal recording behavior of assert and fail calls. -# -# Args: -# None -endSkipping() -{ - __shunit_skip=${SHUNIT_FALSE} -} - -# Returns the state of assert and fail call skipping. -# -# Args: -# None -# Returns: -# boolean: (TRUE/FALSE constant) -isSkipping() -{ - return ${__shunit_skip} -} - -#----------------------------------------------------------------------------- -# suite functions -# - -# Stub. This function should contains all unit test calls to be made. -# -# DEPRECATED (as of 2.1.0) -# -# This function can be optionally overridden by the user in their test suite. -# -# If this function exists, it will be called when shunit2 is sourced. If it -# does not exist, shunit2 will search the parent script for all functions -# beginning with the word 'test', and they will be added dynamically to the -# test suite. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Adds a function name to the list of tests schedule for execution. -# -# This function should only be called from within the suite() function. -# -# Args: -# function: string: name of a function to add to current unit test suite -suite_addTest() -{ - shunit_func_=${1:-} - - __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" - __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1` - - unset shunit_func_ -} - -# Stub. This function will be called once before any tests are run. -# -# Common one-time environment preparation tasks shared by all tests can be -# defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Stub. This function will be called once after all tests are finished. -# -# Common one-time environment cleanup tasks shared by all tests can be defined -# here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Stub. This function will be called before each test is run. -# -# Common environment preparation tasks shared by all tests can be defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#setUp() { :; } - -# Note: see _shunit_mktempFunc() for actual implementation -# Stub. This function will be called after each test is run. -# -# Common environment cleanup tasks shared by all tests can be defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -#------------------------------------------------------------------------------ -# internal shUnit2 functions -# - -# Create a temporary directory to store various run-time files in. -# -# This function is a cross-platform temporary directory creation tool. Not all -# OSes have the mktemp function, so one is included here. -# -# Args: -# None -# Outputs: -# string: the temporary directory that was created -_shunit_mktempDir() -{ - # try the standard mktemp function - ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return - - # the standard mktemp didn't work. doing our own. - if [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then - _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 "${_shunit_file_}" -#! /bin/sh -exit ${SHUNIT_TRUE} -EOF - chmod +x "${_shunit_file_}" - done - - unset _shunit_file_ -} - -# Final cleanup function to leave things as we found them. -# -# Besides removing the temporary directory, this function is in charge of the -# final exit code of the unit test. The exit code is based on how the script -# was ended (e.g. normal exit, or via Ctrl-C). -# -# Args: -# name: string: name of the trap called (specified when trap defined) -_shunit_cleanup() -{ - _shunit_name_=$1 - - case ${_shunit_name_} in - EXIT) _shunit_signal_=0 ;; - INT) _shunit_signal_=2 ;; - TERM) _shunit_signal_=15 ;; - *) - _shunit_warn "unrecognized trap value (${_shunit_name_})" - _shunit_signal_=0 - ;; - esac - - # do our work - rm -fr "${__shunit_tmpDir}" - - # exit for all non-EXIT signals - if [ ${_shunit_name_} != 'EXIT' ]; then - _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" - # disable EXIT trap - trap 0 - # add 128 to signal and exit - exit `expr ${_shunit_signal_} + 128` - elif [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ] ; then - _shunit_assertFail 'Unknown failure encountered running a test' - _shunit_generateReport - exit ${SHUNIT_ERROR} - fi - - unset _shunit_name_ _shunit_signal_ -} - -# The actual running of the tests happens here. -# -# Args: -# None -_shunit_execSuite() -{ - for _shunit_test_ in ${__shunit_suite}; do - __shunit_testSuccess=${SHUNIT_TRUE} - - # disable skipping - endSkipping - - # execute the per-test setup function - setUp - - # execute the test - echo "${_shunit_test_}" - eval ${_shunit_test_} - - # execute the per-test tear-down function - tearDown - - # update stats - if [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then - __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1` - else - __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1` - fi - done - - unset _shunit_test_ -} - -# Generates the user friendly report with appropriate OK/FAILED message. -# -# Args: -# None -# Output: -# string: the report of successful and failed tests, as well as totals. -_shunit_generateReport() -{ - _shunit_ok_=${SHUNIT_TRUE} - - # if no exit code was provided one, determine an appropriate one - [ ${__shunit_testsFailed} -gt 0 \ - -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \ - && _shunit_ok_=${SHUNIT_FALSE} - - echo - if [ ${__shunit_testsTotal} -eq 1 ]; then - echo "Ran ${__shunit_testsTotal} test." - else - echo "Ran ${__shunit_testsTotal} tests." - fi - - _shunit_failures_='' - _shunit_skipped_='' - [ ${__shunit_assertsFailed} -gt 0 ] \ - && _shunit_failures_="failures=${__shunit_assertsFailed}" - [ ${__shunit_assertsSkipped} -gt 0 ] \ - && _shunit_skipped_="skipped=${__shunit_assertsSkipped}" - - if [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then - _shunit_msg_='OK' - [ -n "${_shunit_skipped_}" ] \ - && _shunit_msg_="${_shunit_msg_} (${_shunit_skipped_})" - else - _shunit_msg_="FAILED (${_shunit_failures_}" - [ -n "${_shunit_skipped_}" ] \ - && _shunit_msg_="${_shunit_msg_},${_shunit_skipped_}" - _shunit_msg_="${_shunit_msg_})" - fi - - echo - echo ${_shunit_msg_} - __shunit_reportGenerated=${SHUNIT_TRUE} - - unset _shunit_failures_ _shunit_msg_ _shunit_ok_ _shunit_skipped_ -} - -# Test for whether a function should be skipped. -# -# Args: -# None -# Returns: -# boolean: whether the test should be skipped (TRUE/FALSE constant) -_shunit_shouldSkip() -{ - [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE} - _shunit_assertSkip -} - -# Records a successful test. -# -# Args: -# None -_shunit_assertPass() -{ - __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` -} - -# Records a test failure. -# -# Args: -# message: string: failure message to provide user -_shunit_assertFail() -{ - _shunit_msg_=$1 - - __shunit_testSuccess=${SHUNIT_FALSE} - __shunit_assertsFailed=`expr ${__shunit_assertsFailed} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` - echo "${__SHUNIT_ASSERT_MSG_PREFIX}${_shunit_msg_}" - - unset _shunit_msg_ -} - -# Records a skipped test. -# -# Args: -# None -_shunit_assertSkip() -{ - __shunit_assertsSkipped=`expr ${__shunit_assertsSkipped} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` -} - -# Prepare a script filename for sourcing. -# -# Args: -# script: string: path to a script to source -# Returns: -# string: filename prefixed with ./ (if necessary) -_shunit_prepForSourcing() -{ - _shunit_script_=$1 - case "${_shunit_script_}" in - /*|./*) echo "${_shunit_script_}" ;; - *) echo "./${_shunit_script_}" ;; - esac - unset _shunit_script_ -} - -# Escape a character in a string. -# -# Args: -# c: string: unescaped character -# s: string: to escape character in -# Returns: -# string: with escaped character(s) -_shunit_escapeCharInStr() -{ - [ -n "$2" ] || return # no point in doing work on an empty string - - # Note: using shorter variable names to prevent conflicts with - # _shunit_escapeCharactersInString(). - _shunit_c_=$1 - _shunit_s_=$2 - - - # escape the character - echo ''${_shunit_s_}'' |sed 's/\'${_shunit_c_}'/\\\'${_shunit_c_}'/g' - - unset _shunit_c_ _shunit_s_ -} - -# Escape a character in a string. -# -# Args: -# str: string: to escape characters in -# Returns: -# string: with escaped character(s) -_shunit_escapeCharactersInString() -{ - [ -n "$1" ] || return # no point in doing work on an empty string - - _shunit_str_=$1 - - # Note: using longer variable names to prevent conflicts with - # _shunit_escapeCharInStr(). - for _shunit_char_ in '"' '$' "'" '`'; do - _shunit_str_=`_shunit_escapeCharInStr "${_shunit_char_}" "${_shunit_str_}"` - done - - echo "${_shunit_str_}" - unset _shunit_char_ _shunit_str_ -} - -# Extract list of functions to run tests against. -# -# Args: -# script: string: name of script to extract functions from -# Returns: -# string: of function names -_shunit_extractTestFunctions() -{ - _shunit_script_=$1 - - # extract the lines with test function names, strip of anything besides the - # function name, and output everything on a single line. - _shunit_regex_='^[ ]*(function )*test[A-Za-z0-9_]* *\(\)' - egrep "${_shunit_regex_}" "${_shunit_script_}" \ - |sed 's/^[^A-Za-z0-9_]*//;s/^function //;s/\([A-Za-z0-9_]*\).*/\1/g' \ - |xargs - - unset _shunit_regex_ _shunit_script_ -} - -#------------------------------------------------------------------------------ -# main -# - -# determine the operating mode -if [ $# -eq 0 ]; then - __shunit_script=${__SHUNIT_PARENT} - __shunit_mode=${__SHUNIT_MODE_SOURCED} -else - __shunit_script=$1 - [ -r "${__shunit_script}" ] || \ - _shunit_fatal "unable to read from ${__shunit_script}" - __shunit_mode=${__SHUNIT_MODE_STANDALONE} -fi - -# create a temporary storage location -__shunit_tmpDir=`_shunit_mktempDir` - -# provide a public temporary directory for unit test scripts -# TODO(kward): document this -SHUNIT_TMPDIR="${__shunit_tmpDir}/tmp" -mkdir "${SHUNIT_TMPDIR}" - -# setup traps to clean up after ourselves -trap '_shunit_cleanup EXIT' 0 -trap '_shunit_cleanup INT' 2 -trap '_shunit_cleanup TERM' 15 - -# create phantom functions to work around issues with Cygwin -_shunit_mktempFunc -PATH="${__shunit_tmpDir}:${PATH}" - -# make sure phantom functions are executable. this will bite if /tmp (or the -# current $TMPDIR) points to a path on a partition that was mounted with the -# 'noexec' option. the noexec command was created with _shunit_mktempFunc(). -noexec 2>/dev/null || _shunit_fatal \ - 'please declare TMPDIR with path on partition with exec permission' - -# we must manually source the tests in standalone mode -if [ "${__shunit_mode}" = "${__SHUNIT_MODE_STANDALONE}" ]; then - . "`_shunit_prepForSourcing \"${__shunit_script}\"`" -fi - -# execute the oneTimeSetUp function (if it exists) -oneTimeSetUp - -# execute the suite function defined in the parent test script -# deprecated as of 2.1.0 -suite - -# if no suite function was defined, dynamically build a list of functions -if [ -z "${__shunit_suite}" ]; then - shunit_funcs_=`_shunit_extractTestFunctions "${__shunit_script}"` - for shunit_func_ in ${shunit_funcs_}; do - suite_addTest ${shunit_func_} - done -fi -unset shunit_func_ shunit_funcs_ - -# execute the tests -_shunit_execSuite - -# execute the oneTimeTearDown function (if it exists) -oneTimeTearDown - -# generate the report -_shunit_generateReport - -# that's it folks -[ ${__shunit_testsFailed} -eq 0 ] -exit $? diff --git a/test/packages/pkg-any-a/PKGBUILD b/test/packages/pkg-any-a/PKGBUILD deleted file mode 100644 index 8749a35..0000000 --- a/test/packages/pkg-any-a/PKGBUILD +++ /dev/null @@ -1,12 +0,0 @@ -pkgname=pkg-any-a -pkgver=1 -pkgrel=1 -pkgdesc="A package called ${pkgname}" -arch=('any') -url='http://www.archlinux.org/' -license=('GPL') - -package() { - install -d -m755 ${pkgdir}/usr/share/${pkgname} - echo 'test' > ${pkgdir}/usr/share/${pkgname}/test -} diff --git a/test/packages/pkg-any-b/PKGBUILD b/test/packages/pkg-any-b/PKGBUILD deleted file mode 100644 index e6a0498..0000000 --- a/test/packages/pkg-any-b/PKGBUILD +++ /dev/null @@ -1,12 +0,0 @@ -pkgname=pkg-any-b -pkgver=1 -pkgrel=1 -pkgdesc="A package called ${pkgname}" -arch=('any') -url='http://www.archlinux.org/' -license=('GPL') - -package() { - install -d -m755 ${pkgdir}/usr/share/${pkgname} - echo 'test' > ${pkgdir}/usr/share/${pkgname}/test -} diff --git a/test/packages/pkg-simple-a/Makefile b/test/packages/pkg-simple-a/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-simple-a/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-simple-a/PKGBUILD b/test/packages/pkg-simple-a/PKGBUILD deleted file mode 100644 index 953ecfa..0000000 --- a/test/packages/pkg-simple-a/PKGBUILD +++ /dev/null @@ -1,22 +0,0 @@ -pkgname=pkg-simple-a -pkgver=1 -pkgrel=1 -pkgdesc="A package called ${pkgname}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} - -package() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname} -} diff --git a/test/packages/pkg-simple-a/test.c b/test/packages/pkg-simple-a/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-simple-a/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-simple-b/Makefile b/test/packages/pkg-simple-b/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-simple-b/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-simple-b/PKGBUILD b/test/packages/pkg-simple-b/PKGBUILD deleted file mode 100644 index 95ffd09..0000000 --- a/test/packages/pkg-simple-b/PKGBUILD +++ /dev/null @@ -1,22 +0,0 @@ -pkgname=pkg-simple-b -pkgver=1 -pkgrel=1 -pkgdesc="A package called ${pkgname}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} - -package() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname} -} diff --git a/test/packages/pkg-simple-b/test.c b/test/packages/pkg-simple-b/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-simple-b/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-simple-epoch/Makefile b/test/packages/pkg-simple-epoch/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-simple-epoch/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-simple-epoch/PKGBUILD b/test/packages/pkg-simple-epoch/PKGBUILD deleted file mode 100644 index eebe2bd..0000000 --- a/test/packages/pkg-simple-epoch/PKGBUILD +++ /dev/null @@ -1,23 +0,0 @@ -pkgname=pkg-simple-epoch -pkgver=1 -pkgrel=1 -epoch=1 -pkgdesc="A package called ${pkgname}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} - -package() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname} -} diff --git a/test/packages/pkg-simple-epoch/test.c b/test/packages/pkg-simple-epoch/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-simple-epoch/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-split-a/Makefile b/test/packages/pkg-split-a/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-split-a/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-split-a/PKGBUILD b/test/packages/pkg-split-a/PKGBUILD deleted file mode 100644 index e941976..0000000 --- a/test/packages/pkg-split-a/PKGBUILD +++ /dev/null @@ -1,28 +0,0 @@ -pkgbase=pkg-split-a -pkgname=('pkg-split-a1' 'pkg-split-a2') -pkgver=1 -pkgrel=1 -pkgdesc="A split package called ${pkgbase}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} - -package_pkg-split-a1() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} -} - -package_pkg-split-a2() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} -} diff --git a/test/packages/pkg-split-a/test.c b/test/packages/pkg-split-a/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-split-a/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-split-b/Makefile b/test/packages/pkg-split-b/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-split-b/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-split-b/PKGBUILD b/test/packages/pkg-split-b/PKGBUILD deleted file mode 100644 index 6ddbc45..0000000 --- a/test/packages/pkg-split-b/PKGBUILD +++ /dev/null @@ -1,29 +0,0 @@ -pkgbase=pkg-split-b -pkgname=('pkg-split-b1' 'pkg-split-b2') -pkgver=1 -pkgrel=1 -pkgdesc="A split package called ${pkgbase}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') - -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} - -package_pkg-split-b1() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} -} - -package_pkg-split-b2() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} -} diff --git a/test/packages/pkg-split-b/test.c b/test/packages/pkg-split-b/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-split-b/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/rsync_output_sample b/test/rsync_output_sample deleted file mode 100644 index 72d9cd0..0000000 --- a/test/rsync_output_sample +++ /dev/null @@ -1,14 +0,0 @@ -dr-xr-sr-x 4096 2010/09/11 11:37:10 . --rw-r--r-- 11 2011/02/08 00:00:01 lastsync -drwxrwxr-x 15 2010/09/11 11:28:50 community-staging -drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os -drwxrwxr-x 8192 2011/02/07 17:00:01 community-staging/os/i686 -lrwxrwxrwx 52 2010/12/23 16:51:01 community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz -> ../../../pool/community/alex-2.3.4-1-i686.pkg.tar.xz -lrwxrwxrwx 27 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db -> community-staging.db.tar.gz --rw-rw-r-- 2237 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db.tar.gz --rw-rw-r-- 3209 2011/02/07 14:00:13 community-staging/os/i686/community-staging.db.tar.gz.old -drwxrwxr-x 15 2009/07/22 15:07:56 community -drwxrwxr-x 40 2009/08/04 15:57:42 community/os -drwxrwsr-x 36864 2011/02/03 05:00:01 community/os/any --rw-rw-r-- 303336 2010/07/16 10:06:28 community/os/any/any2dvd-0.34-4-any.pkg.tar.xz --rw-rw-r-- 221664 2010/03/28 15:55:48 community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz diff --git a/test/runTest b/test/runTest deleted file mode 100755 index b8713d8..0000000 --- a/test/runTest +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -. "$(dirname ${BASH_SOURCE[0]})/lib/common.inc" - -for t in "$(dirname ${BASH_SOURCE[0]})/test.d/"*.sh; do - l=$(basename ${t} .sh) - if [ -x ${t} ]; then - msg "Running test '${l}'" - ${t} - [ $? -ne 0 ] && die "Test '${l}' failed" - echo -e "\n\n\n" - else - warning "Skipping test ${l}" - fi -done diff --git a/test/src/Makefile b/test/src/Makefile deleted file mode 100644 index 105b730..0000000 --- a/test/src/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -all: - gcc $(CFLAGS) -o test test.c - -install: - install -D -m755 test $(DESTDIR)/usr/bin/$(DESTBIN) diff --git a/test/src/test.c b/test/src/test.c deleted file mode 100644 index a661689..0000000 --- a/test/src/test.c +++ /dev/null @@ -1,7 +0,0 @@ -#include -#include - -int main(void) { - printf("Arch is the best!\n"); - return EXIT_SUCCESS; -} diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh deleted file mode 100755 index 49734c4..0000000 --- a/test/test.d/create-filelists.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testCreateSimpleFileLists() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then - fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}" - fi - done - done -} - -testCreateAnyFileLists() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then - fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}" - fi - done - done -} - -testCreateSplitFileLists() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local pkgname - local pkgnames - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - - for pkgbase in ${pkgs[@]}; do - pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) - for pkgname in ${pkgnames[@]}; do - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then - fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}" - fi - done - done - done -} - - -testCleanupFileLists() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - - for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-a - done - - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then - fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}" - fi - if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-a"; then - fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}" - fi - done - -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-move.sh b/test/test.d/db-move.sh deleted file mode 100755 index 9d7c1f6..0000000 --- a/test/test.d/db-move.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testMoveSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - - ../db-move testing extra pkg-simple-a - - for arch in ${arches[@]}; do - checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} - - checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} - done -} - -testMoveMultiplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - - ../db-move testing extra pkg-simple-a pkg-simple-b - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done - done -} - -testMoveEpochPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - - ../db-move testing extra pkg-simple-epoch - - for arch in ${arches[@]}; do - checkPackage extra pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} - done -} - -testMoveAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any - done - - ../db-update - ../db-move testing extra pkg-any-a - - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz - checkRemovedAnyPackage testing pkg-any-a - checkAnyPackage testing pkg-any-b-1-1-any.pkg.tar.xz -} - -testMoveSplitPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - ../db-move testing extra pkg-split-a - - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/pkg-split-a"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} - done - done - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/pkg-split-b"/*-${arch}${PKGEXT}; do - checkPackage testing $(basename ${pkg}) ${arch} - done - done - - checkRemovedAnyPackage testing pkg-split-a -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-remove.sh b/test/test.d/db-remove.sh deleted file mode 100755 index 416e693..0000000 --- a/test/test.d/db-remove.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testRemovePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgbase} - done - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackage extra ${pkgbase} ${arch} - done - done -} - -testRemoveMultiplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgs[@]} - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackage extra ${pkgbase} ${arch} - done - done -} - -testRemoveAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - ../db-remove extra any ${pkgbase} - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedAnyPackage extra ${pkgbase} - done -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-repo-add.sh b/test/test.d/db-repo-add.sh deleted file mode 100755 index 8603104..0000000 --- a/test/test.d/db-repo-add.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testAddSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" - touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" - ../db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz - done - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done - done -} - -testAddMultiplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for arch in ${arches[@]}; do - add_pkgs=() - for pkgbase in ${pkgs[@]}; do - cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" - touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" - add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz - done - ../db-repo-add extra ${arch} ${add_pkgs[@]} - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done - done -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-repo-remove.sh b/test/test.d/db-repo-remove.sh deleted file mode 100755 index 315d63d..0000000 --- a/test/test.d/db-repo-remove.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testRemovePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - ../db-repo-remove extra ${arch} ${pkgbase} - done - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackageDB extra ${pkgbase} ${arch} - done - done -} - -testRemoveMultiplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for arch in ${arches[@]}; do - ../db-repo-remove extra ${arch} ${pkgs[@]} - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackageDB extra ${pkgbase} ${arch} - done - done -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh deleted file mode 100755 index e38c328..0000000 --- a/test/test.d/db-update.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testAddSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done - done -} - -testAddSingleSimplePackage() { - releasePackage extra 'pkg-simple-a' 'i686' - ../db-update - checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' -} - -testAddSingleEpochPackage() { - releasePackage extra 'pkg-simple-epoch' 'i686' - ../db-update - checkPackage extra 'pkg-simple-epoch-1:1-1-i686.pkg.tar.xz' 'i686' -} - -testAddAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz - done -} - -testAddSplitPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} - done - done - done -} - -testUpdateAnyPackage() { - releasePackage extra pkg-any-a any - ../db-update - - pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 - mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" - popd >/dev/null - - releasePackage extra pkg-any-a any - ../db-update - - checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any - - rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" -} - -testUpdateAnyPackageToDifferentRepositoriesAtOnce() { - releasePackage extra pkg-any-a any - - pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 - mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" - popd >/dev/null - - releasePackage testing pkg-any-a any - - ../db-update - - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any - - rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" -} - -testUpdateSameAnyPackageToSameRepository() { - releasePackage extra pkg-any-a any - ../db-update - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - - releasePackage extra pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to the same repository should fail'; return 1) -} - -testUpdateSameAnyPackageToDifferentRepositories() { - releasePackage extra pkg-any-a any - ../db-update - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - - releasePackage testing pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) - - local arch - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" - done -} - - -testAddIncompleteSplitPackage() { - local arches=('i686' 'x86_64') - local repo='extra' - local pkgbase='pkg-split-a' - local arch - - for arch in ${arches[@]}; do - releasePackage ${repo} ${pkgbase} ${arch} - done - - # remove a split package to make db-update fail - rm "${STAGING}"/extra/${pkgbase}1-* - - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a split package is missing!" - - for arch in ${arches[@]}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" - done -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh deleted file mode 100755 index 20026b4..0000000 --- a/test/test.d/ftpdir-cleanup.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testCleanupSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-a - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for arch in ${arches[@]}; do - local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" - checkRemovedPackage extra 'pkg-simple-a' ${arch} - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" - - local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" - checkPackage extra ${pkg2} ${arch} - done -} - -testCleanupEpochPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-epoch - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for arch in ${arches[@]}; do - local pkg1="pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz" - checkRemovedPackage extra 'pkg-simple-epoch' ${arch} - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" - done -} - -testCleanupAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch='any' - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - ../db-update - ../db-remove extra any pkg-any-a - ../cron-jobs/ftpdir-cleanup >/dev/null - - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - checkRemovedAnyPackage extra 'pkg-any-a' - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" - - local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" - checkAnyPackage extra ${pkg2} -} - -testCleanupSplitPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgs[0]} - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}${PKGEXT}; do - checkRemovedPackage extra ${pkgs[0]} ${arch} - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg}" ] && fail "${PKGPOOL}/${pkg} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" - done - - for pkg in "${pkgdir}/${pkgs[1]}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} - done - done -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/packages.sh b/test/test.d/packages.sh deleted file mode 100755 index 488cb15..0000000 --- a/test/test.d/packages.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testPackages() { - # TODO: namcap -r sodepends fails with i686 packages - find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e sodepends,pkgnameindesc {} + || fail 'namcap failed' -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/pool-transition.sh b/test/test.d/pool-transition.sh deleted file mode 100755 index 5873f00..0000000 --- a/test/test.d/pool-transition.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testMovePackagesWithoutPool() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') - local pkgbase - local arch - local pkg - local old - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - - # transform two packages to old style layout - for arch in ${arches[@]}; do - for old in 0 2; do - for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}${PKGEXT}; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} - done - checkRemovedPackage testing ${pkgbase} ${arch} - done - done -} - -testUpdateAnyPackageWithoutPool() { - local pkgname='pkg-any-a' - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - local pkg2='pkg-any-a-1-2-any.pkg.tar.xz' - local arch - - - releasePackage extra pkg-any-a any - ../db-update - # transform two packages to old style layout - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" "${FTP_BASE}/extra/os/any" - for arch in i686 x86_64; do - ln -sf "../any/${pkg1}" "${FTP_BASE}/extra/os/${arch}" - done - - pushd "${TMP}/svn-packages-copy/${pkgname}/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 - mv "${pkg2}" "${pkgdir}/${pkgname}/" - popd >/dev/null - - releasePackage extra ${pkgname} any - ../db-update - rm -f "${pkgdir}/${pkgname}/${pkg2}" - - ../cron-jobs/ftpdir-cleanup >/dev/null - - checkAnyPackage extra "${pkg2}" - - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/extra/os/${arch}/${pkg1}" ] && fail "extra/os/${arch}/${pkg1} found" - done -} - -testMoveAnyPackagesWithoutPool() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch - local pkg - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any - done - - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-any${PKGEXT}; do - checkAnyPackage extra $(basename ${pkg}) - done - checkRemovedAnyPackage testing ${pkgbase} - done - - for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=$(basename $pkg) - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/testing/os/${arch}/${pkg}" ] && fail "testing/os/${arch}/${pkg} found" - done - done -} - -testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { - local pkg - local arch - - releasePackage extra pkg-any-a any - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/pkg-any-a"/*-any${PKGEXT}; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" - done - done - - releasePackage testing pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) - - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q pkg-any-a) \ - && fail "pkg-any-a should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" - done -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/signed-packages.sh b/test/test.d/signed-packages.sh deleted file mode 100755 index 5d6f4ff..0000000 --- a/test/test.d/signed-packages.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testAddUnsignedPackage() { - releasePackage extra 'pkg-simple-a' 'i686' - # remove any signature - rm "${STAGING}"/extra/*.sig - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/sourceballs.sh b/test/test.d/sourceballs.sh deleted file mode 100755 index fdcf08c..0000000 --- a/test/test.d/sourceballs.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testSourceballs() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - - ../cron-jobs/sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" - done -} - -testAnySourceballs() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - ../db-update - - ../cron-jobs/sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" - done -} - -testSplitSourceballs() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - ../cron-jobs/sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" - done -} - -testSourceballsCleanup() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - ../cron-jobs/sourceballs - - for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-a - done - - ../cron-jobs/sourceballs - [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" - [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh deleted file mode 100755 index eda6cd6..0000000 --- a/test/test.d/testing2x.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testTesting2xAnyPackage() { - releasePackage core pkg-any-a any - ../db-update - - pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 - mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" - popd >/dev/null - - releasePackage testing pkg-any-a any - ../db-update - rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" - - ../testing2x pkg-any-a - - checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz any - checkRemovedAnyPackage testing pkg-any-a -} - -. "${curdir}/../lib/shunit2" diff --git a/test/test_filter.py b/test/test_filter.py deleted file mode 100644 index d8006f9..0000000 --- a/test/test_filter.py +++ /dev/null @@ -1,196 +0,0 @@ -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -class pkginfo_from_file_KnownValues(unittest.TestCase): - # (filename, name, version, release, arch) - # filename is location - known=( - ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), - ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), - ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), - ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), - ) - - def generate_results(self, example_tuple, attr): - location, name, version, release, arch = example_tuple - return pkginfo_from_filename(location)[attr], locals()[attr] - - def testReturnPackageObject(self): - for i in self.known: - location, name, version, release, arch = i - self.assertIsInstance(pkginfo_from_filename(location),Package) - - def testNames(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) - -class pkginfo_from_file_BadInput(unittest.TestCase): - bad=("community-testing/os/i686/community-testing.db", - "community-testing/os/i686/community-testing.db.tar.gz", - "community-testing/os/i686/community-testing.db.tar.gz.old", - "community-testing/os/i686/community-testing.files", - "community-testing/os/i686/community-testing.files.tar.gz", - "community-testing/os/x86_64") - - def testBadInput(self): - for i in self.bad: - self.assertRaises(NonValidFile,pkginfo_from_filename,i) - -class pkginfoFromRsyncOutput(unittest.TestCase): - example_package_list=(Package(),Package(),Package()) - example_package_list[0].package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", - "depends" : False,} - example_package_list[1].package_info={ "name" : "any2dvd", - "version" : "0.34", - "release" : "4", - "arch" : "any", - "license" : False, - "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", - "depends" : False,} - example_package_list[2].package_info={ "name" : "gmime22", - "version" : "2.2.26", - "release" : "1", - "arch" : "x86_64", - "license" : False, - "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", - "depends" : False,} - - try: - output_file = open("rsync_output_sample") - rsync_out= output_file.read() - output_file.close() - except IOError: print("There is no rsync_output_sample file") - - pkglist = pkginfo_from_rsync_output(rsync_out) - - def testOutputArePackages(self): - if not self.pkglist: - self.fail("not pkglist:" + str(self.pkglist)) - for pkg in self.pkglist: - self.assertIsInstance(pkg,Package) - - def testPackageInfo(self): - if not self.pkglist: - self.fail("Pkglist doesn't exist: " + str(self.pkglist)) - self.assertEqual(self.pkglist,self.example_package_list) - -class generateRsyncBlacklist(unittest.TestCase): - example_package_list=(Package(),Package(),Package()) - example_package_list[0].package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", - "depends" : False,} - example_package_list[1].package_info={ "name" : "any2dvd", - "version" : "0.34", - "release" : "4", - "arch" : "any", - "license" : False, - "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", - "depends" : False,} - example_package_list[2].package_info={ "name" : "gmime22", - "version" : "2.2.26", - "release" : "1", - "arch" : "x86_64", - "license" : False, - "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", - "depends" : False,} - - def testListado(self): - self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) - - def testExcludeFiles(self): - a=rsyncBlacklist_from_blacklist(self.example_package_list, - listado("blacklist_sample"), - False) - b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] - self.assertEqual(a,b) - -class pkginfo_from_descKnownValues(unittest.TestCase): - pkgsample=Package() - pkgsample.package_info={"name" : "binutils", - "version" : "2.21", - "release" : "4", - "arch" : "x86_64", - "license" : "GPL", - "location": "binutils-2.21-4-x86_64.pkg.tar.xz", - "depends" : False,} - fsock=open("desc") - pkggen=pkginfo_from_desc(fsock.read()) - fsock.close() - def testPkginfoFromDesc(self): - if self.pkggen is None: - self.fail("return value is None") - self.assertEqual(self.pkgsample,self.pkggen) - -class pkginfo_from_db(unittest.TestCase): - archdb = os.path.join("./workdir") - example_package_list=(Package(),Package(),Package()) - example_package_list[0].package_info={ "name" : "acl", - "version" : "2.2.49", - "release" : "2", - "arch" : "x86_64", - "license" : ("LGPL",), - "location": "acl-2.2.49-2-x86_64.pkg.tar.xz", - "depends" : ("attr>=2.4.41"),} - example_package_list[1].package_info={ "name" : "glibc", - "version" : "2.13", - "release" : "4", - "arch" : "x86_64", - "license" : ("GPL","LGPL"), - "location": "glibc-2.13-4-x86_64.pkg.tar.xz", - "depends" : ("linux-api-headers>=2.6.37","tzdata",),} - example_package_list[2].package_info={ "name" : "", - "version" : "2.2.26", - "release" : "1", - "arch" : "x86_64", - "license" : False, - "location": "", - "depends" : False,} - - -if __name__ == "__main__": - unittest.main() - diff --git a/xbslibre b/xbslibre deleted file mode 100755 index e40865a..0000000 --- a/xbslibre +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -set -e - -# Steps -# * Sync bs derivation (abs) -# * Download blacklist.txt -# * Sync bs (abslibre) from bs derivation (abs) excluding from blacklist -# * Create repo.abs.tar.gz tarballs - -source "$(dirname "$(readlink -e "$0")")/config_platforms" - -for 'platform' in "${DERIVATIVE_PLATFORMS[@]}"; do - - source "$(dirname "$(readlink -e "$0")")/config_${platform}" - - function sync_bs_derivation() { - for 'ARCH' in 'any' "${ARCHES[@]}"; do - rsync "${SYNC_ARGS}" "${SYNC_DERIVATION_SERVER}::${BS_DERIVATION_NAME}/${ARCH}" "${BS_DERIVATION_DIR}/${ARCH}" || return $? - done - - # fix some permissions - find "${BS_DERIVATION_DIR}" -type d -print0 | xargs -0 chmod 755 - find "${BS_DERIVATION_DIR}" -type f -print0 | xargs -0 chmod 644 - } - - function get_blacklist() { - printf ":: Updating blacklist...\t" - cat "${BLACKLIST_FILE}" | cut -d':' -f1 | sort -u | \ - sed "s/^/**\//" > "${BLACKLIST_TMP}" || { - printf "[FAILED]\n" - return 1 - } - - # Prevent using an empty blacklist - [ $(wc -l "${BLACKLIST_TMP}" | cut -d " " -f1) -eq 0 ] && return 1 - - printf "[OK]\n" - } - - function sync_bs() { - - # Clone BS git repo - if [ -d "${BS_GIT_TMP}"/.git ]; then - pushd "${BS_GIT_TMP}" >/dev/null 2>&1 - git pull - popd >/dev/null 2>&1 - else - git clone "${BS_GIT}" "${BS_GIT_TMP}" - fi - - # Sync from BS_DERIVATION and then sync from BS - printf ":: Syncing ${BS_MAIN_NAME}...\t" - (rsync "${SYNC_ARGS}" --delete-excluded \ - --exclude-from="${BLACKLIST_TMP}" \ - "${BS_DERIVATION_DIR}" \ - "${BS_MAIN_DIR}" \ - && - for 'ARCH' in "${ARCHES[@]}"; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ "${BS_GIT_TMP}" "${BS_MAIN_DIR}/${ARCH}/"; done) || { - printf "[FAILED]\n" - return 1 - } - - # fix some permissions - find "${BS_MAIN_DIR}" -type d -print0 | xargs -0 chmod 755 - find "${BS_MAIN_DIR}" -type f -print0 | xargs -0 chmod 644 - - printf "[OK]\n" - } - - # Create .abs.tar.gz tarballs - create_tarballs() { - for 'repo' in "${BS_MAIN_DIR}/${ARCHES[@]}/"*; do - baserepo="${repo##*/}" - arch="$(basename $(dirname ${repo}))" - - # Remove the old one - mkdir -p "${REPO_DIR}/${baserepo}/os/${arch}" - rm -fv "${REPO_DIR}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" - # Create a new one joining arch and any - # Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czf "${REPO_DIR}/${baserepo}/os/${arch}/${baserepo}.abs.tar.gz" \ - -s ":${BS_MAIN_DIR}/[a-z0-9_]\+/[a-z]\+::" \ - "${repo}"/* "${BS_MAIN_DIR}/any/${baserepo}"/* - - done - } - - sync_bs_derivation - get_blacklist - sync_bs - create_tarballs -done -- cgit v1.2.3