From cdf17a4c4c83c1363a59fa688654276c3d807eae Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 14 Aug 2010 20:15:50 +0200 Subject: Rewrite of db-update * db-update now updates all repos with packages in its staging dirs * sanity checks are performed before any repo is touched * improved performance * less code; easier to maintain --- db-community | 3 - db-community-testing | 3 - db-core | 3 - db-extra | 3 - db-functions | 50 ++++++++++-- db-testing | 3 - db-update | 227 +++++++++++++-------------------------------------- test/runTest | 26 +++--- 8 files changed, 114 insertions(+), 204 deletions(-) delete mode 100755 db-community delete mode 100755 db-community-testing delete mode 100755 db-core delete mode 100755 db-extra delete mode 100755 db-testing diff --git a/db-community b/db-community deleted file mode 100755 index 44767a5..0000000 --- a/db-community +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "community" diff --git a/db-community-testing b/db-community-testing deleted file mode 100755 index eb4cf67..0000000 --- a/db-community-testing +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "community-testing" diff --git a/db-core b/db-core deleted file mode 100755 index eaa5032..0000000 --- a/db-core +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "core" diff --git a/db-extra b/db-extra deleted file mode 100755 index d14b50e..0000000 --- a/db-extra +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "extra" diff --git a/db-functions b/db-functions index 0c96913..16b5ad3 100644 --- a/db-functions +++ b/db-functions @@ -211,6 +211,18 @@ getpkgver() { echo "$_ver" } +getpkgarch() { + local _ver + + _ver="$(_grep_pkginfo "$1" "^arch")" + if [ -z "$_ver" ]; then + error "Package '$1' has no arch in the PKGINFO. Fail!" + exit 1 + fi + + echo "$_ver" +} + getpkgfile() { if [[ ${#} -ne 1 ]]; then error 'No canonical package found!' @@ -240,16 +252,40 @@ getpkgfiles() { echo ${@} } -#check_pkg_arch pkgfile arch -check_pkg_arch () { - local _arch - _arch="$(_grep_pkginfo "$1" "^arch")" +check_pkgfile() { + local pkgfile=$1 - if [ -z "$_arch" ]; then - error "Package '$1' has no arch in the PKGINFO. Fail!" + local pkgname="$(getpkgname ${pkgfile})" + [ $? -ge 1 ] && return 1 + local pkgver="$(getpkgver ${pkgfile})" + [ $? -ge 1 ] && return 1 + local pkgarch="$(getpkgarch ${pkgfile})" + [ $? -ge 1 ] && return 1 + + if echo "$(basename ${pkgfile})" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then + return 0 + else return 1 fi - if [ "$_arch" = "$2" ]; then +} + +check_pkgsvn() { + local pkgfile="${1}" + local pkgbase="$(getpkgbase $pkg)" + [ $? -ge 1 ] && return 1 + local pkgarch="$(getpkgarch ${pkgfile})" + [ $? -ge 1 ] && return 1 + local repo="${2}" + + if [ ! -f "${WORKDIR}/pkgbuilds/${pkgbase}" ]; then + mkdir -p "${WORKDIR}/pkgbuilds" + svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}/PKGBUILD" \ + "${WORKDIR}/pkgbuilds/${pkgbase}" >/dev/null + [ $? -ge 1 ] && return 1 + fi + + local pkgver="$(. "${WORKDIR}/pkgbuilds/${pkgbase}"; echo "${pkgver}-${pkgrel}")" + if echo "$(basename ${pkgfile})" | grep -q "${pkgver}-${pkgarch}"; then return 0 else return 1 diff --git a/db-testing b/db-testing deleted file mode 100755 index 579ae37..0000000 --- a/db-testing +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "testing" diff --git a/db-update b/db-update index 78a08b5..43c44f7 100755 --- a/db-update +++ b/db-update @@ -1,189 +1,78 @@ #!/bin/bash -if [ $# -ne 1 ]; then - msg "usage: $(basename $0) " - exit 1 -fi - . "$(dirname $0)/db-functions" . "$(dirname $0)/config" -reponame="$1" - -if ! check_repo_permission "$reponame"; then - error "you shouldn't be updating $reponame on this server!" +if [ $# -ge 1 ]; then + warning "Calling $(basename $0) with a specific repository is no longer supported" exit 1 fi -ADDPKGS="" -ANYPKGS="" - -stagedir="$STAGING/$reponame" -if [ ! -d $stagedir ]; then - error "staging directory missing: $stagedir" +# Find repos with packages to release +repos=($(find "${STAGING}" -mindepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +if [ $? -ge 1 ]; then + error "Could not read ${STAGING}" exit 1 fi -msg "Updating $reponame..." - -for f in $stagedir/*$PKGEXT; do - if [ -f "$FTP_BASE/$(get_pkgpool_for_host)/$(basename f)" ]; then - error "Package $(basename f) already exists" +for repo in ${repos[@]}; do + if ! check_repo_permission "${repo}"; then + error "You don't have permission to update packages in ${repo}" exit 1 fi -done - -# Process architecture-independent packages first. -ANYPKGS="$(getpkgfiles $stagedir/*-any$PKGEXT 2>/dev/null)" - -cd "$WORKDIR" - -if [ -n "$ANYPKGS" ]; then - /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null - cd checkout - to_add_any="" - for pkg in $ANYPKGS; do - pkgfile=$(basename $pkg) - pkgname="$(getpkgname $pkg)" - pkgbase="$(getpkgbase $pkg)" - svnrepo="$reponame-any" - if ! check_pkg_arch "$pkg" "any"; then - error "$pkgfile is not architecture independent!" - else - /usr/bin/svn up -q $pkgbase - if [ -d "$pkgbase/repos/$svnrepo" ]; then - pkgver=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgver}) - pkgrel=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgrel}) - if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-any"; then - to_add_any="$to_add_any $pkg" - else - warning "$pkgfile does not match PKGBUILD in $svnrepo" - fi - else - warning "Package $pkgbase not found in $svnrepo" + pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) + if [ $? -eq 0 ]; then + for pkg in ${pkgs[@]}; do + if ! check_pkgfile "${pkg}"; then + error "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" + exit 1 fi - fi - done -fi - -for current_arch in ${ARCHES[@]}; do - ftppath="$FTP_BASE/$reponame/os/$current_arch" - ftppath_any="$FTP_BASE/$reponame/os/any" - poolpath="$FTP_BASE/$(get_pkgpool_for_host)" - # The following is used to create relative symlinks - poolrel="../../../$(get_pkgpool_for_host)" - svnrepo="$reponame-$current_arch" - - repo_lock $reponame $current_arch || continue - - /bin/mkdir -p "$WORKDIR/build" - cd "$WORKDIR" - - # copy the db file into our working area - if [ -f "$ftppath/$reponame$DBEXT" ]; then - /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT - fi - - to_add="" - if [ -d "$stagedir" ]; then - ADDPKGS="$(getpkgfiles $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)" - fi - - if [ -n "$ADDPKGS" -o -n "$ANYPKGS" ]; then - - if [ -f "$ftppath/$reponame$DBEXT" ]; then - /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT - fi - - cd "$WORKDIR" - /usr/bin/svn checkout -N $SVNREPO checkout >/dev/null - cd checkout - - if [ -n "$ADDPKGS" ]; then - for pkg in $ADDPKGS; do - pkgfile=$(basename $pkg) - pkgname="$(getpkgname $pkg)" - pkgbase="$(getpkgbase $pkg)" - - if ! check_pkg_arch "$pkg" "$current_arch"; then - error "$pkgfile was built for the wrong architecture" - else - /usr/bin/svn up -q $pkgbase - if [ -d "$pkgbase/repos/$svnrepo" ]; then - pkgver=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgver}) - pkgrel=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgrel}) - if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-$current_arch"; then - to_add="$to_add $pkg" - else - warning "$pkgfile does not match PKGBUILD in $svnrepo" - fi - else - warning "Package $pkgbase not found in $svnrepo" - fi - fi - done - fi - - if [ -n "$to_add" -o -n "$to_add_any" ]; then - cd "$WORKDIR/build/" - for f in $to_add $to_add_any; do /bin/cp "$f" .; done - - pkgs="" - for pkg in $to_add $to_add_any; do pkgs="$pkgs $(basename $pkg)"; done - - /usr/bin/repo-add -q "$reponame-$current_arch$DBEXT" $pkgs >/dev/null - else - rm -f "build/$reponame-$current_arch$DBEXT" - error "Errors found when adding packages" - fi - else - warning "No packages to add" - fi - - # if non empty, move all build dirs - if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then - if [ $(getpkgfiles "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do - /bin/chmod 664 "$f" &>/dev/null - if ! /bin/cp "$f" "$poolpath/"; then - die "failure while copying files to $poolpath" - fi - fname="$(basename $f)" - if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then - die "failure symlinking $fname to $ftppath" - fi - done - fi - if [ $(getpkgfiles "$WORKDIR/build/"*-any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - for f in "$WORKDIR/build/"*-any$PKGEXT; do - /bin/chmod 664 "$f" &>/dev/null - fname="$(basename $f)" - if ! /bin/cp "$f" "$poolpath/"; then - die "failure while copying files to $poolpath" - fi - - if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then - die "failure symlinking $fname to $ftppath" - fi - done - fi - if ! /bin/cp "$WORKDIR/build/$reponame-$current_arch$DBEXT" "$ftppath/$reponame$DBEXT"; then - die "failed to move repository $reponame-$current_arch". - fi - ln -sf "$reponame$DBEXT" "$ftppath/$reponame${DBEXT%.tar.*}" + if ! check_pkgsvn "${pkg}" "${repo}"; then + error "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" + exit 1 + fi + if [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/$(basename ${pkg})" ]; then + error "Package ${repo}/$(basename pkg) already exists in ${FTP_BASE}/$(get_pkgpool_for_host)" + exit 1 + fi + done else - warning "Nothing to copy, no work done" - fi - - if [ -n "$to_add" ]; then - /bin/rm $to_add + exit 1 fi +done - repo_unlock $reponame $current_arch +# TODO: this might lock too much (architectures) +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_lock ${repo} ${pkgarch} || exit 1 + done done -if [ -n "$to_add_any" ]; then - /bin/rm $to_add_any -fi +for repo in ${repos[@]}; do + any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) + for pkgarch in ${ARCHES[@]}; do + msg "Updating [${repo}] (${pkgarch})..." + add_pkgs=() + arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) + for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do + pkgfile="$(basename ${pkg})" + msg2 "${pkgfile}" + # any packages might have been moved by the previous run + if [ -f "${pkg}" ]; then + mv "${pkg}" "$FTP_BASE/$(get_pkgpool_for_host)" + fi + ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" + add_pkgs[${#add_pkgs[*]}]=${pkgfile} + done + pushd "$FTP_BASE/$repo/os/${pkgarch}" >/dev/null + /usr/bin/repo-add -q "$repo$DBEXT" ${add_pkgs[@]} >/dev/null \ + || die "Could not add packages" + popd >/dev/null + done +done -# vim: set ts=4 sw=4 noet ft=sh: +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_unlock ${repo} ${pkgarch} + done +done diff --git a/test/runTest b/test/runTest index 75c7a05..06ca000 100755 --- a/test/runTest +++ b/test/runTest @@ -33,7 +33,7 @@ testAddSimplePackages() { done done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -50,7 +50,7 @@ testAddAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz @@ -70,7 +70,7 @@ testAddSplitPackages() { done done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -98,7 +98,7 @@ testRemovePackages() { done done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -121,7 +121,7 @@ testRemoveAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do ../db-remove ${pkgbase} extra any @@ -149,7 +149,7 @@ testMoveSimplePackages() { done done - ../db-update testing + ../db-update for arch in ${arches[@]}; do ../db-move pkg-simple-a testing extra ${arch} @@ -171,7 +171,7 @@ testMoveAnyPackages() { releasePackage testing ${pkgbase} any done - ../db-update testing + ../db-update ../db-move pkg-any-a testing extra any checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz @@ -192,7 +192,7 @@ testMoveSplitPackages() { done done - ../db-update testing + ../db-update for arch in ${arches[@]}; do ../db-move pkg-split-a testing extra ${arch} @@ -229,7 +229,7 @@ testCleanupSimplePackages() { done done - ../db-update extra + ../db-update for arch in ${arches[@]}; do ../db-remove pkg-simple-a extra ${arch} @@ -257,7 +257,7 @@ testCleanupAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update extra + ../db-update ../db-remove pkg-any-a extra any ../cron-jobs/ftpdir-cleanup >/dev/null @@ -283,7 +283,7 @@ testCleanupSplitPackages() { done done - ../db-update extra + ../db-update for arch in ${arches[@]}; do ../db-remove ${pkgs[0]} extra ${arch} @@ -322,7 +322,7 @@ testMovePackagesWithoutPool() { done done - ../db-update testing + ../db-update # transform two packages to old style layout for arch in ${arches[@]}; do @@ -364,7 +364,7 @@ testMoveAnyPackagesWithoutPool() { releasePackage testing ${pkgbase} any done - ../db-update testing + ../db-update # transform a package to old style layout for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do -- cgit v1.2.3-54-g00ecf