summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2010-08-14 20:15:50 +0200
committerPierre Schmitz <pierre@archlinux.de>2010-08-14 20:15:50 +0200
commitcdf17a4c4c83c1363a59fa688654276c3d807eae (patch)
treed46f88d11558362bc6e3ea90b5998a89eef541e9
parent27abfce26dc4f667fb4702db40acf27317ecfd9f (diff)
Rewrite of db-update
* db-update now updates all repos with packages in its staging dirs * sanity checks are performed before any repo is touched * improved performance * less code; easier to maintain
-rwxr-xr-xdb-community3
-rwxr-xr-xdb-community-testing3
-rwxr-xr-xdb-core3
-rwxr-xr-xdb-extra3
-rw-r--r--db-functions50
-rwxr-xr-xdb-testing3
-rwxr-xr-xdb-update227
-rwxr-xr-xtest/runTest26
8 files changed, 114 insertions, 204 deletions
diff --git a/db-community b/db-community
deleted file mode 100755
index 44767a5..0000000
--- a/db-community
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-$(dirname $0)/db-update "community"
diff --git a/db-community-testing b/db-community-testing
deleted file mode 100755
index eb4cf67..0000000
--- a/db-community-testing
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-$(dirname $0)/db-update "community-testing"
diff --git a/db-core b/db-core
deleted file mode 100755
index eaa5032..0000000
--- a/db-core
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-$(dirname $0)/db-update "core"
diff --git a/db-extra b/db-extra
deleted file mode 100755
index d14b50e..0000000
--- a/db-extra
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-$(dirname $0)/db-update "extra"
diff --git a/db-functions b/db-functions
index 0c96913..16b5ad3 100644
--- a/db-functions
+++ b/db-functions
@@ -211,6 +211,18 @@ getpkgver() {
echo "$_ver"
}
+getpkgarch() {
+ local _ver
+
+ _ver="$(_grep_pkginfo "$1" "^arch")"
+ if [ -z "$_ver" ]; then
+ error "Package '$1' has no arch in the PKGINFO. Fail!"
+ exit 1
+ fi
+
+ echo "$_ver"
+}
+
getpkgfile() {
if [[ ${#} -ne 1 ]]; then
error 'No canonical package found!'
@@ -240,16 +252,40 @@ getpkgfiles() {
echo ${@}
}
-#check_pkg_arch pkgfile arch
-check_pkg_arch () {
- local _arch
- _arch="$(_grep_pkginfo "$1" "^arch")"
+check_pkgfile() {
+ local pkgfile=$1
- if [ -z "$_arch" ]; then
- error "Package '$1' has no arch in the PKGINFO. Fail!"
+ local pkgname="$(getpkgname ${pkgfile})"
+ [ $? -ge 1 ] && return 1
+ local pkgver="$(getpkgver ${pkgfile})"
+ [ $? -ge 1 ] && return 1
+ local pkgarch="$(getpkgarch ${pkgfile})"
+ [ $? -ge 1 ] && return 1
+
+ if echo "$(basename ${pkgfile})" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then
+ return 0
+ else
return 1
fi
- if [ "$_arch" = "$2" ]; then
+}
+
+check_pkgsvn() {
+ local pkgfile="${1}"
+ local pkgbase="$(getpkgbase $pkg)"
+ [ $? -ge 1 ] && return 1
+ local pkgarch="$(getpkgarch ${pkgfile})"
+ [ $? -ge 1 ] && return 1
+ local repo="${2}"
+
+ if [ ! -f "${WORKDIR}/pkgbuilds/${pkgbase}" ]; then
+ mkdir -p "${WORKDIR}/pkgbuilds"
+ svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}/PKGBUILD" \
+ "${WORKDIR}/pkgbuilds/${pkgbase}" >/dev/null
+ [ $? -ge 1 ] && return 1
+ fi
+
+ local pkgver="$(. "${WORKDIR}/pkgbuilds/${pkgbase}"; echo "${pkgver}-${pkgrel}")"
+ if echo "$(basename ${pkgfile})" | grep -q "${pkgver}-${pkgarch}"; then
return 0
else
return 1
diff --git a/db-testing b/db-testing
deleted file mode 100755
index 579ae37..0000000
--- a/db-testing
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-$(dirname $0)/db-update "testing"
diff --git a/db-update b/db-update
index 78a08b5..43c44f7 100755
--- a/db-update
+++ b/db-update
@@ -1,189 +1,78 @@
#!/bin/bash
-if [ $# -ne 1 ]; then
- msg "usage: $(basename $0) <reponame>"
- exit 1
-fi
-
. "$(dirname $0)/db-functions"
. "$(dirname $0)/config"
-reponame="$1"
-
-if ! check_repo_permission "$reponame"; then
- error "you shouldn't be updating $reponame on this server!"
+if [ $# -ge 1 ]; then
+ warning "Calling $(basename $0) with a specific repository is no longer supported"
exit 1
fi
-ADDPKGS=""
-ANYPKGS=""
-
-stagedir="$STAGING/$reponame"
-if [ ! -d $stagedir ]; then
- error "staging directory missing: $stagedir"
+# Find repos with packages to release
+repos=($(find "${STAGING}" -mindepth 1 -type d ! -empty -printf '%f ' 2>/dev/null))
+if [ $? -ge 1 ]; then
+ error "Could not read ${STAGING}"
exit 1
fi
-msg "Updating $reponame..."
-
-for f in $stagedir/*$PKGEXT; do
- if [ -f "$FTP_BASE/$(get_pkgpool_for_host)/$(basename f)" ]; then
- error "Package $(basename f) already exists"
+for repo in ${repos[@]}; do
+ if ! check_repo_permission "${repo}"; then
+ error "You don't have permission to update packages in ${repo}"
exit 1
fi
-done
-
-# Process architecture-independent packages first.
-ANYPKGS="$(getpkgfiles $stagedir/*-any$PKGEXT 2>/dev/null)"
-
-cd "$WORKDIR"
-
-if [ -n "$ANYPKGS" ]; then
- /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null
- cd checkout
- to_add_any=""
- for pkg in $ANYPKGS; do
- pkgfile=$(basename $pkg)
- pkgname="$(getpkgname $pkg)"
- pkgbase="$(getpkgbase $pkg)"
- svnrepo="$reponame-any"
- if ! check_pkg_arch "$pkg" "any"; then
- error "$pkgfile is not architecture independent!"
- else
- /usr/bin/svn up -q $pkgbase
- if [ -d "$pkgbase/repos/$svnrepo" ]; then
- pkgver=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgver})
- pkgrel=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgrel})
- if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-any"; then
- to_add_any="$to_add_any $pkg"
- else
- warning "$pkgfile does not match PKGBUILD in $svnrepo"
- fi
- else
- warning "Package $pkgbase not found in $svnrepo"
+ pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT}))
+ if [ $? -eq 0 ]; then
+ for pkg in ${pkgs[@]}; do
+ if ! check_pkgfile "${pkg}"; then
+ error "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data"
+ exit 1
fi
- fi
- done
-fi
-
-for current_arch in ${ARCHES[@]}; do
- ftppath="$FTP_BASE/$reponame/os/$current_arch"
- ftppath_any="$FTP_BASE/$reponame/os/any"
- poolpath="$FTP_BASE/$(get_pkgpool_for_host)"
- # The following is used to create relative symlinks
- poolrel="../../../$(get_pkgpool_for_host)"
- svnrepo="$reponame-$current_arch"
-
- repo_lock $reponame $current_arch || continue
-
- /bin/mkdir -p "$WORKDIR/build"
- cd "$WORKDIR"
-
- # copy the db file into our working area
- if [ -f "$ftppath/$reponame$DBEXT" ]; then
- /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT
- fi
-
- to_add=""
- if [ -d "$stagedir" ]; then
- ADDPKGS="$(getpkgfiles $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)"
- fi
-
- if [ -n "$ADDPKGS" -o -n "$ANYPKGS" ]; then
-
- if [ -f "$ftppath/$reponame$DBEXT" ]; then
- /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT
- fi
-
- cd "$WORKDIR"
- /usr/bin/svn checkout -N $SVNREPO checkout >/dev/null
- cd checkout
-
- if [ -n "$ADDPKGS" ]; then
- for pkg in $ADDPKGS; do
- pkgfile=$(basename $pkg)
- pkgname="$(getpkgname $pkg)"
- pkgbase="$(getpkgbase $pkg)"
-
- if ! check_pkg_arch "$pkg" "$current_arch"; then
- error "$pkgfile was built for the wrong architecture"
- else
- /usr/bin/svn up -q $pkgbase
- if [ -d "$pkgbase/repos/$svnrepo" ]; then
- pkgver=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgver})
- pkgrel=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgrel})
- if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-$current_arch"; then
- to_add="$to_add $pkg"
- else
- warning "$pkgfile does not match PKGBUILD in $svnrepo"
- fi
- else
- warning "Package $pkgbase not found in $svnrepo"
- fi
- fi
- done
- fi
-
- if [ -n "$to_add" -o -n "$to_add_any" ]; then
- cd "$WORKDIR/build/"
- for f in $to_add $to_add_any; do /bin/cp "$f" .; done
-
- pkgs=""
- for pkg in $to_add $to_add_any; do pkgs="$pkgs $(basename $pkg)"; done
-
- /usr/bin/repo-add -q "$reponame-$current_arch$DBEXT" $pkgs >/dev/null
- else
- rm -f "build/$reponame-$current_arch$DBEXT"
- error "Errors found when adding packages"
- fi
- else
- warning "No packages to add"
- fi
-
- # if non empty, move all build dirs
- if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then
- if [ $(getpkgfiles "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then
- for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do
- /bin/chmod 664 "$f" &>/dev/null
- if ! /bin/cp "$f" "$poolpath/"; then
- die "failure while copying files to $poolpath"
- fi
- fname="$(basename $f)"
- if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then
- die "failure symlinking $fname to $ftppath"
- fi
- done
- fi
- if [ $(getpkgfiles "$WORKDIR/build/"*-any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then
- for f in "$WORKDIR/build/"*-any$PKGEXT; do
- /bin/chmod 664 "$f" &>/dev/null
- fname="$(basename $f)"
- if ! /bin/cp "$f" "$poolpath/"; then
- die "failure while copying files to $poolpath"
- fi
-
- if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then
- die "failure symlinking $fname to $ftppath"
- fi
- done
- fi
- if ! /bin/cp "$WORKDIR/build/$reponame-$current_arch$DBEXT" "$ftppath/$reponame$DBEXT"; then
- die "failed to move repository $reponame-$current_arch".
- fi
- ln -sf "$reponame$DBEXT" "$ftppath/$reponame${DBEXT%.tar.*}"
+ if ! check_pkgsvn "${pkg}" "${repo}"; then
+ error "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository"
+ exit 1
+ fi
+ if [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/$(basename ${pkg})" ]; then
+ error "Package ${repo}/$(basename pkg) already exists in ${FTP_BASE}/$(get_pkgpool_for_host)"
+ exit 1
+ fi
+ done
else
- warning "Nothing to copy, no work done"
- fi
-
- if [ -n "$to_add" ]; then
- /bin/rm $to_add
+ exit 1
fi
+done
- repo_unlock $reponame $current_arch
+# TODO: this might lock too much (architectures)
+for repo in ${repos[@]}; do
+ for pkgarch in ${ARCHES[@]}; do
+ repo_lock ${repo} ${pkgarch} || exit 1
+ done
done
-if [ -n "$to_add_any" ]; then
- /bin/rm $to_add_any
-fi
+for repo in ${repos[@]}; do
+ any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null))
+ for pkgarch in ${ARCHES[@]}; do
+ msg "Updating [${repo}] (${pkgarch})..."
+ add_pkgs=()
+ arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null))
+ for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do
+ pkgfile="$(basename ${pkg})"
+ msg2 "${pkgfile}"
+ # any packages might have been moved by the previous run
+ if [ -f "${pkg}" ]; then
+ mv "${pkg}" "$FTP_BASE/$(get_pkgpool_for_host)"
+ fi
+ ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}"
+ add_pkgs[${#add_pkgs[*]}]=${pkgfile}
+ done
+ pushd "$FTP_BASE/$repo/os/${pkgarch}" >/dev/null
+ /usr/bin/repo-add -q "$repo$DBEXT" ${add_pkgs[@]} >/dev/null \
+ || die "Could not add packages"
+ popd >/dev/null
+ done
+done
-# vim: set ts=4 sw=4 noet ft=sh:
+for repo in ${repos[@]}; do
+ for pkgarch in ${ARCHES[@]}; do
+ repo_unlock ${repo} ${pkgarch}
+ done
+done
diff --git a/test/runTest b/test/runTest
index 75c7a05..06ca000 100755
--- a/test/runTest
+++ b/test/runTest
@@ -33,7 +33,7 @@ testAddSimplePackages() {
done
done
- ../db-update extra
+ ../db-update
for pkgbase in ${pkgs[@]}; do
for arch in ${arches[@]}; do
@@ -50,7 +50,7 @@ testAddAnyPackages() {
releasePackage extra ${pkgbase} any
done
- ../db-update extra
+ ../db-update
for pkgbase in ${pkgs[@]}; do
checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz
@@ -70,7 +70,7 @@ testAddSplitPackages() {
done
done
- ../db-update extra
+ ../db-update
for pkgbase in ${pkgs[@]}; do
for arch in ${arches[@]}; do
@@ -98,7 +98,7 @@ testRemovePackages() {
done
done
- ../db-update extra
+ ../db-update
for pkgbase in ${pkgs[@]}; do
for arch in ${arches[@]}; do
@@ -121,7 +121,7 @@ testRemoveAnyPackages() {
releasePackage extra ${pkgbase} any
done
- ../db-update extra
+ ../db-update
for pkgbase in ${pkgs[@]}; do
../db-remove ${pkgbase} extra any
@@ -149,7 +149,7 @@ testMoveSimplePackages() {
done
done
- ../db-update testing
+ ../db-update
for arch in ${arches[@]}; do
../db-move pkg-simple-a testing extra ${arch}
@@ -171,7 +171,7 @@ testMoveAnyPackages() {
releasePackage testing ${pkgbase} any
done
- ../db-update testing
+ ../db-update
../db-move pkg-any-a testing extra any
checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz
@@ -192,7 +192,7 @@ testMoveSplitPackages() {
done
done
- ../db-update testing
+ ../db-update
for arch in ${arches[@]}; do
../db-move pkg-split-a testing extra ${arch}
@@ -229,7 +229,7 @@ testCleanupSimplePackages() {
done
done
- ../db-update extra
+ ../db-update
for arch in ${arches[@]}; do
../db-remove pkg-simple-a extra ${arch}
@@ -257,7 +257,7 @@ testCleanupAnyPackages() {
releasePackage extra ${pkgbase} any
done
- ../db-update extra
+ ../db-update
../db-remove pkg-any-a extra any
../cron-jobs/ftpdir-cleanup >/dev/null
@@ -283,7 +283,7 @@ testCleanupSplitPackages() {
done
done
- ../db-update extra
+ ../db-update
for arch in ${arches[@]}; do
../db-remove ${pkgs[0]} extra ${arch}
@@ -322,7 +322,7 @@ testMovePackagesWithoutPool() {
done
done
- ../db-update testing
+ ../db-update
# transform two packages to old style layout
for arch in ${arches[@]}; do
@@ -364,7 +364,7 @@ testMoveAnyPackagesWithoutPool() {
releasePackage testing ${pkgbase} any
done
- ../db-update testing
+ ../db-update
# transform a package to old style layout
for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do