summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2010-11-20 16:11:59 +0100
committerPierre Schmitz <pierre@archlinux.de>2010-11-20 16:11:59 +0100
commitcb2dcc6ee207e9c5ba4b875d70e387e6347591ed (patch)
tree5874d805bb6fdca2011417fc23f34a09a943d458
parenta1ba979a0e3c763449d91fc787333f4058e89aa8 (diff)
Fix sourceballs cron job
* add unit test for sourceballs and cleanup * introduce SRCPOOL and LOGDIR variables in config
-rw-r--r--config2
-rw-r--r--config.local.gerolde1
-rw-r--r--config.local.sigurd1
-rwxr-xr-xcron-jobs/sourceballs21
-rwxr-xr-xmisc-scripts/make-sourceball13
-rwxr-xr-xmisc-scripts/sourceballs-cleanup9
-rw-r--r--test/lib/common.inc7
-rwxr-xr-xtest/runTest59
8 files changed, 84 insertions, 29 deletions
diff --git a/config b/config
index 89e89fd..efc4381 100644
--- a/config
+++ b/config
@@ -5,6 +5,7 @@ SVNREPOCOMMUNITY="file:///srv/svn-community"
PKGREPOS=()
PKGPOOL=''
+SRCPOOL=''
CLEANUP_DESTDIR="/srv/package-cleanup"
CLEANUP_DRYRUN=false
@@ -15,6 +16,7 @@ LOCK_TIMEOUT=300
STAGING="$HOME/staging"
TMPDIR="/srv/tmp"
+LOGDIR="/var/log/dbscripts"
ARCHES=(i686 x86_64)
DBEXT=".db.tar.gz"
PKGEXT=".pkg.tar.*"
diff --git a/config.local.gerolde b/config.local.gerolde
index dd8ba86..03e8a70 100644
--- a/config.local.gerolde
+++ b/config.local.gerolde
@@ -1,2 +1,3 @@
PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable')
PKGPOOL='pool/packages'
+SRCPOOL='sources/packages'
diff --git a/config.local.sigurd b/config.local.sigurd
index c8c3c9f..ddb5ba4 100644
--- a/config.local.sigurd
+++ b/config.local.sigurd
@@ -1,2 +1,3 @@
PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing')
PKGPOOL='pool/community'
+SRCPOOL='sources/community'
diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs
index 0b01321..4fc194f 100755
--- a/cron-jobs/sourceballs
+++ b/cron-jobs/sourceballs
@@ -3,9 +3,6 @@
. "$(dirname $0)/../db-functions"
. "$(dirname $0)/../config"
-ftpbase="${FTP_BASE}"
-srcbase="${FTP_BASE}/sources"
-
script_lock
set_umask
@@ -14,12 +11,12 @@ dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))"
FAILED_PKGS=""
-[ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old"
-echo "Errors occured during run:" > "$srcbase/errors.txt"
+[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs"
+[ -e "${LOGDIR}/sourceballs/errors.txt" ] && /bin/mv "${LOGDIR}/sourceballs/errors.txt" "${LOGDIR}/sourceballs/errors.txt.old"
for repo in ${PKGREPOS[@]}; do
- for arch in ${ARCHES[@]} any; do
- ftppath="$ftpbase/$repo/os/$arch"
+ for arch in ${ARCHES[@]}; do
+ ftppath="${FTP_BASE}/$repo/os/$arch"
if [ ! -d "$ftppath" ]; then
error "FTP path does not exist: $ftppath"
continue
@@ -28,7 +25,6 @@ for repo in ${PKGREPOS[@]}; do
for pkg in *$PKGEXT; do
[ -f "$pkg" ] || continue
pkgbase=$(getpkgbase $pkg)
- srcpath="$srcbase/"
srcpkg="${pkg//$PKGEXT/$SRCEXT}"
srcpkg="${srcpkg//-$arch/}"
srcpkgname="${srcpkg%-*-*$SRCEXT}"
@@ -51,9 +47,9 @@ for repo in ${PKGREPOS[@]}; do
force="-f"
fi
- if [ ! \( -f "$srcpath$srcpkg" -o -f "$srcpath$srcpkgbase" \) ]; then
+ if [ ! \( -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" -o -f "${FTP_BASE}/${SRCPOOL}/$srcpkgbase" \) ]; then
if ! $dirname/../misc-scripts/make-sourceball $force \
- $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then
+ $pkgbase $repo $arch 2>>"${LOGDIR}/sourceballs/errors.txt"; then
FAILED_PKGS="$FAILED_PKGS $pkgbase"
fi
fi
@@ -62,9 +58,8 @@ for repo in ${PKGREPOS[@]}; do
done
if [ -n "$FAILED_PKGS" ]; then
- [ -e "$srcbase/failed.txt" ] && /bin/mv "$srcbase/failed.txt" "$srcbase/failed.txt.old"
- echo "The following package bases failed:" > "$srcbase/failed.txt"
- echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt"
+ [ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old"
+ echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "${LOGDIR}/sourceballs/failed.txt"
fi
$dirname/../misc-scripts/sourceballs-cleanup
diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball
index 5bd155f..221d0ce 100755
--- a/misc-scripts/make-sourceball
+++ b/misc-scripts/make-sourceball
@@ -19,9 +19,6 @@ packagename="$1"
reponame="$2"
arch="$3"
-srcpath="$FTP_BASE/sources/"
-logpath="/var/log/sourceballs/"
-
script_lock
create_srcpackage() {
@@ -29,6 +26,7 @@ create_srcpackage() {
pushd "$1" >/dev/null
pkgver=$(. PKGBUILD; echo ${pkgver})
pkgrel=$(. PKGBUILD; echo ${pkgrel})
+ license=($(. PKGBUILD; echo ${license[@]}))
if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then
#Removed so as not to clutter failed.txt
#warning "$packagename license (${license[@]}) does not require source tarballs"
@@ -37,7 +35,7 @@ create_srcpackage() {
msg "Creating source tarball for $packagename-$pkgver-$pkgrel"
fi
- local logfile="$logpath/$packagename"
+ local logfile="${LOGDIR}/sourceballs/$packagename"
if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then
popd >/dev/null
/bin/gzip -f -9 "$logfile"
@@ -47,10 +45,7 @@ create_srcpackage() {
local pkg_file="${packagename}-${pkgver}-${pkgrel}${SRCEXT}"
- if [ ! -d "$srcpath" ]; then
- mkdir -p "$srcpath"
- fi
- cp "$pkg_file" "$srcpath"
+ cp "$pkg_file" "${FTP_BASE}/${SRCPOOL}"
popd >/dev/null
@@ -59,7 +54,7 @@ create_srcpackage() {
}
set_umask
-/bin/mkdir -p "$logpath"
+[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs"
cd "$WORKDIR"
if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then
diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup
index e407989..788cc98 100755
--- a/misc-scripts/sourceballs-cleanup
+++ b/misc-scripts/sourceballs-cleanup
@@ -3,8 +3,8 @@
. "$(dirname $0)/../db-functions"
. "$(dirname $0)/../config"
-srcpath="$FTP_BASE/sources/"
-logfile="$srcpath/cleanup.txt"
+[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs"
+logfile="${LOGDIR}/sourceballs/cleanup.txt"
script_lock
@@ -20,7 +20,7 @@ remove_old() {
cd ..
done
- for srcpkg in "$srcpath/$packagename-"*; do
+ for srcpkg in "${FTP_BASE}/${SRCPOOL}/$packagename-"*; do
[ -f "$srcpkg" ] || continue
if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then
skip=0
@@ -47,10 +47,11 @@ remove_old() {
set_umask
cd "$WORKDIR"
+
[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old"
echo "Orphaned sourceballs:" > "$logfile"
-for sourceball in "$srcpath"/*$SRCEXT; do
+for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do
packagename=$(basename $sourceball)
packagename=${packagename%-*-*$SRCEXT}
diff --git a/test/lib/common.inc b/test/lib/common.inc
index a0d864a..14dc000 100644
--- a/test/lib/common.inc
+++ b/test/lib/common.inc
@@ -43,9 +43,9 @@ setUp() {
mkdir -p "${TMP}/ftp/${r}/os/${a}"
done
done
- for p in ${PKGPOOL[@]}; do
- mkdir -p "${TMP}/ftp/${p}"
- done
+ mkdir -p "${TMP}/ftp/${PKGPOOL}"
+ mkdir -p "${TMP}/ftp/${SRCPOOL}"
+ mkdir -p "${TMP}/log"
msg 'Creating svn repository...'
svnadmin create "${TMP}/svn-packages-repo"
@@ -69,6 +69,7 @@ setUp() {
SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup"
STAGING="${TMP}/staging"
TMPDIR="${TMP}/tmp"
+ LOGDIR="${TMP}/log"
CLEANUP_DRYRUN=false
eot
. "${curdir}/../config"
diff --git a/test/runTest b/test/runTest
index 4681944..d3e2eff 100755
--- a/test/runTest
+++ b/test/runTest
@@ -535,5 +535,64 @@ testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() {
done
}
+#
+#sourceballs tests
+#
+
+testSourceballs() {
+ local arches=('i686' 'x86_64')
+ local pkgs=('pkg-simple-a' 'pkg-simple-b')
+ local pkgbase
+ local arch
+
+ for pkgbase in ${pkgs[@]}; do
+ for arch in ${arches[@]}; do
+ releasePackage extra ${pkgbase} ${arch}
+ done
+ done
+ ../db-update
+
+ ../cron-jobs/sourceballs
+
+ for log in errors failed; do
+ if [ -s "${LOGDIR}/sourceballs/${log}.txt" ]; then
+ fail "${LOGDIR}/sourceballs/${log}.txt includes the following errors:"
+ cat "${LOGDIR}/sourceballs/${log}.txt"
+ fi
+ done
+ for pkgbase in ${pkgs[@]}; do
+ [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!"
+ done
+}
+
+testSourceballsCleanup() {
+ local arches=('i686' 'x86_64')
+ local pkgs=('pkg-simple-a' 'pkg-simple-b')
+ local pkgbase
+ local arch
+
+ for pkgbase in ${pkgs[@]}; do
+ for arch in ${arches[@]}; do
+ releasePackage extra ${pkgbase} ${arch}
+ done
+ done
+ ../db-update
+ ../cron-jobs/sourceballs
+
+ for arch in ${arches[@]}; do
+ ../db-remove pkg-simple-a extra ${arch}
+ done
+ ../cron-jobs/ftpdir-cleanup >/dev/null
+
+ ../cron-jobs/sourceballs
+ for log in errors failed; do
+ if [ -s "${LOGDIR}/sourceballs/${log}.txt" ]; then
+ fail "${LOGDIR}/sourceballs/${log}.txt includes the following errors:"
+ cat "${LOGDIR}/sourceballs/${log}.txt"
+ fi
+ done
+ [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!"
+ [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!"
+}
. "${curdir}/lib/shunit2"