Fix sourceballs cron job

* add unit test for sourceballs and cleanup
* introduce SRCPOOL and LOGDIR variables in config

8 files changed, 84 insertions, 29 deletions

diff --git a/config b/config
index 89e89fd..efc4381 100644
--- a/config
+++ b/config
@@ -5,6 +5,7 @@ SVNREPOCOMMUNITY="file:///srv/svn-community"
 
 PKGREPOS=()
 PKGPOOL=''
+SRCPOOL=''
 
 CLEANUP_DESTDIR="/srv/package-cleanup"
 CLEANUP_DRYRUN=false
@@ -15,6 +16,7 @@ LOCK_TIMEOUT=300
 
 STAGING="$HOME/staging"
 TMPDIR="/srv/tmp"
+LOGDIR="/var/log/dbscripts"
 ARCHES=(i686 x86_64)
 DBEXT=".db.tar.gz"
 PKGEXT=".pkg.tar.*"
diff --git a/config.local.gerolde b/config.local.gerolde
index dd8ba86..03e8a70 100644
--- a/config.local.gerolde
+++ b/config.local.gerolde
@@ -1,2 +1,3 @@
 PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable')
 PKGPOOL='pool/packages'
+SRCPOOL='sources/packages'
diff --git a/config.local.sigurd b/config.local.sigurd
index c8c3c9f..ddb5ba4 100644
--- a/config.local.sigurd
+++ b/config.local.sigurd
@@ -1,2 +1,3 @@
 PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing')
 PKGPOOL='pool/community'
+SRCPOOL='sources/community'
diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs
index 0b01321..4fc194f 100755
--- a/cron-jobs/sourceballs
+++ b/cron-jobs/sourceballs
@@ -3,9 +3,6 @@
 . "$(dirname $0)/../db-functions"
 . "$(dirname $0)/../config"
 
-ftpbase="${FTP_BASE}"
-srcbase="${FTP_BASE}/sources"
-
 script_lock
 
 set_umask
@@ -14,12 +11,12 @@ dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))"
 
 FAILED_PKGS=""
 
-[ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old"
-echo "Errors occured during run:" > "$srcbase/errors.txt"
+[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs"
+[ -e "${LOGDIR}/sourceballs/errors.txt" ] && /bin/mv "${LOGDIR}/sourceballs/errors.txt" "${LOGDIR}/sourceballs/errors.txt.old"
 
 for repo in ${PKGREPOS[@]}; do
-    for arch in ${ARCHES[@]} any; do
-        ftppath="$ftpbase/$repo/os/$arch"
+    for arch in ${ARCHES[@]}; do
+        ftppath="${FTP_BASE}/$repo/os/$arch"
         if [ ! -d "$ftppath" ]; then
             error "FTP path does not exist: $ftppath"
             continue
@@ -28,7 +25,6 @@ for repo in ${PKGREPOS[@]}; do
         for pkg in *$PKGEXT; do
             [ -f "$pkg" ] || continue
             pkgbase=$(getpkgbase $pkg)
-            srcpath="$srcbase/"
             srcpkg="${pkg//$PKGEXT/$SRCEXT}"
             srcpkg="${srcpkg//-$arch/}"
             srcpkgname="${srcpkg%-*-*$SRCEXT}"
@@ -51,9 +47,9 @@ for repo in ${PKGREPOS[@]}; do
                 force="-f"
             fi
 
-            if [ ! \( -f "$srcpath$srcpkg" -o -f "$srcpath$srcpkgbase" \) ]; then
+            if [ ! \( -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" -o -f "${FTP_BASE}/${SRCPOOL}/$srcpkgbase" \) ]; then
                 if ! $dirname/../misc-scripts/make-sourceball $force \
-                    $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then
+                    $pkgbase $repo $arch 2>>"${LOGDIR}/sourceballs/errors.txt"; then
                     FAILED_PKGS="$FAILED_PKGS $pkgbase"
                 fi
             fi
@@ -62,9 +58,8 @@ for repo in ${PKGREPOS[@]}; do
 done
 
 if [ -n "$FAILED_PKGS" ]; then
-    [ -e "$srcbase/failed.txt" ] && /bin/mv "$srcbase/failed.txt" "$srcbase/failed.txt.old"
-    echo "The following package bases failed:" > "$srcbase/failed.txt"
-    echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt"
+    [ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old"
+    echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "${LOGDIR}/sourceballs/failed.txt"
 fi
 
 $dirname/../misc-scripts/sourceballs-cleanup
diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball
index 5bd155f..221d0ce 100755
--- a/misc-scripts/make-sourceball
+++ b/misc-scripts/make-sourceball
@@ -19,9 +19,6 @@ packagename="$1"
 reponame="$2"
 arch="$3"
 
-srcpath="$FTP_BASE/sources/"
-logpath="/var/log/sourceballs/"
-
 script_lock
 
 create_srcpackage() {
@@ -29,6 +26,7 @@ create_srcpackage() {
         pushd "$1" >/dev/null
 	pkgver=$(. PKGBUILD; echo ${pkgver})
 	pkgrel=$(. PKGBUILD; echo ${pkgrel})
+	license=($(. PKGBUILD; echo ${license[@]}))
 	if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then
             #Removed so as not to clutter failed.txt
             #warning "$packagename license (${license[@]}) does not require source tarballs"
@@ -37,7 +35,7 @@ create_srcpackage() {
             msg "Creating source tarball for $packagename-$pkgver-$pkgrel"
         fi
 
-        local logfile="$logpath/$packagename"
+        local logfile="${LOGDIR}/sourceballs/$packagename"
         if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then
             popd >/dev/null
             /bin/gzip -f -9 "$logfile"
@@ -47,10 +45,7 @@ create_srcpackage() {
 
         local pkg_file="${packagename}-${pkgver}-${pkgrel}${SRCEXT}"
 
-        if [ ! -d "$srcpath" ]; then
-            mkdir -p "$srcpath"
-        fi
-        cp "$pkg_file" "$srcpath"
+        cp "$pkg_file" "${FTP_BASE}/${SRCPOOL}"
 
         popd >/dev/null
 
@@ -59,7 +54,7 @@ create_srcpackage() {
 }
 
 set_umask
-/bin/mkdir -p "$logpath"
+[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs"
 cd "$WORKDIR"
 
 if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then
diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup
index e407989..788cc98 100755
--- a/misc-scripts/sourceballs-cleanup
+++ b/misc-scripts/sourceballs-cleanup
@@ -3,8 +3,8 @@
 . "$(dirname $0)/../db-functions"
 . "$(dirname $0)/../config"
 
-srcpath="$FTP_BASE/sources/"
-logfile="$srcpath/cleanup.txt"
+[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs"
+logfile="${LOGDIR}/sourceballs/cleanup.txt"
 
 script_lock
 
@@ -20,7 +20,7 @@ remove_old() {
             cd ..
         done
 
-        for srcpkg in "$srcpath/$packagename-"*; do
+        for srcpkg in "${FTP_BASE}/${SRCPOOL}/$packagename-"*; do
             [ -f "$srcpkg" ] || continue
             if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then
                 skip=0
@@ -47,10 +47,11 @@ remove_old() {
 set_umask
 cd "$WORKDIR"
 
+
 [ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old"
 echo  "Orphaned sourceballs:" > "$logfile"
 
-for sourceball in "$srcpath"/*$SRCEXT; do
+for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do
     packagename=$(basename $sourceball)
     packagename=${packagename%-*-*$SRCEXT}
 
diff --git a/test/lib/common.inc b/test/lib/common.inc
index a0d864a..14dc000 100644
--- a/test/lib/common.inc
+++ b/test/lib/common.inc
@@ -43,9 +43,9 @@ setUp() {
 			mkdir -p "${TMP}/ftp/${r}/os/${a}"
 		done
 	done
-	for p in ${PKGPOOL[@]}; do
-		mkdir -p "${TMP}/ftp/${p}"
-	done
+	mkdir -p "${TMP}/ftp/${PKGPOOL}"
+	mkdir -p "${TMP}/ftp/${SRCPOOL}"
+	mkdir -p "${TMP}/log"
 
 	msg 'Creating svn repository...'
 	svnadmin create "${TMP}/svn-packages-repo"
@@ -69,6 +69,7 @@ setUp() {
 	SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup"
 	STAGING="${TMP}/staging"
 	TMPDIR="${TMP}/tmp"
+	LOGDIR="${TMP}/log"
 	CLEANUP_DRYRUN=false
 eot
 	. "${curdir}/../config"
diff --git a/test/runTest b/test/runTest
index 4681944..d3e2eff 100755
--- a/test/runTest
+++ b/test/runTest
@@ -535,5 +535,64 @@ testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() {
 	done
 }
 
+#
+#sourceballs tests
+#
+
+testSourceballs() {
+	local arches=('i686' 'x86_64')
+	local pkgs=('pkg-simple-a' 'pkg-simple-b')
+	local pkgbase
+	local arch
+
+	for pkgbase in ${pkgs[@]}; do
+		for arch in ${arches[@]}; do
+			releasePackage extra ${pkgbase} ${arch}
+		done
+	done
+	../db-update
+
+	../cron-jobs/sourceballs
+
+	for log in errors failed; do
+		if [ -s "${LOGDIR}/sourceballs/${log}.txt" ]; then
+			fail "${LOGDIR}/sourceballs/${log}.txt includes the following errors:"
+			cat "${LOGDIR}/sourceballs/${log}.txt"
+		fi
+	done
+	for pkgbase in ${pkgs[@]}; do
+		[ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!"
+	done
+}
+
+testSourceballsCleanup() {
+	local arches=('i686' 'x86_64')
+	local pkgs=('pkg-simple-a' 'pkg-simple-b')
+	local pkgbase
+	local arch
+
+	for pkgbase in ${pkgs[@]}; do
+		for arch in ${arches[@]}; do
+			releasePackage extra ${pkgbase} ${arch}
+		done
+	done
+	../db-update
+	../cron-jobs/sourceballs
+
+	for arch in ${arches[@]}; do
+		../db-remove pkg-simple-a extra ${arch}
+	done
+	../cron-jobs/ftpdir-cleanup >/dev/null
+
+	../cron-jobs/sourceballs
+	for log in errors failed; do
+		if [ -s "${LOGDIR}/sourceballs/${log}.txt" ]; then
+			fail "${LOGDIR}/sourceballs/${log}.txt includes the following errors:"
+			cat "${LOGDIR}/sourceballs/${log}.txt"
+		fi
+	done
+	[ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!"
+	[ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!"
+}
 
 . "${curdir}/lib/shunit2"