Rewrite sourceballs to increase performance and reliability

* Decrease file stats as much as possible
	* Create a list of all packages and meta data only once
	* Create a list of available source packages only once
	* Create a list of expected packages only once
* Combine all three scripts into one to share data and code
* Use as much information from the db files as possible and avoid using svn
* Avoid attempting to create the same source package twice

Logic works as follows:
1) create a list of all packages
2) Check for each package if we need a src package and create one
3) During this process create a list of all src packages that should be there
4) Diff both lists for the cleanup

4 files changed, 101 insertions, 257 deletions

diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs
index b708009..3f690e7 100755
--- a/cron-jobs/sourceballs
+++ b/cron-jobs/sourceballs
@@ -1,57 +1,111 @@
 #!/bin/bash
 
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
+dirname="$(dirname $0)"
+. "${dirname}/../db-functions"
+. "${dirname}/../config"
 
 script_lock
 
-set_umask
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repo_lock ${repo} ${arch} || exit 1
+	done
+done
+
+# Create a readable file for each repo with the following format
+# <pkgbase|pkgname> <pkgver>-<pkgrel> <arch> <license>[ <license>]
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then
+			warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping"
+			continue
+		fi
+		bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \
+			| awk '/^%NAME%/ { getline b };
+				/^%BASE%/ { getline b };
+				/^%VERSION%/ { getline v };
+				/^%LICENSE%/,/^$/ {
+					if ( !/^%LICENSE%/ ) { l=l" "$0 }
+					};
+				/^%ARCH%/ {
+					getline a;
+					printf "%s %s %s %s\n", b, v, a, l;
+					l="";
+				}'
+	done | sort -u > "${WORKDIR}/db-${repo}"
+done
+
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repo_unlock ${repo} ${arch}
+	done
+done
 
-dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))"
+# Create a list of all available source package file names
+find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs"
 
+# Check for all packages if we need to build a source package
 for repo in ${PKGREPOS[@]}; do
-    for arch in ${ARCHES[@]}; do
-        ftppath="${FTP_BASE}/$repo/os/$arch"
-        dbfile="${ftppath}/${repo}${DBEXT}"
-        if [ ! -r "${dbfile}" ]; then
-            warning "${dbfile} not found, skipping"
-            continue
-        fi
-
-        repo_lock ${repo} ${arch} || exit 1
-        # Read packages from db file
-        # Format is: <pkgbase|pkgname>/<pkgver>-<pkgrel>
-        pkgs=($(bsdtar -xOf "${dbfile}" \
-            | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \
-            | sort -u))
-        repo_unlock ${repo} ${arch}
-
-        for pkg in ${pkgs[@]}; do
-            pkgbase=${pkg%/*}
-            pkgver=${pkg#*/}
-            srcpkg="${pkgbase}-${pkgver}${SRCEXT}"
-
-            # Don't do anything for package in this 'blacklist'
-            if grep -q "^${pkgbase}\$" "$dirname/sourceballs.skip"; then
-                continue
-            fi
-
-            # Use this file to 'whitelist' or force building some sourceballs,
-            # skipping the license check
-            force=""
-            if grep -q "^$pkgbase\$" "$dirname/sourceballs.force"; then
-                force="-f"
-            fi
-
-            if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then
-                if ! $dirname/../misc-scripts/make-sourceball $force $pkgbase $repo $arch; then
-                    error "Failed to download sources for $pkgbase"
-                fi
-            fi
-        done
-    done
+	msg "Updating source packages for ${repo}..."
+	while read line; do
+		pkginfo=(${line})
+		pkgbase=${pkginfo[0]}
+		pkgver=${pkginfo[1]}
+		pkgarch=${pkginfo[2]}
+		pkglicense=(${pkginfo[@]:3})
+
+		# Should this package be skipped?
+		if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then
+			continue
+		fi
+		# Check if the license or .force file does not enforce creating a source package
+		if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then
+			continue
+		fi
+		# Store the expected file name of the source package
+		echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs"
+
+		# Build the source package if its not already there
+		if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then
+			msg2 "${pkgbase}"
+			# Looks like a previous arch faild; skip it
+			if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then
+				continue
+			fi
+			mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}"
+			svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \
+				"${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null
+			if [ $? -ge 1 ]; then
+				error "Could not check out ${pkgbase}/repos/${repo}-${pkgarch}"
+				continue
+			fi
+
+			pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null
+			makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1
+			if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then
+				mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}"
+				# Avoid creating the same source package for every arch
+				echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs"
+			else
+				error "Could not create source package for ${pkgbase}/repos/${repo}-${pkgarch}"
+			fi
+			popd >/dev/null
+		fi
+	done < "${WORKDIR}/db-${repo}"
 done
 
-$dirname/../misc-scripts/sourceballs-cleanup
+# Cleanup old source packages
+cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort"
+cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort"
+old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort"))
+
+if [ ${#old_pkgs[@]} -ge 1 ]; then
+	msg "Removing old source packages..."
+	${CLEANUP_DRYRUN} && warning 'dry run mode is active'
+	for old_pkg in ${old_pkgs[@]}; do
+		msg2 "${old_pkg}"
+		${CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}"
+	done
+fi
 
 script_unlock
diff --git a/db-functions b/db-functions
index d2df1a7..67a52aa 100644
--- a/db-functions
+++ b/db-functions
@@ -347,44 +347,13 @@ check_pkgrepos() {
 #usage: chk_license ${license[@]}"
 chk_license() {
 	local l
-	local allowed
-	for l in "$@"; do
-		l="$(echo $l | tr '[:upper:]' '[:lower:]')"
-		for allowed in ${ALLOWED_LICENSES[@]}; do
-			allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')"
-			if [ "$l" = "$allowed" ]; then
-				return 0
-			fi
-		done
+	for l in ${@}; do
+		in_array ${l} ${ALLOWED_LICENSES[@]} && return 0
 	done
 
 	return 1
 }
 
-pkgname_from_src() {
-	local tmp
-	local a
-	tmp=${1##*/}
-	tmp=${tmp%$SRCEXT}
-	for a in ${ARCHES[@]}; do
-		tmp=${tmp%-$a}
-	done
-	tmp=${tmp%-any}
-	echo ${tmp%-*-*}
-}
-
-pkgver_from_src() {
-	local tmp
-	local a
-	tmp=${1##*/}
-	tmp=${tmp%$SRCEXT}
-	for a in ${ARCHES[@]}; do
-		tmp=${tmp%-$a}
-	done
-	tmp=${tmp%-any}
-	echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g'
-}
-
 check_repo_permission() {
 	local repo=$1
 
diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball
deleted file mode 100755
index 9a43376..0000000
--- a/misc-scripts/make-sourceball
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/bin/bash
-
-if [ $# -ne 3 -a $# -ne 4 ]; then
-    msg "usage: $(basename $0) [-f] <packagename> <repo> <arch>"
-    msg "  -f  Force building. Skip license checks"
-    exit 1
-fi
-
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
-
-FORCE=0
-if [ "$1" = "-f" ]; then
-    FORCE=1
-    shift
-fi
-
-packagename="$1"
-reponame="$2"
-arch="$3"
-
-script_lock
-
-create_srcpackage() {
-    if [ -f "$1/PKGBUILD" ]; then
-        pushd "$1" >/dev/null
-        pkgver=$(. PKGBUILD; echo ${pkgver})
-        if [ $? -ne 0 ]; then
-            error "Failed to read pkgver from $1"
-            return 1
-        fi
-        pkgrel=$(. PKGBUILD; echo ${pkgrel})
-        if [ $? -ne 0 ]; then
-            error "Failed to read pkgrel from $1"
-            return 1
-        fi
-        license=($(. PKGBUILD; echo ${license[@]}))
-        if [ $? -ne 0 ]; then
-            error "Failed to read license from $1"
-            return 1
-        fi
-        if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then
-            # Removed so as not to clutter failed.txt
-            #warning "$packagename license (${license[@]}) does not require source tarballs"
-            return 0
-        else
-            msg "Creating source tarball for $packagename-$pkgver-$pkgrel"
-        fi
-
-        local line
-        /usr/bin/makepkg --allsource --ignorearch > "$WORKDIR/makepkg.log" 2>&1
-        if [ $? -ne 0 ]; then
-            error "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)"
-            while read line; do
-                msg2 "${line}"
-            done < "$WORKDIR/makepkg.log"
-            return 1
-        fi
-
-        if [ -f "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" ]; then
-            mv "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}"
-        else
-            error "Source package not found: ${packagename}-${pkgver}-${pkgrel}${SRCEXT}"
-            while read line; do
-                msg2 "${line}"
-            done < "$WORKDIR/makepkg.log"
-            return 1
-        fi
-
-        popd >/dev/null
-
-        return 0
-    fi
-}
-
-set_umask
-cd "$WORKDIR"
-
-failed=0
-
-if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then
-  if [ -f "$packagename/repos/$reponame-any/PKGBUILD" ]; then
-    create_srcpackage "$packagename/repos/$reponame-any" || failed=1
-  elif [ -f "$packagename/repos/$reponame-$arch/PKGBUILD" ]; then
-    create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1
-  else
-    error "PKGBUILD for '$packagename' does not exist in repo '$reponame-$arch' or '$reponame-any'"
-    failed=1
-  fi
-else
-  error "Package '$packagename' does not exist in repo '$reponame-$arch'"
-  failed=1
-fi
-
-script_unlock
-
-exit $failed
\ No newline at end of file
diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup
deleted file mode 100755
index 595f501..0000000
--- a/misc-scripts/sourceballs-cleanup
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/bash
-
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
-
-script_lock
-
-${CLEANUP_DRYRUN} && warning 'dry run mode is active'
-
-remove_old() {
-    if [ -d "$1" ]; then
-        pushd "$1" >/dev/null
-        PKGVERS=""
-        for repo in *; do
-            cd "$repo"
-            if [ -f PKGBUILD ]; then
-                pkgver=$(. PKGBUILD; echo ${pkgver})
-                pkgrel=$(. PKGBUILD; echo ${pkgrel})
-                PKGVERS="$PKGVERS $pkgver-$pkgrel"
-            else
-                error "PKGBUILD not found in $1/$repo"
-            fi
-            cd ..
-        done
-
-        for srcpkg in "${FTP_BASE}/${SRCPOOL}/$packagename-"*; do
-            [ -f "$srcpkg" ] || continue
-            if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then
-                skip=0
-                pver="$(pkgver_from_src $srcpkg)"
-                for v in $PKGVERS; do
-                    if [ "$v" = "$pver" ]; then
-                        skip=1
-                        break
-                    fi
-                done
-                if [ $skip -ne 1 ]; then
-                    ${CLEANUP_DRYRUN} || mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR
-                fi
-            fi
-        done
-
-        popd >/dev/null
-    fi
-}
-
-#adjust the nice level to run at a lower priority
-/usr/bin/renice +10 -p $$ > /dev/null
-
-set_umask
-cd "$WORKDIR"
-
-for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do
-    [ -f "$sourceball" ] || continue
-    packagename=$(basename $sourceball)
-    packagename=${packagename%-*-*$SRCEXT}
-
-    if grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.skip"; then
-      msg "$packagename : package found in sourceballs.skip. Removing sourceball."
-      ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
-    fi
-    if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1; then
-      if [ $? -ne 1 ]; then
-        error "$packagename : svn died during export. Skipping sourceball."
-      else
-        msg "$packagename : no longer in svn.  Removing sourceball."
-        ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
-      fi
-    elif [ -z "$(ls -A "$packagename/repos")" ]; then
-      warning "$packagename : no longer in repos but trunk is still in svn.  Removing sourceball."
-      ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
-    elif ! chk_license $(. "$packagename/trunk/PKGBUILD"; echo ${license[@]}); then
-      if ! grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.force"; then
-        msg "$packagename : source hosting no longer required by license.  Removing sourceball."
-        ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
-      fi
-    else
-      remove_old "$packagename/repos/"
-    fi
-done
-
-script_unlock