summaryrefslogtreecommitdiff
path: root/cron-jobs
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2010-11-23 12:04:30 +0100
committerPierre Schmitz <pierre@archlinux.de>2010-11-23 12:04:30 +0100
commit4d16d294f338d5e90a92acb2096bca77e9e924e7 (patch)
tree7c977635edc813dfb1a766d214d56ecf63a49867 /cron-jobs
parent1dca721c5149f69067f38d4a33dabbebef98009b (diff)
Rewrite sourceballs to increase performance and reliability
* Decrease file stats as much as possible * Create a list of all packages and meta data only once * Create a list of available source packages only once * Create a list of expected packages only once * Combine all three scripts into one to share data and code * Use as much information from the db files as possible and avoid using svn * Avoid attempting to create the same source package twice Logic works as follows: 1) create a list of all packages 2) Check for each package if we need a src package and create one 3) During this process create a list of all src packages that should be there 4) Diff both lists for the cleanup
Diffstat (limited to 'cron-jobs')
-rwxr-xr-xcron-jobs/sourceballs144
1 files changed, 99 insertions, 45 deletions
diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs
index b708009..3f690e7 100755
--- a/cron-jobs/sourceballs
+++ b/cron-jobs/sourceballs
@@ -1,57 +1,111 @@
#!/bin/bash
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
+dirname="$(dirname $0)"
+. "${dirname}/../db-functions"
+. "${dirname}/../config"
script_lock
-set_umask
+for repo in ${PKGREPOS[@]}; do
+ for arch in ${ARCHES[@]}; do
+ repo_lock ${repo} ${arch} || exit 1
+ done
+done
+
+# Create a readable file for each repo with the following format
+# <pkgbase|pkgname> <pkgver>-<pkgrel> <arch> <license>[ <license>]
+for repo in ${PKGREPOS[@]}; do
+ for arch in ${ARCHES[@]}; do
+ if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then
+ warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping"
+ continue
+ fi
+ bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \
+ | awk '/^%NAME%/ { getline b };
+ /^%BASE%/ { getline b };
+ /^%VERSION%/ { getline v };
+ /^%LICENSE%/,/^$/ {
+ if ( !/^%LICENSE%/ ) { l=l" "$0 }
+ };
+ /^%ARCH%/ {
+ getline a;
+ printf "%s %s %s %s\n", b, v, a, l;
+ l="";
+ }'
+ done | sort -u > "${WORKDIR}/db-${repo}"
+done
+
+for repo in ${PKGREPOS[@]}; do
+ for arch in ${ARCHES[@]}; do
+ repo_unlock ${repo} ${arch}
+ done
+done
-dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))"
+# Create a list of all available source package file names
+find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs"
+# Check for all packages if we need to build a source package
for repo in ${PKGREPOS[@]}; do
- for arch in ${ARCHES[@]}; do
- ftppath="${FTP_BASE}/$repo/os/$arch"
- dbfile="${ftppath}/${repo}${DBEXT}"
- if [ ! -r "${dbfile}" ]; then
- warning "${dbfile} not found, skipping"
- continue
- fi
-
- repo_lock ${repo} ${arch} || exit 1
- # Read packages from db file
- # Format is: <pkgbase|pkgname>/<pkgver>-<pkgrel>
- pkgs=($(bsdtar -xOf "${dbfile}" \
- | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \
- | sort -u))
- repo_unlock ${repo} ${arch}
-
- for pkg in ${pkgs[@]}; do
- pkgbase=${pkg%/*}
- pkgver=${pkg#*/}
- srcpkg="${pkgbase}-${pkgver}${SRCEXT}"
-
- # Don't do anything for package in this 'blacklist'
- if grep -q "^${pkgbase}\$" "$dirname/sourceballs.skip"; then
- continue
- fi
-
- # Use this file to 'whitelist' or force building some sourceballs,
- # skipping the license check
- force=""
- if grep -q "^$pkgbase\$" "$dirname/sourceballs.force"; then
- force="-f"
- fi
-
- if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then
- if ! $dirname/../misc-scripts/make-sourceball $force $pkgbase $repo $arch; then
- error "Failed to download sources for $pkgbase"
- fi
- fi
- done
- done
+ msg "Updating source packages for ${repo}..."
+ while read line; do
+ pkginfo=(${line})
+ pkgbase=${pkginfo[0]}
+ pkgver=${pkginfo[1]}
+ pkgarch=${pkginfo[2]}
+ pkglicense=(${pkginfo[@]:3})
+
+ # Should this package be skipped?
+ if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then
+ continue
+ fi
+ # Check if the license or .force file does not enforce creating a source package
+ if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then
+ continue
+ fi
+ # Store the expected file name of the source package
+ echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs"
+
+ # Build the source package if its not already there
+ if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then
+ msg2 "${pkgbase}"
+ # Looks like a previous arch faild; skip it
+ if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then
+ continue
+ fi
+ mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}"
+ svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \
+ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null
+ if [ $? -ge 1 ]; then
+ error "Could not check out ${pkgbase}/repos/${repo}-${pkgarch}"
+ continue
+ fi
+
+ pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null
+ makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1
+ if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then
+ mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}"
+ # Avoid creating the same source package for every arch
+ echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs"
+ else
+ error "Could not create source package for ${pkgbase}/repos/${repo}-${pkgarch}"
+ fi
+ popd >/dev/null
+ fi
+ done < "${WORKDIR}/db-${repo}"
done
-$dirname/../misc-scripts/sourceballs-cleanup
+# Cleanup old source packages
+cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort"
+cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort"
+old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort"))
+
+if [ ${#old_pkgs[@]} -ge 1 ]; then
+ msg "Removing old source packages..."
+ ${CLEANUP_DRYRUN} && warning 'dry run mode is active'
+ for old_pkg in ${old_pkgs[@]}; do
+ msg2 "${old_pkg}"
+ ${CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}"
+ done
+fi
script_unlock