summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2010-11-23 12:04:30 +0100
committerPierre Schmitz <pierre@archlinux.de>2010-11-23 12:04:30 +0100
commit4d16d294f338d5e90a92acb2096bca77e9e924e7 (patch)
tree7c977635edc813dfb1a766d214d56ecf63a49867
parent1dca721c5149f69067f38d4a33dabbebef98009b (diff)
Rewrite sourceballs to increase performance and reliability
* Decrease file stats as much as possible * Create a list of all packages and meta data only once * Create a list of available source packages only once * Create a list of expected packages only once * Combine all three scripts into one to share data and code * Use as much information from the db files as possible and avoid using svn * Avoid attempting to create the same source package twice Logic works as follows: 1) create a list of all packages 2) Check for each package if we need a src package and create one 3) During this process create a list of all src packages that should be there 4) Diff both lists for the cleanup
-rwxr-xr-xcron-jobs/sourceballs144
-rw-r--r--db-functions35
-rwxr-xr-xmisc-scripts/make-sourceball97
-rwxr-xr-xmisc-scripts/sourceballs-cleanup82
4 files changed, 101 insertions, 257 deletions
diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs
index b708009..3f690e7 100755
--- a/cron-jobs/sourceballs
+++ b/cron-jobs/sourceballs
@@ -1,57 +1,111 @@
#!/bin/bash
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
+dirname="$(dirname $0)"
+. "${dirname}/../db-functions"
+. "${dirname}/../config"
script_lock
-set_umask
+for repo in ${PKGREPOS[@]}; do
+ for arch in ${ARCHES[@]}; do
+ repo_lock ${repo} ${arch} || exit 1
+ done
+done
+
+# Create a readable file for each repo with the following format
+# <pkgbase|pkgname> <pkgver>-<pkgrel> <arch> <license>[ <license>]
+for repo in ${PKGREPOS[@]}; do
+ for arch in ${ARCHES[@]}; do
+ if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then
+ warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping"
+ continue
+ fi
+ bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \
+ | awk '/^%NAME%/ { getline b };
+ /^%BASE%/ { getline b };
+ /^%VERSION%/ { getline v };
+ /^%LICENSE%/,/^$/ {
+ if ( !/^%LICENSE%/ ) { l=l" "$0 }
+ };
+ /^%ARCH%/ {
+ getline a;
+ printf "%s %s %s %s\n", b, v, a, l;
+ l="";
+ }'
+ done | sort -u > "${WORKDIR}/db-${repo}"
+done
+
+for repo in ${PKGREPOS[@]}; do
+ for arch in ${ARCHES[@]}; do
+ repo_unlock ${repo} ${arch}
+ done
+done
-dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))"
+# Create a list of all available source package file names
+find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs"
+# Check for all packages if we need to build a source package
for repo in ${PKGREPOS[@]}; do
- for arch in ${ARCHES[@]}; do
- ftppath="${FTP_BASE}/$repo/os/$arch"
- dbfile="${ftppath}/${repo}${DBEXT}"
- if [ ! -r "${dbfile}" ]; then
- warning "${dbfile} not found, skipping"
- continue
- fi
-
- repo_lock ${repo} ${arch} || exit 1
- # Read packages from db file
- # Format is: <pkgbase|pkgname>/<pkgver>-<pkgrel>
- pkgs=($(bsdtar -xOf "${dbfile}" \
- | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \
- | sort -u))
- repo_unlock ${repo} ${arch}
-
- for pkg in ${pkgs[@]}; do
- pkgbase=${pkg%/*}
- pkgver=${pkg#*/}
- srcpkg="${pkgbase}-${pkgver}${SRCEXT}"
-
- # Don't do anything for package in this 'blacklist'
- if grep -q "^${pkgbase}\$" "$dirname/sourceballs.skip"; then
- continue
- fi
-
- # Use this file to 'whitelist' or force building some sourceballs,
- # skipping the license check
- force=""
- if grep -q "^$pkgbase\$" "$dirname/sourceballs.force"; then
- force="-f"
- fi
-
- if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then
- if ! $dirname/../misc-scripts/make-sourceball $force $pkgbase $repo $arch; then
- error "Failed to download sources for $pkgbase"
- fi
- fi
- done
- done
+ msg "Updating source packages for ${repo}..."
+ while read line; do
+ pkginfo=(${line})
+ pkgbase=${pkginfo[0]}
+ pkgver=${pkginfo[1]}
+ pkgarch=${pkginfo[2]}
+ pkglicense=(${pkginfo[@]:3})
+
+ # Should this package be skipped?
+ if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then
+ continue
+ fi
+ # Check if the license or .force file does not enforce creating a source package
+ if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then
+ continue
+ fi
+ # Store the expected file name of the source package
+ echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs"
+
+ # Build the source package if its not already there
+ if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then
+ msg2 "${pkgbase}"
+ # Looks like a previous arch faild; skip it
+ if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then
+ continue
+ fi
+ mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}"
+ svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \
+ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null
+ if [ $? -ge 1 ]; then
+ error "Could not check out ${pkgbase}/repos/${repo}-${pkgarch}"
+ continue
+ fi
+
+ pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null
+ makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1
+ if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then
+ mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}"
+ # Avoid creating the same source package for every arch
+ echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs"
+ else
+ error "Could not create source package for ${pkgbase}/repos/${repo}-${pkgarch}"
+ fi
+ popd >/dev/null
+ fi
+ done < "${WORKDIR}/db-${repo}"
done
-$dirname/../misc-scripts/sourceballs-cleanup
+# Cleanup old source packages
+cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort"
+cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort"
+old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort"))
+
+if [ ${#old_pkgs[@]} -ge 1 ]; then
+ msg "Removing old source packages..."
+ ${CLEANUP_DRYRUN} && warning 'dry run mode is active'
+ for old_pkg in ${old_pkgs[@]}; do
+ msg2 "${old_pkg}"
+ ${CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}"
+ done
+fi
script_unlock
diff --git a/db-functions b/db-functions
index d2df1a7..67a52aa 100644
--- a/db-functions
+++ b/db-functions
@@ -347,44 +347,13 @@ check_pkgrepos() {
#usage: chk_license ${license[@]}"
chk_license() {
local l
- local allowed
- for l in "$@"; do
- l="$(echo $l | tr '[:upper:]' '[:lower:]')"
- for allowed in ${ALLOWED_LICENSES[@]}; do
- allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')"
- if [ "$l" = "$allowed" ]; then
- return 0
- fi
- done
+ for l in ${@}; do
+ in_array ${l} ${ALLOWED_LICENSES[@]} && return 0
done
return 1
}
-pkgname_from_src() {
- local tmp
- local a
- tmp=${1##*/}
- tmp=${tmp%$SRCEXT}
- for a in ${ARCHES[@]}; do
- tmp=${tmp%-$a}
- done
- tmp=${tmp%-any}
- echo ${tmp%-*-*}
-}
-
-pkgver_from_src() {
- local tmp
- local a
- tmp=${1##*/}
- tmp=${tmp%$SRCEXT}
- for a in ${ARCHES[@]}; do
- tmp=${tmp%-$a}
- done
- tmp=${tmp%-any}
- echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g'
-}
-
check_repo_permission() {
local repo=$1
diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball
deleted file mode 100755
index 9a43376..0000000
--- a/misc-scripts/make-sourceball
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/bin/bash
-
-if [ $# -ne 3 -a $# -ne 4 ]; then
- msg "usage: $(basename $0) [-f] <packagename> <repo> <arch>"
- msg " -f Force building. Skip license checks"
- exit 1
-fi
-
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
-
-FORCE=0
-if [ "$1" = "-f" ]; then
- FORCE=1
- shift
-fi
-
-packagename="$1"
-reponame="$2"
-arch="$3"
-
-script_lock
-
-create_srcpackage() {
- if [ -f "$1/PKGBUILD" ]; then
- pushd "$1" >/dev/null
- pkgver=$(. PKGBUILD; echo ${pkgver})
- if [ $? -ne 0 ]; then
- error "Failed to read pkgver from $1"
- return 1
- fi
- pkgrel=$(. PKGBUILD; echo ${pkgrel})
- if [ $? -ne 0 ]; then
- error "Failed to read pkgrel from $1"
- return 1
- fi
- license=($(. PKGBUILD; echo ${license[@]}))
- if [ $? -ne 0 ]; then
- error "Failed to read license from $1"
- return 1
- fi
- if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then
- # Removed so as not to clutter failed.txt
- #warning "$packagename license (${license[@]}) does not require source tarballs"
- return 0
- else
- msg "Creating source tarball for $packagename-$pkgver-$pkgrel"
- fi
-
- local line
- /usr/bin/makepkg --allsource --ignorearch > "$WORKDIR/makepkg.log" 2>&1
- if [ $? -ne 0 ]; then
- error "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)"
- while read line; do
- msg2 "${line}"
- done < "$WORKDIR/makepkg.log"
- return 1
- fi
-
- if [ -f "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" ]; then
- mv "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}"
- else
- error "Source package not found: ${packagename}-${pkgver}-${pkgrel}${SRCEXT}"
- while read line; do
- msg2 "${line}"
- done < "$WORKDIR/makepkg.log"
- return 1
- fi
-
- popd >/dev/null
-
- return 0
- fi
-}
-
-set_umask
-cd "$WORKDIR"
-
-failed=0
-
-if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then
- if [ -f "$packagename/repos/$reponame-any/PKGBUILD" ]; then
- create_srcpackage "$packagename/repos/$reponame-any" || failed=1
- elif [ -f "$packagename/repos/$reponame-$arch/PKGBUILD" ]; then
- create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1
- else
- error "PKGBUILD for '$packagename' does not exist in repo '$reponame-$arch' or '$reponame-any'"
- failed=1
- fi
-else
- error "Package '$packagename' does not exist in repo '$reponame-$arch'"
- failed=1
-fi
-
-script_unlock
-
-exit $failed \ No newline at end of file
diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup
deleted file mode 100755
index 595f501..0000000
--- a/misc-scripts/sourceballs-cleanup
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/bash
-
-. "$(dirname $0)/../db-functions"
-. "$(dirname $0)/../config"
-
-script_lock
-
-${CLEANUP_DRYRUN} && warning 'dry run mode is active'
-
-remove_old() {
- if [ -d "$1" ]; then
- pushd "$1" >/dev/null
- PKGVERS=""
- for repo in *; do
- cd "$repo"
- if [ -f PKGBUILD ]; then
- pkgver=$(. PKGBUILD; echo ${pkgver})
- pkgrel=$(. PKGBUILD; echo ${pkgrel})
- PKGVERS="$PKGVERS $pkgver-$pkgrel"
- else
- error "PKGBUILD not found in $1/$repo"
- fi
- cd ..
- done
-
- for srcpkg in "${FTP_BASE}/${SRCPOOL}/$packagename-"*; do
- [ -f "$srcpkg" ] || continue
- if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then
- skip=0
- pver="$(pkgver_from_src $srcpkg)"
- for v in $PKGVERS; do
- if [ "$v" = "$pver" ]; then
- skip=1
- break
- fi
- done
- if [ $skip -ne 1 ]; then
- ${CLEANUP_DRYRUN} || mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR
- fi
- fi
- done
-
- popd >/dev/null
- fi
-}
-
-#adjust the nice level to run at a lower priority
-/usr/bin/renice +10 -p $$ > /dev/null
-
-set_umask
-cd "$WORKDIR"
-
-for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do
- [ -f "$sourceball" ] || continue
- packagename=$(basename $sourceball)
- packagename=${packagename%-*-*$SRCEXT}
-
- if grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.skip"; then
- msg "$packagename : package found in sourceballs.skip. Removing sourceball."
- ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
- fi
- if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1; then
- if [ $? -ne 1 ]; then
- error "$packagename : svn died during export. Skipping sourceball."
- else
- msg "$packagename : no longer in svn. Removing sourceball."
- ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
- fi
- elif [ -z "$(ls -A "$packagename/repos")" ]; then
- warning "$packagename : no longer in repos but trunk is still in svn. Removing sourceball."
- ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
- elif ! chk_license $(. "$packagename/trunk/PKGBUILD"; echo ${license[@]}); then
- if ! grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.force"; then
- msg "$packagename : source hosting no longer required by license. Removing sourceball."
- ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR
- fi
- else
- remove_old "$packagename/repos/"
- fi
-done
-
-script_unlock