summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2010-11-25 16:03:44 +0100
committerPierre Schmitz <pierre@archlinux.de>2010-11-25 16:03:44 +0100
commite792abefa6eff4cf54616add0b19d731d486045f (patch)
treecaf37f3b5d53945285303ac48929359028bd44a1
parent72ed0011f7ff967d8a402c84c5ab5783f9db9987 (diff)
Rewrote create-filelists
* use correct locking of the repos * added test case * removes file lists of deleted packages * add compression independent symlink to files db
-rwxr-xr-xcron-jobs/create-filelists147
-rwxr-xr-xtest/test.d/create-filelists.sh110
2 files changed, 191 insertions, 66 deletions
diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 010194b..c3e7d72 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -3,20 +3,42 @@
. "$(dirname $0)/../db-functions"
. "$(dirname $0)/../config"
-reposdir=${FTP_BASE}
-targetdir=${FTP_BASE}
-
script_lock
-# location where the package DB is extracted so we know what to include
-DBDIR="$(mktemp -d ${WORKDIR}/create-filelists.dbdir.XXXXXX)" || exit 1
-# location where the old files DB is extracted to save us some work
-CACHEDIR="$(mktemp -d ${WORKDIR}/create-filelists.cachedir.XXXXXX)" || exit 1
-# location where the new files DB is built up and eventually zipped
-MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1
+for repo in ${PKGREPOS[@]}; do
+ for arch in ${ARCHES[@]}; do
+ repo_lock ${repo} ${arch} || exit 1
+ done
+done
#adjust the nice level to run at a lower priority
-/usr/bin/renice +10 -p $$ > /dev/null
+renice +10 -p $$ > /dev/null
+
+for repo in ${PKGREPOS[@]}; do
+ for arch in ${ARCHES[@]}; do
+ repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}"
+ filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
+
+ if [ ! -f "${repodb}" ]; then
+ continue
+ fi
+ # get a list of package files defined in the repo db
+ mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}"
+ bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}"
+ find "${WORKDIR}/db-dir-${repo}-${arch}" -name 'desc' \
+ -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/db-${repo}-${arch}"
+
+ # get a list of package files defined in the files db
+ mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}"
+ if [ ! -f "${filedb}" ]; then
+ echo > "${WORKDIR}/files-${repo}-${arch}"
+ else
+ bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}"
+ find "${WORKDIR}/files-current-dir-${repo}-${arch}" -name 'desc' \
+ -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/files-${repo}-${arch}"
+ fi
+ done
+done
case "${FILESEXT}" in
*.gz) TAR_OPT="z" ;;
@@ -26,62 +48,55 @@ case "${FILESEXT}" in
esac
for repo in ${PKGREPOS[@]}; do
- REPO_DB_FILE="${repo}$DBEXT"
- FILES_DB_FILE="${repo}$FILESEXT"
- for arch in ${ARCHES[@]}; do
- pushd "$reposdir" >/dev/null
-
- repodir="${repo}/os/${arch}"
- cached="no"
-
- # extract package db archive
- if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then
- mkdir -p "${DBDIR}/${repodir}"
- bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}"
- else
- warning "The repo $repo with arch $arch does not exist"
- continue
- fi
-
- # extract old file archive
- if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then
- mkdir -p "${CACHEDIR}/${repodir}"
- bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}"
- cached="yes"
- fi
-
- # create file lists
- for pkg in $(ls ${DBDIR}/${repodir}); do
- dbpkgdir="${DBDIR}/${repodir}/${pkg}"
- cachepkgdir="${CACHEDIR}/${repodir}/${pkg}"
- tmppkgdir="${MYTMPDIR}/${repodir}/${pkg}"
- mkdir -p "$tmppkgdir"
- ln "${dbpkgdir}/desc" "${tmppkgdir}/desc"
- ln "${dbpkgdir}/depends" "${tmppkgdir}/depends"
- if [ -f "${cachepkgdir}/files" ]; then
- ln "${cachepkgdir}/files" "${tmppkgdir}/files"
- else
- filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1)
- echo '%FILES%' > "${tmppkgdir}/files"
- bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files"
- cached="no"
- fi
- done
-
- # create new file archive
- if [ "$cached" == "no" -a -d "${MYTMPDIR}/${repodir}" ]; then
- # at least one package has changed, so let's rebuild the archive
- pkgdir="${targetdir}/${repodir}"
- mkdir -p "$pkgdir"
- pushd "${MYTMPDIR}/${repodir}" >/dev/null
- [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}"
- bsdtar --exclude=*${FILESEXT} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" *
- popd >/dev/null
- fi
- popd >/dev/null
- done
+ for arch in ${ARCHES[@]}; do
+ filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
+
+ if [ ! -f "${WORKDIR}/db-${repo}-${arch}" ]; then
+ # remove any files db that might be in this empty repo
+ if [ -f "${filedb}" ]; then
+ rm -f "${filedb}"
+ fi
+ continue
+ fi
+
+ # Check if updating the files db is needed
+ if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then
+ mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}"
+
+ # Include all unchanged file lists
+ # Note: deleted packages are implicitly excluded
+ for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
+ mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \
+ "${WORKDIR}/files-new-dir-${repo}-${arch}"
+ done
+
+ # Create file lists for new packages
+ for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
+ tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
+ mkdir "${tdir}"
+ echo '%FILES%' > "${tdir}/files"
+ bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files"
+
+ # add desc and depends file from db
+ dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
+ mv "${dbdir}/desc" "${tdir}/desc"
+ mv "${dbdir}/depends" "${tdir}/depends"
+ done
+
+ # Create the actual file db
+ pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null
+ bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" *
+ popd >/dev/null
+ mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}"
+ ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}"
+ fi
+ done
+done
+
+for repo in ${PKGREPOS[@]}; do
+ for arch in ${ARCHES[@]}; do
+ repo_unlock ${repo} ${arch}
+ done
done
script_unlock
-
-# vim: set ts=4 sw=4 et ft=sh:
diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh
new file mode 100755
index 0000000..da76710
--- /dev/null
+++ b/test/test.d/create-filelists.sh
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+curdir=$(readlink -e $(dirname $0))
+. "${curdir}/../lib/common.inc"
+
+testCreateSimpleFileLists() {
+ local arches=('i686' 'x86_64')
+ local pkgs=('pkg-simple-a' 'pkg-simple-b')
+ local pkgbase
+ local arch
+
+ for pkgbase in ${pkgs[@]}; do
+ for arch in ${arches[@]}; do
+ releasePackage extra ${pkgbase} ${arch}
+ done
+ done
+ ../db-update
+
+ ../cron-jobs/create-filelists
+ for pkgbase in ${pkgs[@]}; do
+ for arch in ${arches[@]}; do
+ if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgbase}"; then
+ fail "usr/bin/${pkgbase} not found in ${arch}/extra.files.tar.gz"
+ fi
+ done
+ done
+}
+
+testCreateAnyFileLists() {
+ local arches=('i686' 'x86_64')
+ local pkgs=('pkg-any-a' 'pkg-any-b')
+ local pkgbase
+ local arch
+
+ for pkgbase in ${pkgs[@]}; do
+ releasePackage extra ${pkgbase} any
+ done
+ ../db-update
+
+ ../cron-jobs/create-filelists
+ for pkgbase in ${pkgs[@]}; do
+ for arch in ${arches[@]}; do
+ if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/share/${pkgbase}/test"; then
+ fail "usr/share/${pkgbase}/test not found in ${arch}/extra.files.tar.gz"
+ fi
+ done
+ done
+}
+
+testCreateSplitFileLists() {
+ local arches=('i686' 'x86_64')
+ local pkgs=('pkg-split-a' 'pkg-split-b')
+ local pkg
+ local pkgbase
+ local pkgname
+ local pkgnames
+ local arch
+
+ for pkgbase in ${pkgs[@]}; do
+ for arch in ${arches[@]}; do
+ releasePackage extra ${pkgbase} ${arch}
+ done
+ done
+ ../db-update
+
+ ../cron-jobs/create-filelists
+ for pkgbase in ${pkgs[@]}; do
+ pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]}))
+ for pkgname in ${pkgnames[@]}; do
+ for arch in ${arches[@]}; do
+ if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgname}"; then
+ fail "usr/bin/${pkgname} not found in ${arch}/extra.files.tar.gz"
+ fi
+ done
+ done
+ done
+}
+
+
+testCleanupFileLists() {
+ local arches=('i686' 'x86_64')
+ local pkgs=('pkg-simple-a' 'pkg-simple-b')
+ local pkgbase
+ local arch
+
+ for pkgbase in ${pkgs[@]}; do
+ for arch in ${arches[@]}; do
+ releasePackage extra ${pkgbase} ${arch}
+ done
+ done
+ ../db-update
+ ../cron-jobs/create-filelists
+
+ for arch in ${arches[@]}; do
+ ../db-remove pkg-simple-a extra ${arch}
+ done
+ ../cron-jobs/create-filelists
+
+ for arch in ${arches[@]}; do
+ if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-b"; then
+ fail "usr/bin/pkg-simple-b not found in ${arch}/extra.files.tar.gz"
+ fi
+ if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-a"; then
+ fail "usr/bin/pkg-simple-a still found in ${arch}/extra.files.tar.gz"
+ fi
+ done
+
+}
+
+. "${curdir}/../lib/shunit2"