Rewrote create-filelists

* use correct locking of the repos
* added test case
* removes file lists of deleted packages
* add compression independent symlink to files db

2 files changed, 191 insertions, 66 deletions

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 010194b..c3e7d72 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -3,20 +3,42 @@
 . "$(dirname $0)/../db-functions"
 . "$(dirname $0)/../config"
 
-reposdir=${FTP_BASE}
-targetdir=${FTP_BASE}
-
 script_lock
 
-# location where the package DB is extracted so we know what to include
-DBDIR="$(mktemp -d ${WORKDIR}/create-filelists.dbdir.XXXXXX)" || exit 1
-# location where the old files DB is extracted to save us some work
-CACHEDIR="$(mktemp -d ${WORKDIR}/create-filelists.cachedir.XXXXXX)" || exit 1
-# location where the new files DB is built up and eventually zipped
-MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repo_lock ${repo} ${arch} || exit 1
+	done
+done
 
 #adjust the nice level to run at a lower priority
-/usr/bin/renice +10 -p $$ > /dev/null
+renice +10 -p $$ > /dev/null
+
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}"
+		filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
+
+		if [ ! -f "${repodb}" ]; then
+			continue
+		fi
+		# get a list of package files defined in the repo db
+		mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}"
+		bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}"
+		find "${WORKDIR}/db-dir-${repo}-${arch}" -name 'desc' \
+			-exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/db-${repo}-${arch}"
+
+		# get a list of package files defined in the files db
+		mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}"
+		if [ ! -f "${filedb}" ]; then
+			echo > "${WORKDIR}/files-${repo}-${arch}"
+		else
+			bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}"
+			find "${WORKDIR}/files-current-dir-${repo}-${arch}" -name 'desc' \
+				-exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/files-${repo}-${arch}"
+		fi
+	done
+done
 
 case "${FILESEXT}" in
 	*.gz)  TAR_OPT="z" ;;
@@ -26,62 +48,55 @@ case "${FILESEXT}" in
 esac
 
 for repo in ${PKGREPOS[@]}; do
-    REPO_DB_FILE="${repo}$DBEXT"
-    FILES_DB_FILE="${repo}$FILESEXT"
-    for arch in ${ARCHES[@]}; do
-        pushd "$reposdir" >/dev/null
-
-        repodir="${repo}/os/${arch}"
-        cached="no"
-
-        # extract package db archive
-        if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then
-            mkdir -p "${DBDIR}/${repodir}"
-            bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}"
-        else
-            warning "The repo $repo with arch $arch does not exist"
-            continue
-        fi
-
-        # extract old file archive
-        if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then
-            mkdir -p "${CACHEDIR}/${repodir}"
-            bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}"
-            cached="yes"
-        fi
-
-        # create file lists
-        for pkg in $(ls ${DBDIR}/${repodir}); do
-            dbpkgdir="${DBDIR}/${repodir}/${pkg}"
-            cachepkgdir="${CACHEDIR}/${repodir}/${pkg}"
-            tmppkgdir="${MYTMPDIR}/${repodir}/${pkg}"
-            mkdir -p "$tmppkgdir"
-            ln "${dbpkgdir}/desc" "${tmppkgdir}/desc"
-            ln "${dbpkgdir}/depends" "${tmppkgdir}/depends"
-            if [ -f "${cachepkgdir}/files" ]; then
-                ln "${cachepkgdir}/files" "${tmppkgdir}/files"
-            else
-                filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1)
-                echo '%FILES%' > "${tmppkgdir}/files"
-                bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files"
-                cached="no"
-            fi
-        done
-
-        # create new file archive
-        if [ "$cached" == "no" -a -d "${MYTMPDIR}/${repodir}" ]; then
-            # at least one package has changed, so let's rebuild the archive
-            pkgdir="${targetdir}/${repodir}"
-            mkdir -p "$pkgdir"
-            pushd "${MYTMPDIR}/${repodir}" >/dev/null
-            [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}"
-            bsdtar --exclude=*${FILESEXT} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" *
-            popd >/dev/null
-        fi
-        popd >/dev/null
-    done
+	for arch in ${ARCHES[@]}; do
+		filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
+
+		if [ ! -f  "${WORKDIR}/db-${repo}-${arch}" ]; then
+			# remove any files db that might be in this empty repo
+			if [ -f "${filedb}" ]; then
+				rm -f "${filedb}"
+			fi
+			continue
+		fi
+
+		# Check if updating the files db is needed
+		if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then
+			mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}"
+
+			# Include all unchanged file lists
+			# Note: deleted packages are implicitly excluded
+			for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
+				mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \
+					"${WORKDIR}/files-new-dir-${repo}-${arch}"
+			done
+
+			# Create file lists for new packages
+			for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
+				tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
+				mkdir "${tdir}"
+				echo '%FILES%' > "${tdir}/files"
+				bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files"
+
+				# add desc and depends file from db
+				dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
+				mv "${dbdir}/desc" "${tdir}/desc"
+				mv "${dbdir}/depends" "${tdir}/depends"
+			done
+
+			# Create the actual file db
+			pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null
+			bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" *
+			popd >/dev/null
+			mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}"
+			ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}"
+		fi
+	done
+done
+
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repo_unlock ${repo} ${arch}
+	done
 done
 
 script_unlock
-
-# vim: set ts=4 sw=4 et ft=sh:
diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh
new file mode 100755
index 0000000..da76710
--- /dev/null
+++ b/test/test.d/create-filelists.sh
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+curdir=$(readlink -e $(dirname $0))
+. "${curdir}/../lib/common.inc"
+
+testCreateSimpleFileLists() {
+	local arches=('i686' 'x86_64')
+	local pkgs=('pkg-simple-a' 'pkg-simple-b')
+	local pkgbase
+	local arch
+
+	for pkgbase in ${pkgs[@]}; do
+		for arch in ${arches[@]}; do
+			releasePackage extra ${pkgbase} ${arch}
+		done
+	done
+	../db-update
+
+	../cron-jobs/create-filelists
+	for pkgbase in ${pkgs[@]}; do
+		for arch in ${arches[@]}; do
+			if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgbase}"; then
+				fail "usr/bin/${pkgbase} not found in ${arch}/extra.files.tar.gz"
+			fi
+		done
+	done
+}
+
+testCreateAnyFileLists() {
+	local arches=('i686' 'x86_64')
+	local pkgs=('pkg-any-a' 'pkg-any-b')
+	local pkgbase
+	local arch
+
+	for pkgbase in ${pkgs[@]}; do
+		releasePackage extra ${pkgbase} any
+	done
+	../db-update
+
+	../cron-jobs/create-filelists
+	for pkgbase in ${pkgs[@]}; do
+		for arch in ${arches[@]}; do
+			if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/share/${pkgbase}/test"; then
+				fail "usr/share/${pkgbase}/test not found in ${arch}/extra.files.tar.gz"
+			fi
+		done
+	done
+}
+
+testCreateSplitFileLists() {
+	local arches=('i686' 'x86_64')
+	local pkgs=('pkg-split-a' 'pkg-split-b')
+	local pkg
+	local pkgbase
+	local pkgname
+	local pkgnames
+	local arch
+
+	for pkgbase in ${pkgs[@]}; do
+		for arch in ${arches[@]}; do
+			releasePackage extra ${pkgbase} ${arch}
+		done
+	done
+	../db-update
+
+	../cron-jobs/create-filelists
+	for pkgbase in ${pkgs[@]}; do
+		pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]}))
+		for pkgname in ${pkgnames[@]}; do
+			for arch in ${arches[@]}; do
+				if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgname}"; then
+					fail "usr/bin/${pkgname} not found in ${arch}/extra.files.tar.gz"
+				fi
+			done
+		done
+	done
+}
+
+
+testCleanupFileLists() {
+	local arches=('i686' 'x86_64')
+	local pkgs=('pkg-simple-a' 'pkg-simple-b')
+	local pkgbase
+	local arch
+
+	for pkgbase in ${pkgs[@]}; do
+		for arch in ${arches[@]}; do
+			releasePackage extra ${pkgbase} ${arch}
+		done
+	done
+	../db-update
+	../cron-jobs/create-filelists
+
+	for arch in ${arches[@]}; do
+		../db-remove pkg-simple-a extra ${arch}
+	done
+	../cron-jobs/create-filelists
+
+	for arch in ${arches[@]}; do
+		if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-b"; then
+			fail "usr/bin/pkg-simple-b not found in ${arch}/extra.files.tar.gz"
+		fi
+		if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-a"; then
+			fail "usr/bin/pkg-simple-a still found in ${arch}/extra.files.tar.gz"
+		fi
+	done
+
+}
+
+. "${curdir}/../lib/shunit2"