Using own db-update and ftpdir-cleanup

3 files changed, 190 insertions, 1 deletions

diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup
new file mode 100755
index 0000000..09e8a49
--- /dev/null
+++ b/cron-jobs/ftpdir-cleanup
@@ -0,0 +1,113 @@
+#!/bin/bash
+
+. "$(dirname $0)/../db-functions"
+. "$(dirname $0)/../config"
+
+clean_pkg() {
+	local pkg
+	local target
+
+	if ! ${CLEANUP_DRYRUN}; then
+		for pkg in "$@"; do
+			if [ -h "$pkg" ]; then
+				rm -f "$pkg"
+			else
+				mv -f "$pkg" "$CLEANUP_DESTDIR"
+				touch "${CLEANUP_DESTDIR}/$(basename ${pkg})"
+			fi
+		done
+	fi
+}
+
+script_lock
+
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repo_lock ${repo} ${arch} || exit 1
+	done
+done
+
+${CLEANUP_DRYRUN} && warning 'dry run mode is active'
+
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then
+			continue
+		fi
+		# get a list of actual available package files
+		find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}"
+		# get a list of package files defined in the repo db
+		bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}"
+
+		missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}"))
+		if [ ${#missing_pkgs[@]} -ge 1 ]; then
+			error "Missing packages in [${repo}] (${arch})..."
+			for missing_pkg in ${missing_pkgs[@]}; do
+				msg2 "${missing_pkg}"
+			done
+		fi
+
+		old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}"))
+		if [ ${#old_pkgs[@]} -ge 1 ]; then
+			msg "Removing old packages from [${repo}] (${arch})..."
+			for old_pkg in ${old_pkgs[@]}; do
+				msg2 "${old_pkg}"
+				clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}"
+			done
+		fi
+	done
+done
+
+# get a list of all available packages in the pacakge pool
+find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool"
+# create a list of packages in our db
+cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db"
+
+old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db"))
+if [ ${#old_pkgs[@]} -ge 1 ]; then
+	msg "Removing old packages from package pool..."
+	for old_pkg in ${old_pkgs[@]}; do
+		msg2 "${old_pkg}"
+		clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}"
+	done
+fi
+
+# cleanup of legacy $repo/os/any directories
+for repo in ${PKGREPOS[@]}; do
+	if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then
+		continue
+	fi
+	if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then
+		msg "Removing empty legacy directory ${repo}/os/any"
+		${CLEANUP_DRYRUN} || rmdir "${FTP_BASE}/${repo}/os/any"
+		continue
+	fi
+	find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}"
+	cat "${WORKDIR}/db-${repo}-"* | sort -u > "${WORKDIR}/all-${repo}"
+
+	old_pkgs=($(comm -23 "${WORKDIR}/any-${repo}" "${WORKDIR}/all-${repo}"))
+	if [ ${#old_pkgs[@]} -ge 1 ]; then
+		msg "Removing old packages from [${repo}] (any)..."
+		for old_pkg in ${old_pkgs[@]}; do
+			msg2 "${old_pkg}"
+			clean_pkg "${FTP_BASE}/${repo}/os/any/${old_pkg}"
+		done
+	fi
+done
+
+old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n'))
+if [ ${#old_pkgs[@]} -ge 1 ]; then
+	msg "Removing old packages from the cleanup directory..."
+	for old_pkg in ${old_pkgs[@]}; do
+		msg2 "${old_pkg}"
+		${CLEANUP_DRYRUN} || rm -f "${CLEANUP_DESTDIR}/${old_pkg}"
+	done
+fi
+
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repo_unlock ${repo} ${arch}
+	done
+done
+
+script_unlock
diff --git a/db-update b/db-update
new file mode 100755
index 0000000..4740809
--- /dev/null
+++ b/db-update
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+. "$(dirname $0)/db-functions"
+. "$(dirname $0)/config"
+
+if [ $# -ge 1 ]; then
+	warning "Calling $(basename $0) with a specific repository is no longer supported"
+	exit 1
+fi
+
+# Find repos with packages to release
+repos=($(find  "${STAGING}" -mindepth 1 -type d ! -empty -printf '%f ' 2>/dev/null))
+if [ $? -ge 1 ]; then
+	die "Could not read ${STAGING}"
+fi
+
+# TODO: this might lock too much (architectures)
+for repo in ${repos[@]}; do
+	for pkgarch in ${ARCHES[@]}; do
+		repo_lock ${repo} ${pkgarch} || exit 1
+	done
+done
+
+# check if packages are valid
+for repo in ${repos[@]}; do
+	if ! check_repo_permission "${repo}"; then
+		die "You don't have permission to update packages in ${repo}"
+	fi
+	pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT}))
+	if [ $? -eq 0 ]; then
+		for pkg in ${pkgs[@]}; do
+			if [ -h "${pkg}" ]; then
+				die "Package ${repo}/$(basename ${pkg}) is a symbolic link"
+			fi
+			if ! check_pkgfile "${pkg}"; then
+				die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data"
+			fi
+			#if ! check_pkgrepos "${pkg}"; then
+			#	die "Package ${repo}/$(basename ${pkg}) already exists in another repository"
+			#fi
+		done
+		if ! check_splitpkgs ${repo} ${pkgs[@]}; then
+			die "Missing split packages for ${repo}"
+		fi
+	else
+		die "Could not read ${STAGING}"
+	fi
+done
+
+for repo in ${repos[@]}; do
+	msg "Updating [${repo}]..."
+	any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null))
+	for pkgarch in ${ARCHES[@]}; do
+		add_pkgs=()
+		arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null))
+		for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do
+			pkgfile="$(basename ${pkg})"
+			msg2 "${pkgfile} (${pkgarch})"
+			# any packages might have been moved by the previous run
+			if [ -f "${pkg}" ]; then
+				mv "${pkg}" "$FTP_BASE/${PKGPOOL}"
+			fi
+			ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}"
+			add_pkgs[${#add_pkgs[*]}]=${pkgfile}
+		done
+		if [ ${#add_pkgs[@]} -ge 1 ]; then
+			arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]}
+		fi
+	done
+done
+
+for repo in ${repos[@]}; do
+	for pkgarch in ${ARCHES[@]}; do
+		repo_unlock ${repo} ${pkgarch}
+	done
+done
diff --git a/repo-update b/repo-update
index 7038789..ff7f8e0 100755
--- a/repo-update
+++ b/repo-update
@@ -4,7 +4,7 @@ source ~/.bashrc # the PATH is updated here
 
 # Updates free/ databases and releases packages on free/staging
 # Also moves old packages to the old/packages dir
-db-update
+~/repm/db-update
 ~/repm/cron-jobs/ftpdir-cleanup
 
 # Syncs repos