diff options
| -rwxr-xr-x | src/bin/db-init | 5 | ||||
| -rwxr-xr-x | src/bin/db-pkg-add | 5 | ||||
| -rw-r--r-- | src/etc/xbs.cfg | 41 | ||||
| -rw-r--r-- | src/etc/xbs.d/main.cfg | 36 | ||||
| -rw-r--r-- | src/share/xbs/db-functions | 456 |
5 files changed, 499 insertions, 44 deletions
diff --git a/src/bin/db-init b/src/bin/db-init index 37ad299..9fe243e 100755 --- a/src/bin/db-init +++ b/src/bin/db-init @@ -3,9 +3,8 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc" -source "${CONFIG_DIR}/dbscripts.d/db-functions.cfg" -source "${CONFIG_DIR}/dbscripts.d/main.cfg" -source "${CONFIG_DIR}/dbscripts.cfg" +source "${CONFIG_DIR}/xbs.cfg" +source "${SHARE_DIR}/db-functions" db-init_usage() { msg "usage: %s [platform1 platform2 ... platformX]" "${0##*/}" diff --git a/src/bin/db-pkg-add b/src/bin/db-pkg-add index 28483b8..26fa483 100755 --- a/src/bin/db-pkg-add +++ b/src/bin/db-pkg-add @@ -2,9 +2,8 @@ CONFIG_DIR="$(dirname "$(readlink -e "$0")")/etc" -source "${CONFIG_DIR}/dbscripts.d/db-functions.cfg" -source "${CONFIG_DIR}/dbscripts.d/main.cfg" -source "${CONFIG_DIR}/dbscripts.cfg" +source "${CONFIG_DIR}/xbs.cfg" +source "${SHARE_DIR}/db-functions" db-pkg-add_usage() { "usage: %s [-p platform] <repo> <arch> <pkgfile> ..." "${0##*/}" diff --git a/src/etc/xbs.cfg b/src/etc/xbs.cfg index 46854ef..2920dce 100644 --- a/src/etc/xbs.cfg +++ b/src/etc/xbs.cfg @@ -1,6 +1,43 @@ #!/bin/bash -# Platforms Distributions -ND_PLATFORMS=('gnu+linux_nd') # 'gnu+hurd_d' +# [ Native || Derivative ] Distribution Platforms +ND_PLATFORMS=('gnu+linux_nd') # 'gnu+hurd_nd' DD_PLATFORMS=('gnu+linux_dd' 'gnu+linux-arm_dd') # 'gnu+hurd_dd' PLATFORMS=("${ND_PLATFORMS[@]}" "${DD_PLATFORMS[@]}") + +# Software Development Distributions +SDD=('rolling' 'testing' 'stable') + +ROLLING_NAME='rolling' +TESTING_NAME='waterbird-v2' +STABLE_NAME='minicat-v1' + +PKG_POOL_DIR='pools' +SRC_POOL_DIR='sources' + +SHARE_DIR="$(dirname "$(readlink -e "$0")")/share/xbs" + +SRV_DIR='/srv' +TMP_DIR='/tmp' +PKG_POOL_DIR='pools' +SRC_POOL_DIR='sources' +GIT_BASE="${SRV_DIR}/git" +SYNC_ARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' + +REQUIRE_SIGNATURE='true' + +LOCK_DELAY='10' +LOCK_TIMEOUT='300' + +[ -n "${STAGING:-}" ] || STAGING="${HOME}/staging/unknown/staging" +DB_EXT='.db.tar.gz' +FILES_EXT='.files.tar.gz' +PKG_EXT=".pkg.tar.?z" +SRC_EXT='.src.tar.gz' + +MIRROR_LIST='/etc/pacman.d/mirrorlist' +MAKEPKG_CONF='/etc/makepkg.conf' +MAKEPKG_ARCH="$(uname -m)" + +# web root +WEB_DIR="${SRV_DIR}/http/www.parabola.nu/web" diff --git a/src/etc/xbs.d/main.cfg b/src/etc/xbs.d/main.cfg deleted file mode 100644 index 906ea0c..0000000 --- a/src/etc/xbs.d/main.cfg +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Software Development Distribution -SDD=('rolling' 'testing' 'stable') - -ROLLING_NAME='rolling' -TESTING_NAME='waterbird-v2' -STABLE_NAME='minicat-v1' - -PKG_POOL_DIR='pools' -SRC_POOL_DIR='sources' - -SRV_DIR='/srv' -TMP_DIR='/tmp' -PKG_POOL_DIR='pools' -SRC_POOL_DIR='sources' -GIT_BASE="${SRV_DIR}/git" -SYNC_ARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' - -REQUIRE_SIGNATURE='true' - -LOCK_DELAY='10' -LOCK_TIMEOUT='300' - -[ -n "${STAGING:-}" ] || STAGING="${HOME}/staging/unknown/staging" -DB_EXT='.db.tar.gz' -FILES_EXT='.files.tar.gz' -PKG_EXT=".pkg.tar.?z" -SRC_EXT='.src.tar.gz' - -MIRROR_LIST='/etc/pacman.d/mirrorlist' -MAKEPKG_CONF='/etc/makepkg.conf' -MAKEPKG_ARCH="$(uname -m)" - -# web root -WEB_DIR="${SRV_DIR}/http/www.parabola.nu/web" diff --git a/src/share/xbs/db-functions b/src/share/xbs/db-functions new file mode 100644 index 0000000..62260bb --- /dev/null +++ b/src/share/xbs/db-functions @@ -0,0 +1,456 @@ +#!/hint/bash + +# Some PKGBUILDs need CARCH to be set +CARCH=$(. "$(librelib conf.sh)"; load_files makepkg; echo "$CARCH") + +# Useful functions +UMASK="" +set_umask () { + [ "$UMASK" == "" ] && UMASK="$(umask)" + export UMASK + umask 002 +} + +restore_umask () { + umask "$UMASK" >/dev/null +} + +# just like mv -f, but we touch the file and then copy the content so +# default ACLs in the target dir will be applied +mv_acl() { + rm -f "$2" + touch "$2" + cat "$1" >"$2" || return 1 + rm -f "$1" +} + +# set up general environment +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +if [ -n "${SVNUSER}" ]; then + setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" + setfacl -m d:u:"${USER}":rwx "${WORKDIR}" + setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" +fi +LOCKS=() +REPO_MODIFIED=0 + +# Used: plain, msg, msg2, warning, error, in_array, get_full_version, abort, die +# Overwritten: cleanup +# Ignored: stat_busy, stat_done, +# setup_workdir, trap_abort, trap_exit, +# lock, slock, lock_close +# pkgver_equal, find_cached_package, check_root +. "$(librelib common)" + +script_lock() { + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" + if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then + local _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" + error "Script %s is already locked by %s." "${0##*/}" "$_owner" + exit 1 + else + set_umask + return 0 + fi +} + +script_unlock() { + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" + if [ ! -d "$LOCKDIR" ]; then + warning "Script %s was not locked!" "${0##*/}" + restore_umask + return 1 + else + rmdir "$LOCKDIR" + restore_umask + return 0 + fi +} + +cleanup() { + local l + local repo + local arch + + trap - EXIT INT QUIT TERM + for l in "${LOCKS[@]}"; do + repo=${l%.*} + arch=${l#*.} + if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then + msg "Removing left over lock from [%s] (%s)" "${repo}" "${arch}" + repo_unlock "$repo" "$arch" + fi + done + if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then + msg "Removing left over lock from %s" "${0##*/}" + script_unlock + fi + rm -rf "$WORKDIR" + + if (( REPO_MODIFIED )); then + date +%s > "${FTP_BASE}/lastupdate" + fi + + [ "$1" ] && exit "$1" +} + +trap abort INT QUIT TERM HUP +trap cleanup EXIT + + +#repo_lock <repo-name> <arch> [timeout] +repo_lock () { + local LOCKDIR="$TMPDIR/.repolock.$1.$2" + local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" + local FILESLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${FILESEXT}.lck" + local _count + local _trial + local _timeout + local _lockblock + local _owner + + # This is the lock file used by repo-add and repo-remove + if [ -f "${DBLOCKFILE}" ]; then + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$DBLOCKFILE")" + return 1 + fi + if [ -f "${FILESLOCKFILE}" ]; then + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$FILESLOCKFILE")" + return 1 + fi + + if [ $# -eq 2 ]; then + _lockblock=true + _trial=0 + elif [ $# -eq 3 ]; then + _lockblock=false + _timeout=$3 + let _trial=$_timeout/$LOCK_DELAY + fi + + _count=0 + while [ "$_count" -le "$_trial" ] || "$_lockblock" ; do + if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then + _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" + warning "Repo [%s] (%s) is already locked by %s." "${1}" "${2}" "$_owner" + msg2 "Retrying in %d seconds..." "$LOCK_DELAY" + else + LOCKS+=("$1.$2") + set_umask + return 0 + fi + sleep "$LOCK_DELAY" + let _count=$_count+1 + done + + error "Repo [%s] (%s) is already locked by %s. Giving up!" "${1}" "${2}" "$_owner" + return 1 +} + +repo_unlock () { #repo_unlock <repo-name> <arch> + local LOCKDIR="$TMPDIR/.repolock.$1.$2" + if [ ! -d "$LOCKDIR" ]; then + warning "Repo lock [%s] (%s) was not locked!" "${1}" "${2}" + restore_umask + return 1 + else + rmdir "$LOCKDIR" + restore_umask + return 0 + fi +} + +# usage: _grep_pkginfo pkgfile pattern +_grep_pkginfo() { + local _ret + + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" + echo "${_ret#${2} = }" +} + + +# Get the package base or name as fallback +getpkgbase() { + local _base + + _base="$(_grep_pkginfo "$1" "pkgbase")" + if [ -z "$_base" ]; then + getpkgname "$1" + else + echo "$_base" + fi +} + +issplitpkg() { + local _base + + _base="$(_grep_pkginfo "$1" "pkgbase")" + if [ -z "$_base" ]; then + return 1 + else + return 0 + fi +} + +# Get the package name +getpkgname() { + local _name + + _name="$(_grep_pkginfo "$1" "pkgname")" + if [ -z "$_name" ]; then + error "Package '%s' has no pkgname in the PKGINFO. Fail!" "$1" + exit 1 + fi + + echo "$_name" +} + +# Get the pkgver-pkgrel of this package +getpkgver() { + local _ver + + _ver="$(_grep_pkginfo "$1" "pkgver")" + if [ -z "$_ver" ]; then + error "Package '%s' has no pkgver in the PKGINFO. Fail!" "$1" + exit 1 + fi + + echo "$_ver" +} + +getpkgarch() { + local _ver + + _ver="$(_grep_pkginfo "$1" "arch")" + if [ -z "$_ver" ]; then + error "Package '%s' has no arch in the PKGINFO. Fail!" "$1" + exit 1 + fi + + echo "$_ver" +} + +check_packager() { + local _packager + + _packager=$(_grep_pkginfo "$1" "packager") + [[ $_packager && $_packager != 'Unknown Packager' ]] +} + +getpkgfile() { + if [[ ${#} -ne 1 ]]; then + error 'No canonical package found!' + exit 1 + elif [ ! -f "${1}" ]; then + error "Package %s not found!" "${1}" + exit 1 + elif "${REQUIRE_SIGNATURE}" && [ ! -f "${1}.sig" ]; then + error "Package signature %s not found!" "${1}.sig" + exit 1 + fi + + echo "${1}" +} + +getpkgfiles() { + local f + if [ ! -z "$(printf '%s\n' "${@%\.*}" | sort | uniq -D)" ]; then + error 'Duplicate packages found!' + exit 1 + fi + + for f in "${@}"; do + if [ ! -f "${f}" ]; then + error "Package %s not found!" "${f}" + exit 1 + elif "${REQUIRE_SIGNATURE}" && [ ! -f "${f}.sig" ]; then + error "Package signature %s not found!" "${f}.sig" + exit 1 + fi + done + + echo "${@}" +} + +check_pkgfile() { + local pkgfile=$1 + + local pkgname="$(getpkgname "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local pkgver="$(getpkgver "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local pkgarch="$(getpkgarch "${pkgfile}")" + [ $? -ge 1 ] && return 1 + + in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 + + if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then + return 0 + else + return 1 + fi +} + +check_pkgxbs() { + local pkgfile="${1}" + local _pkgbase="$(getpkgbase "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local _pkgname="$(getpkgname "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local _pkgver="$(getpkgver "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local _pkgarch="$(getpkgarch "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local repo="${2}" + + in_array "${repo}" "${PKGREPOS[@]}" || return 1 + + local xbsver="$(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; get_full_version "${_pkgname}")" + [ "${xbsver}" == "${_pkgver}" ] || return 1 + + local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) + in_array "${_pkgname}" "${xbsnames[@]}" || return 1 + + return 0 +} + +check_splitpkgs() { + local repo="${1}" + shift + local pkgfiles=("${@}") + local pkgfile + local pkgdir + local xbsname + + mkdir -p "${WORKDIR}/check_splitpkgs/" + pushd "${WORKDIR}/check_splitpkgs" >/dev/null + + for pkgfile in "${pkgfiles[@]}"; do + issplitpkg "${pkgfile}" || continue + local _pkgbase="$(getpkgbase "${pkgfile}")" + local _pkgname="$(getpkgname "${pkgfile}")" + local _pkgarch="$(getpkgarch "${pkgfile}")" + mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" + echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" + + local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) + printf '%s\n' "${xbsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" + done + popd >/dev/null + + for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do + [ ! -d "${pkgdir}" ] && continue + sort -u "${pkgdir}/staging" -o "${pkgdir}/staging" + sort -u "${pkgdir}/xbs" -o "${pkgdir}/xbs" + if [ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/xbs")" ]; then + return 1 + fi + done + + return 0 +} + +check_pkgrepos() { + local pkgfile=$1 + + local pkgname="$(getpkgname "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local pkgver="$(getpkgver "${pkgfile}")" + [ $? -ge 1 ] && return 1 + local pkgarch="$(getpkgarch "${pkgfile}")" + [ $? -ge 1 ] && return 1 + + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 + + return 0 +} + +#usage: chk_license ${license[@]}" +chk_license() { + local l + for l in "${@}"; do + in_array "${l}" "${ALLOWED_LICENSES[@]}" && return 0 + done + + return 1 +} + +check_repo_permission() { + local repo=$1 + + [ ${#PKGREPOS[@]} -eq 0 ] && return 1 + [ -z "${PKGPOOL}" ] && return 1 + + in_array "${repo}" "${PKGREPOS[@]}" || return 1 + + [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 + + local arch + for arch in "${ARCHES[@]}"; do + local dir="${FTP_BASE}/${repo}/os/${arch}/" + [ -w "${dir}" ] || return 1 + [ -f "${dir}${repo}"${DBEXT} -a ! -w "${dir}${repo}"${DBEXT} ] && return 1 + [ -f "${dir}${repo}"${FILESEXT} -a ! -w "${dir}${repo}"${FILESEXT} ] && return 1 + done + + return 0 +} + +set_repo_permission() { + local repo=$1 + local arch=$2 + local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ -w "${dbfile}" ]; then + local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "${dbfile}" "$group" + chgrp "$group" "${filesfile}" || error "Could not change group of %s to %s" "${filesfile}" "$group" + chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "${dbfile}" + chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "${filesfile}" + else + error "You don't have permission to change %s" "${dbfile}" + fi +} + +arch_repo_add() { + local repo=$1 + local arch=$2 + local pkgs=("${@:3}") + + printf -v pkgs_str -- '%q ' "${pkgs[@]}" + # package files might be relative to repo dir + pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null + /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" \ + || error 'repo-add %q %s' "${repo}${DBEXT}" "${pkgs_str% }" + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ + || error 'repo-add -f %q %s' "${repo}${FILESEXT}" "${pkgs_str% }" + popd >/dev/null + set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 +} + +arch_repo_remove() { + local repo=$1 + local arch=$2 + local pkgs=("${@:3}") + local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${dbfile}" ]; then + error "No database found at '%s'" "${dbfile}" + return 1 + fi + printf -v pkgs_str -- '%q ' "${pkgs[@]}" + /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" \ + || error 'repo-remove %q %s' "${dbfile}" "${pkgs_str% }" + /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ + || error 'repo-remove %q %s' "${filesfile}" "${pkgs_str% }" + set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 +} |