From 61b19919818a0b87aef5544ce2853f2ef2b76e58 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Sun, 22 Jan 2012 14:48:51 -0300 Subject: Fullpkg redone :) --- treepkg | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100755 treepkg diff --git a/treepkg b/treepkg new file mode 100755 index 0000000..4ad02fc --- /dev/null +++ b/treepkg @@ -0,0 +1,125 @@ +#!/bin/bash +#set -x +source /etc/libretools.conf +source $(dirname $0)/libremessages +source $XDG_CONFIG_HOME/libretools/libretools.conf >/dev/null 2>&1|| true + +# End inmediately but print a useful message +trap_exit() { + error "($(basename $0)) $@" + exit 1 +} + +# Trap signals from makepkg +set -E +trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT +trap 'trap_exit "Aborted by user! Exiting..."' INT +trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR + +# return : full version spec, including epoch (if necessary), pkgver, pkgrel +# usage : get_fullver( ${epoch:-0}, $pkgver, $pkgrel ) +get_fullver() { + if [ $1 -eq 0 ]; then +# zero epoch case, don't include it in version + echo $2-$3 + else + echo $1:$2-$3 + fi + +} + +# Add line to build order cache in CSV format +# $1 status +# $2 pkgname +add_order() { + echo "${1};${DEPTH};${2:-${pkgbase}};${fullver};${PWD}" >> "${BUILDORDER}" + ${VERBOSE} && msg2 "%${DEPTH}s${2:-${pkgbase}} [${1}]" || true +} + +# Finds a PKGBUILD on toru's path cache +# Look in all caches but pick the first one +# TODO move to a toru flag (-p?) +where_is() { + grep -m1 "^${1}:" "${TORUPATH}/paths" 2>/dev/null| \ + cut -d: -f2 2>/dev/null +} + +if [ ! -f PKGBUILD ]; then + error "Missing PKGBUILD" + exit 1 +fi + +if ! source PKGBUILD ; then + error "Can't source PKGBUILD" + exit 1 +fi + +# Save resources +unset pkgdesc arch license groups backup install md5sums sha1sums \ + sha256sums source options >/dev/null 2>&1 + +unset build package >/dev/null 2>&1 + +for _pkg in ${pkgname[@]}; do + unset package_${_pkg} >/dev/null 2>&1 || true +done +## + +# Get useful values +pkgbase="${pkgbase:-${pkgname[0]}}" +fullver=$(get_fullver ${epoch:-0} ${pkgver} ${pkgrel}) + +# Get or set the build order cache (canonical) +BUILDORDER="$(readlink -f "${1:-$(mktemp /tmp/${pkgbase}.buildorder.XXXX)}")" +DEPTH=${2:-0} +NEXTDEPTH=$((${DEPTH} + 1)) +VERBOSE=true + +# ensure it exists +touch "${BUILDORDER}" + +# If this package is already built quit silently +if is_built "${pkgbase}>=${fullver}"; then + add_order "ignore" + exit 0 +fi + +# Ignore if already in build order +egrep -q ";${pkgbase};" "${BUILDORDER}" && exit 0 + +# Add pkgbase to build order +add_order "build" + +# Cleanup dep versioning +deps=($(echo "${depends[@]} ${makedepends[@]}" | \ + sed "s/[=<>]\+[^ ]\+//g" | \ + tr ' ' "\n" | \ + sort -u)) + +# NOTE: getting depends from package() is a PITA +for _dep in ${deps[@]}; do +# Ignore if already in build order + egrep -q ";${_dep};" "${BUILDORDER}" && continue + + depdir="$(where_is ${_dep})" + + if [ -z "${depdir}" -o ! -d "${depdir}" ]; then +# We specify the pkgname because we can't source the dep PKGBUILD + add_order "missing" "${_dep}" + continue + fi + + pushd "${depdir}" >/dev/null + +# Run itself over dependencies + $0 "${BUILDORDER}" ${NEXTDEPTH} + +done + +# Only print build order at the end +if [ ${DEPTH} -eq 0 ]; then + ${VERBOSE} && msg "Build tree stored in ${BUILDORDER}" || true + ${VERBOSE} || echo "${BUILDORDER}" || true +fi + +exit $? -- cgit v1.2.3-54-g00ecf From 787ef61e1997deeda97e055f7f0fe27a1b7ab2b5 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Sun, 22 Jan 2012 22:53:26 -0300 Subject: Create a work dir and build in order --- treepkg | 51 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/treepkg b/treepkg index 4ad02fc..622b979 100755 --- a/treepkg +++ b/treepkg @@ -69,11 +69,15 @@ done pkgbase="${pkgbase:-${pkgname[0]}}" fullver=$(get_fullver ${epoch:-0} ${pkgver} ${pkgrel}) -# Get or set the build order cache (canonical) -BUILDORDER="$(readlink -f "${1:-$(mktemp /tmp/${pkgbase}.buildorder.XXXX)}")" +# Get or set the work dir +BUILDDIR="${1:-$(mktemp -d /tmp/${pkgbase}-treepkg-XXXx)}" +BUILDORDER="${BUILDDIR}/BUILDORDER" DEPTH=${2:-0} NEXTDEPTH=$((${DEPTH} + 1)) -VERBOSE=true +# This can be set as env vars (ie: $ V=false B=false treepkg) +# TODO Turn into flags? +VERBOSE=${V:-true} +BUILD=${B:-true} # ensure it exists touch "${BUILDORDER}" @@ -90,6 +94,10 @@ egrep -q ";${pkgbase};" "${BUILDORDER}" && exit 0 # Add pkgbase to build order add_order "build" +# Copy the directory to the build dir +# TODO run makepkg --source to avoid moving garbage around? +cp -r "${PWD}" "${BUILDDIR}/$(printf "%03d" ${DEPTH})_${pkgbase}" + # Cleanup dep versioning deps=($(echo "${depends[@]} ${makedepends[@]}" | \ sed "s/[=<>]\+[^ ]\+//g" | \ @@ -99,12 +107,18 @@ deps=($(echo "${depends[@]} ${makedepends[@]}" | \ # NOTE: getting depends from package() is a PITA for _dep in ${deps[@]}; do # Ignore if already in build order +# TODO move deps deeper in the tree if +# pkgbase - dep1 +# \ dep2 - dep1 +# dep1 should be depth + 1 egrep -q ";${_dep};" "${BUILDORDER}" && continue +# Ask toru where's a PKGBUILD depdir="$(where_is ${_dep})" if [ -z "${depdir}" -o ! -d "${depdir}" ]; then # We specify the pkgname because we can't source the dep PKGBUILD +# Normally 'any' packages are missing from our work ABS add_order "missing" "${_dep}" continue fi @@ -112,14 +126,37 @@ for _dep in ${deps[@]}; do pushd "${depdir}" >/dev/null # Run itself over dependencies - $0 "${BUILDORDER}" ${NEXTDEPTH} + $0 "${BUILDDIR}" ${NEXTDEPTH} done -# Only print build order at the end +# Only build at the end if [ ${DEPTH} -eq 0 ]; then - ${VERBOSE} && msg "Build tree stored in ${BUILDORDER}" || true - ${VERBOSE} || echo "${BUILDORDER}" || true + +# TODO fill this var with a flag + if ${BUILD}; then + ${VERBOSE} && msg "Build tree stored in ${BUILDORDER}" || true + +# Build everything sorting the build dir +# The reverse order ensures we start by the deepest packages + ${VERBOSE} && msg "Starting build" || true + for _pkg in $(ls -r "${BUILDDIR}"); do +# Ignore if there's no PKGBUILD + if [ ! -f "${BUILDDIR}/${_pkg}/PKGBUILD" ]; then continue; fi + + ${VERBOSE} && msg "Building ${_pkg/_/ }" || true + +# Run build command + pushd "${BUILDDIR}/${_pkg}" >/dev/null + ${FULLBUILDCMD} + popd >/dev/null + done + + else +# Just print the working dir + ${VERBOSE} || echo "${BUILDORDER}" || true + fi + fi exit $? -- cgit v1.2.3-54-g00ecf From 0f72189f9a62af346592ce54a55810f973d8d7ed Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Sun, 22 Jan 2012 23:32:00 -0300 Subject: Quicker is_built, only one call to pacman time shows this version is 1s long against 5s from before --- fullpkg-find | 2 +- is_built | 31 ++++++++++++++++++++++--------- treepkg | 5 ++--- 3 files changed, 25 insertions(+), 13 deletions(-) mode change 100644 => 100755 fullpkg-find diff --git a/fullpkg-find b/fullpkg-find old mode 100644 new mode 100755 index d253913..8c0c063 --- a/fullpkg-find +++ b/fullpkg-find @@ -47,7 +47,7 @@ find_deps() { fi fi - if is_built "${pkgbase}>=${fullver}"; then + if is_built "${pkgbase}" "${fullver}"; then exit 0 # pkg is built and updated fi diff --git a/is_built b/is_built index 60f24e9..c01ad53 100755 --- a/is_built +++ b/is_built @@ -2,9 +2,10 @@ usage() { echo "$0 " echo - echo "Detect is a package is installed or in a database" + echo "Detect if a given package version is already in repos" + echo "Assuming you want greater or equal" echo - echo "Example usage: is_built \"pcre>=20\"" + echo "Example usage: is_built 'pcre' '20'" } while getopts 'h' arg; do @@ -14,12 +15,24 @@ while getopts 'h' arg; do esac done -# Checks for package, if -T returns non-zero output, egrep will return 0 -# because it finds it, so we negate the value to say it's not built. -# -Sp works backwards, it will print output only when the package already -# exists +set -x -!(sudo pacman -T "$1" | egrep "*" >/dev/null) || \ -sudo pacman -Sp "$1" --print-format "%n-%v" 2>/dev/null | egrep "*" >/dev/null +ver=${2} +pkg=${1} +pver=$(LC_ALL=C pacman -Sp --print-format "%v" "${pkg}" 2>/dev/null) -exit $? +# if pacman fails or returns nothing +r=$? +[ "${pver}" = " there is nothing to do" ] && r=1 + +result=$(vercmp "${pver}" "${ver}") + +# if vercmp > 1 means our version is bigger +if [ ${result} -ge 0 -a ${r} -eq 0 ]; then + exit 0 +else + exit 1 +fi + +# just in case +exit 1 diff --git a/treepkg b/treepkg index 622b979..ccabe20 100755 --- a/treepkg +++ b/treepkg @@ -83,7 +83,7 @@ BUILD=${B:-true} touch "${BUILDORDER}" # If this package is already built quit silently -if is_built "${pkgbase}>=${fullver}"; then +if is_built "${pkgbase}" "${fullver}"; then add_order "ignore" exit 0 fi @@ -132,14 +132,13 @@ done # Only build at the end if [ ${DEPTH} -eq 0 ]; then + ${VERBOSE} && msg "Starting build" || true -# TODO fill this var with a flag if ${BUILD}; then ${VERBOSE} && msg "Build tree stored in ${BUILDORDER}" || true # Build everything sorting the build dir # The reverse order ensures we start by the deepest packages - ${VERBOSE} && msg "Starting build" || true for _pkg in $(ls -r "${BUILDDIR}"); do # Ignore if there's no PKGBUILD if [ ! -f "${BUILDDIR}/${_pkg}/PKGBUILD" ]; then continue; fi -- cgit v1.2.3-54-g00ecf From 96697c7de79d18524eb168dd6d4d7cad412b0719 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Mon, 23 Jan 2012 11:43:32 -0300 Subject: Use -dd to avoid having dependencies versions too --- is_built | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/is_built b/is_built index c01ad53..99b871b 100755 --- a/is_built +++ b/is_built @@ -15,11 +15,9 @@ while getopts 'h' arg; do esac done -set -x - ver=${2} pkg=${1} -pver=$(LC_ALL=C pacman -Sp --print-format "%v" "${pkg}" 2>/dev/null) +pver=$(LC_ALL=C pacman -Spdd --print-format "%v" "${pkg}" 2>/dev/null) # if pacman fails or returns nothing r=$? -- cgit v1.2.3-54-g00ecf From 7d6e9b08d749983263ed3ee890f73d13e73740c8 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Sun, 5 Feb 2012 02:45:18 -0300 Subject: Make treepkg run HOOKLOCALRELEASE and mipsrelease update the databases treepkg should stay away of updating databases, since it can be used inside or outside the chroot. (updating dbs from outside is unnecessary) --- mips64el/mipsrelease | 2 ++ treepkg | 1 + 2 files changed, 3 insertions(+) diff --git a/mips64el/mipsrelease b/mips64el/mipsrelease index 1a4aade..e1035aa 100755 --- a/mips64el/mipsrelease +++ b/mips64el/mipsrelease @@ -57,4 +57,6 @@ popd >/dev/null $libretoolsdir/chcleanup || true +sudo pacman -Sy + exit $? diff --git a/treepkg b/treepkg index ccabe20..376af91 100755 --- a/treepkg +++ b/treepkg @@ -148,6 +148,7 @@ if [ ${DEPTH} -eq 0 ]; then # Run build command pushd "${BUILDDIR}/${_pkg}" >/dev/null ${FULLBUILDCMD} + ${HOOKLOCALRELEASE} popd >/dev/null done -- cgit v1.2.3-54-g00ecf From 6225a97d6d1102d8d5fc85dc59ad91b60f416a66 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Sun, 5 Feb 2012 03:38:41 -0300 Subject: Use libretools' cleansystem --- chcleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chcleanup b/chcleanup index 83c9f3f..a893320 100755 --- a/chcleanup +++ b/chcleanup @@ -2,6 +2,6 @@ [ ! -f ~/cleansystem ] && exit 1 -sudo pacman --noconfirm -Rcs $(comm -23 <(pacman -Qq | sort) <(sort ~/cleansystem)) +sudo pacman --noconfirm -Rcs $(comm -23 <(pacman -Qq | sort) <(sort /etc/libretools.d/cleansystem)) exit $? -- cgit v1.2.3-54-g00ecf From 94b94c9ea476e4e791219bd2c017741a3274bbc5 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Mon, 6 Feb 2012 02:26:50 -0300 Subject: Treepkg guess repo and put it on BUILDORDER. Also remove libremessages since libretools.conf does this for us. --- mips64el/mipsrelease | 5 +++-- treepkg | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/mips64el/mipsrelease b/mips64el/mipsrelease index e1035aa..0d14c72 100755 --- a/mips64el/mipsrelease +++ b/mips64el/mipsrelease @@ -48,6 +48,9 @@ done repo-add ${PKGDEST}/stage3.db.tar.gz ${pkgs[@]} +sudo pacman -Sy +librestage ${repo} + mkdir -p ${WORKDIR}/abs/${CARCH}/${repo} >/dev/null @@ -57,6 +60,4 @@ popd >/dev/null $libretoolsdir/chcleanup || true -sudo pacman -Sy - exit $? diff --git a/treepkg b/treepkg index 376af91..167ccbd 100755 --- a/treepkg +++ b/treepkg @@ -1,9 +1,12 @@ #!/bin/bash #set -x source /etc/libretools.conf -source $(dirname $0)/libremessages source $XDG_CONFIG_HOME/libretools/libretools.conf >/dev/null 2>&1|| true +# Get system variables +source /etc/makepkg.conf +source $HOME/makepkg.conf >/dev/null 2>&1|| true + # End inmediately but print a useful message trap_exit() { error "($(basename $0)) $@" @@ -32,7 +35,7 @@ get_fullver() { # $1 status # $2 pkgname add_order() { - echo "${1};${DEPTH};${2:-${pkgbase}};${fullver};${PWD}" >> "${BUILDORDER}" + echo "${1};${DEPTH};${2:-${pkgbase}};${fullver};${PWD};$(guess_repo "$PWD")" >> "${BUILDORDER}" ${VERBOSE} && msg2 "%${DEPTH}s${2:-${pkgbase}} [${1}]" || true } @@ -44,8 +47,14 @@ where_is() { cut -d: -f2 2>/dev/null } +# Guess the repo from the pkgbase path +# $1 path, pwd or where_is +guess_repo() { + basename "$(dirname "${1}")" +} + if [ ! -f PKGBUILD ]; then - error "Missing PKGBUILD" + error "Missing PKGBUILD ($PWD)" exit 1 fi @@ -148,7 +157,8 @@ if [ ${DEPTH} -eq 0 ]; then # Run build command pushd "${BUILDDIR}/${_pkg}" >/dev/null ${FULLBUILDCMD} - ${HOOKLOCALRELEASE} +# Run local release hook with $1 = $repo + ${HOOKLOCALRELEASE} $(egrep ";${_pkg};" "${BUILDORDER}" | cut -d';' -f6) popd >/dev/null done -- cgit v1.2.3-54-g00ecf From ebc66c503aad4df99f8660f2e4557c23cdc24fd3 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Mon, 6 Feb 2012 03:00:34 -0300 Subject: _pkg contained the depth also --- mips64el/mipsrelease | 5 +++++ treepkg | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mips64el/mipsrelease b/mips64el/mipsrelease index 0d14c72..5d0034d 100755 --- a/mips64el/mipsrelease +++ b/mips64el/mipsrelease @@ -34,6 +34,11 @@ get_full_version() { repo=$1; shift +if [ -z "${repo}" ]; then + error "Empty repo" + exit 1 +fi + # Get all needed sources source PKGBUILD fullver=$(get_full_version ${epoch:-0} ${pkgver} ${pkgrel}) diff --git a/treepkg b/treepkg index 167ccbd..bd95253 100755 --- a/treepkg +++ b/treepkg @@ -158,7 +158,7 @@ if [ ${DEPTH} -eq 0 ]; then pushd "${BUILDDIR}/${_pkg}" >/dev/null ${FULLBUILDCMD} # Run local release hook with $1 = $repo - ${HOOKLOCALRELEASE} $(egrep ";${_pkg};" "${BUILDORDER}" | cut -d';' -f6) + ${HOOKLOCALRELEASE} $(egrep ";${_pkg#*_};" "${BUILDORDER}" | cut -d';' -f6) popd >/dev/null done -- cgit v1.2.3-54-g00ecf From 6fbba8f74c28d58731c9611839e31955708d4b65 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Wed, 8 Feb 2012 14:09:53 -0300 Subject: Perform cleanups + "bury" packages if needed --- treepkg | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/treepkg b/treepkg index bd95253..b3e4fe0 100755 --- a/treepkg +++ b/treepkg @@ -7,9 +7,9 @@ source $XDG_CONFIG_HOME/libretools/libretools.conf >/dev/null 2>&1|| true source /etc/makepkg.conf source $HOME/makepkg.conf >/dev/null 2>&1|| true -# End inmediately but print a useful message +# End inmediately but print an useful message trap_exit() { - error "($(basename $0)) $@" + error "($(basename $0)) $@ (leftovers on ${BUILDDIR})" exit 1 } @@ -32,6 +32,8 @@ get_fullver() { } # Add line to build order cache in CSV format +# *must* be run from the PKGBUILD path +# status;depth;pkgbase;[epoch:]pkgver-pkgrel;path;repo # $1 status # $2 pkgname add_order() { @@ -39,6 +41,30 @@ add_order() { ${VERBOSE} && msg2 "%${DEPTH}s${2:-${pkgbase}} [${1}]" || true } +# Bury a package deeper in the tree +# $1 pkgbase +# $2 nextdepth +bury() { +# Bury only if we are going to build the dep +# Get it's current depth and dir name + local current_depth=$(egrep "build;[0-9]\+;${1};" "${BUILDORDER}" | cut -d ';' -f 2) + local current_name="$(printf "%03d" ${current_depth})_${1}" + +# If there's a depth or the package is not the root of the build tree (which +# can lead to funny chicken-and-egg problems), update the depth to the current +# package next-depth and rename the dir too + if [ -z "${current_depth}" ]; then return; fi + if [ -z "${current_name}" ]; then return; fi + if [ ${current_depth} -eq 0 ]; then return; fi + + ${VERBOSE} && msg "Burying ${1} from ${current_depth} to ${2}" + + { + sed -i "s|^\(build;\)\([0-9]\+\)\(;${1};.*\)$|\1${2}\3|" "${BUILDORDER}" && \ + mv "${BUILDDIR}/${current_name}" "${BUILDDIR}/$(printf "%03d" ${2})_${1}" + } || return 1 +} + # Finds a PKGBUILD on toru's path cache # Look in all caches but pick the first one # TODO move to a toru flag (-p?) @@ -87,6 +113,7 @@ NEXTDEPTH=$((${DEPTH} + 1)) # TODO Turn into flags? VERBOSE=${V:-true} BUILD=${B:-true} +CLEANUP=${C:-true} # ensure it exists touch "${BUILDORDER}" @@ -120,7 +147,9 @@ for _dep in ${deps[@]}; do # pkgbase - dep1 # \ dep2 - dep1 # dep1 should be depth + 1 - egrep -q ";${_dep};" "${BUILDORDER}" && continue +# probably sed -i buildorder and increase dep's dep depth :D +# remember to change dir name from xxx_pkgbase to xxx+1_pkgbase + egrep -q ";${_dep};" "${BUILDORDER}" && bury "${_dep}" ${NEXTDEPTH} # Ask toru where's a PKGBUILD depdir="$(where_is ${_dep})" @@ -167,6 +196,11 @@ if [ ${DEPTH} -eq 0 ]; then ${VERBOSE} || echo "${BUILDORDER}" || true fi +if (( CLEANUP )); then + msg2 "Removing ${BUILDDIR}" + rm -rf "${BUILDDIR}" +fi + fi exit $? -- cgit v1.2.3-54-g00ecf From 9e8d1be6f00e61f15ab58459dcfb7b6e98609e2f Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Mon, 13 Feb 2012 12:08:49 -0300 Subject: Add pkgbase to the path list --- toru-path | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toru-path b/toru-path index 7500aed..25d3995 100755 --- a/toru-path +++ b/toru-path @@ -19,7 +19,7 @@ for _pkgbuild in ${pkgbuilds[@]}; do fullpath=$(dirname $(readlink -f ${_pkgbuild})) - for _pkg in ${pkgname[@]} ${provides[@]}; do + for _pkg in ${pkgbase} ${pkgname[@]} ${provides[@]}; do paths+=(${_pkg/[<>=]*}:${fullpath}) done -- cgit v1.2.3-54-g00ecf From 875ec16b3d25c2b4d9dc5c940702f5ecde76cf7f Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Tue, 14 Feb 2012 11:41:40 -0300 Subject: Update the system before building --- mips64el/mipsrelease | 2 +- treepkg | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mips64el/mipsrelease b/mips64el/mipsrelease index 5d0034d..f245c47 100755 --- a/mips64el/mipsrelease +++ b/mips64el/mipsrelease @@ -53,7 +53,7 @@ done repo-add ${PKGDEST}/stage3.db.tar.gz ${pkgs[@]} -sudo pacman -Sy +#sudo pacman -Sy librestage ${repo} diff --git a/treepkg b/treepkg index b3e4fe0..a49301e 100755 --- a/treepkg +++ b/treepkg @@ -185,6 +185,8 @@ if [ ${DEPTH} -eq 0 ]; then # Run build command pushd "${BUILDDIR}/${_pkg}" >/dev/null + sudo pacman -Syu + ${FULLBUILDCMD} # Run local release hook with $1 = $repo ${HOOKLOCALRELEASE} $(egrep ";${_pkg#*_};" "${BUILDORDER}" | cut -d';' -f6) -- cgit v1.2.3-54-g00ecf From c4a93a978c2c97e14612b185b22de8dfc42b14ae Mon Sep 17 00:00:00 2001 From: Michał Masłowski Date: Wed, 15 Feb 2012 11:48:26 +0100 Subject: Fix passing arguments of libremakepkg to makepkg. --- libremakepkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libremakepkg b/libremakepkg index 44c0d29..7db0e3c 100755 --- a/libremakepkg +++ b/libremakepkg @@ -118,7 +118,7 @@ fi unset CLEANFIRST UPDATEFIRST LIBRECHROOT_ARGS -makechrootpkg -d -r "$CHROOTDIR" -l "$CHROOT" $MAKEPKG_ARGS +makechrootpkg -d -r "$CHROOTDIR" -l "$CHROOT" -- $MAKEPKG_ARGS ev="$?" # exit value copy_log -- cgit v1.2.3-54-g00ecf From 2f7e3155e766e552f53dbadab18f4e2aff824cd4 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Fri, 24 Feb 2012 18:50:44 -0300 Subject: Informe removed packages --- chcleanup | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/chcleanup b/chcleanup index a893320..d1c71c7 100755 --- a/chcleanup +++ b/chcleanup @@ -2,6 +2,10 @@ [ ! -f ~/cleansystem ] && exit 1 -sudo pacman --noconfirm -Rcs $(comm -23 <(pacman -Qq | sort) <(sort /etc/libretools.d/cleansystem)) +packages=($(comm -23 <(pacman -Qq | sort) <(sort /etc/libretools.d/cleansystem))) + +echo "Removing: ${packages[@]}" + +sudo pacman --noconfirm -Rcs ${packages[@]} exit $? -- cgit v1.2.3-54-g00ecf From ee706c5c36fd8993905a14aa2afeb5951dac6ec9 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Fri, 24 Feb 2012 18:51:14 -0300 Subject: Start from a build tree using `N=true treepkg /path/to/build/dir` Needs flags :) --- treepkg | 51 ++++++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/treepkg b/treepkg index a49301e..e742858 100755 --- a/treepkg +++ b/treepkg @@ -114,34 +114,37 @@ NEXTDEPTH=$((${DEPTH} + 1)) VERBOSE=${V:-true} BUILD=${B:-true} CLEANUP=${C:-true} +# Skip BUILDORDER creation and build anything on BUILDDIR +BUILDNOW=${N:-false} +if ! ${BUILDNOW}; then # ensure it exists -touch "${BUILDORDER}" + touch "${BUILDORDER}" # If this package is already built quit silently -if is_built "${pkgbase}" "${fullver}"; then - add_order "ignore" - exit 0 -fi + if is_built "${pkgbase}" "${fullver}"; then + add_order "ignore" + exit 0 + fi # Ignore if already in build order -egrep -q ";${pkgbase};" "${BUILDORDER}" && exit 0 + egrep -q ";${pkgbase};" "${BUILDORDER}" && exit 0 # Add pkgbase to build order -add_order "build" + add_order "build" # Copy the directory to the build dir # TODO run makepkg --source to avoid moving garbage around? -cp -r "${PWD}" "${BUILDDIR}/$(printf "%03d" ${DEPTH})_${pkgbase}" + cp -r "${PWD}" "${BUILDDIR}/$(printf "%03d" ${DEPTH})_${pkgbase}" # Cleanup dep versioning -deps=($(echo "${depends[@]} ${makedepends[@]}" | \ - sed "s/[=<>]\+[^ ]\+//g" | \ - tr ' ' "\n" | \ - sort -u)) + deps=($(echo "${depends[@]} ${makedepends[@]}" | \ + sed "s/[=<>]\+[^ ]\+//g" | \ + tr ' ' "\n" | \ + sort -u)) # NOTE: getting depends from package() is a PITA -for _dep in ${deps[@]}; do + for _dep in ${deps[@]}; do # Ignore if already in build order # TODO move deps deeper in the tree if # pkgbase - dep1 @@ -149,24 +152,26 @@ for _dep in ${deps[@]}; do # dep1 should be depth + 1 # probably sed -i buildorder and increase dep's dep depth :D # remember to change dir name from xxx_pkgbase to xxx+1_pkgbase - egrep -q ";${_dep};" "${BUILDORDER}" && bury "${_dep}" ${NEXTDEPTH} + egrep -q ";${_dep};" "${BUILDORDER}" && bury "${_dep}" ${NEXTDEPTH} # Ask toru where's a PKGBUILD - depdir="$(where_is ${_dep})" + depdir="$(where_is ${_dep})" - if [ -z "${depdir}" -o ! -d "${depdir}" ]; then + if [ -z "${depdir}" -o ! -d "${depdir}" ]; then # We specify the pkgname because we can't source the dep PKGBUILD # Normally 'any' packages are missing from our work ABS - add_order "missing" "${_dep}" - continue - fi + add_order "missing" "${_dep}" + continue + fi - pushd "${depdir}" >/dev/null + pushd "${depdir}" >/dev/null # Run itself over dependencies - $0 "${BUILDDIR}" ${NEXTDEPTH} + $0 "${BUILDDIR}" ${NEXTDEPTH} -done + done +# End BUILD now +fi # Only build at the end if [ ${DEPTH} -eq 0 ]; then @@ -198,7 +203,7 @@ if [ ${DEPTH} -eq 0 ]; then ${VERBOSE} || echo "${BUILDORDER}" || true fi -if (( CLEANUP )); then +if ${CLEANUP} ; then msg2 "Removing ${BUILDDIR}" rm -rf "${BUILDDIR}" fi -- cgit v1.2.3-54-g00ecf From b3047be050482448ef6487aefe8193bf911807d6 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Mon, 27 Feb 2012 11:30:10 -0300 Subject: Unset pkgbase too --- toru-path | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toru-path b/toru-path index b82dbb2..22b991b 100755 --- a/toru-path +++ b/toru-path @@ -23,7 +23,7 @@ for _pkgbuild in ${pkgbuilds[@]}; do paths+=(${_pkg/[<>=]*}:${fullpath}) done - unset pkgname provides + unset pkgbase pkgname provides done # TODO remove old paths -- cgit v1.2.3-54-g00ecf From da785c8272457a70f21942bf4943bfc69f4fbeb8 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Mon, 27 Feb 2012 11:35:55 -0300 Subject: Use only one find --- toru-path | 2 ++ toru-utils | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/toru-path b/toru-path index 22b991b..71dc85c 100755 --- a/toru-path +++ b/toru-path @@ -27,6 +27,8 @@ for _pkgbuild in ${pkgbuilds[@]}; do done # TODO remove old paths +# by joining new paths to old paths and exclude the joining points from the +# cache (sort of a sql join) echo ${paths[@]} | tr ' ' "\n" | sort >> ${TORUPATH}/paths lastsync ${LASTSYNCFILE} diff --git a/toru-utils b/toru-utils index bb0aef4..2488091 100755 --- a/toru-utils +++ b/toru-utils @@ -59,17 +59,14 @@ get_pkgbuilds() { $QUIET || warning "Forcing upgrade" # Get all PKGBUILDs - pkgbuilds=($(find $@ -mindepth 2 -maxdepth 3 -type f -name 'PKGBUILD')) - + extra="" else - # Only find newer than lastsyncfile and read everything else from cache - pkgbuilds=($(find $@ -mindepth 2 -maxdepth 3 -type f -name 'PKGBUILD' -newer ${LASTSYNCFILE})) - + extra=" -newer ${LASTSYNCFILE}" fi # Return all PKGBUILDs found - echo ${pkgbuilds[@]} + find $@ -mindepth 2 -maxdepth 3 -type f -name 'PKGBUILD' ${extra} } # End inmediately but print a useful message -- cgit v1.2.3-54-g00ecf From b53d6c027ad0a90c676419d2246fa76a7682221a Mon Sep 17 00:00:00 2001 From: Nicolás Reynolds Date: Wed, 4 Apr 2012 20:42:38 -0300 Subject: Pass the build dir as first argument to retake a build --- treepkg | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/treepkg b/treepkg index e742858..7d210cf 100755 --- a/treepkg +++ b/treepkg @@ -117,6 +117,10 @@ CLEANUP=${C:-true} # Skip BUILDORDER creation and build anything on BUILDDIR BUILDNOW=${N:-false} +if [ ! -z "${1}" -a ${DEPTH} -eq 0 ]; then + BUILDNOW=true +fi + if ! ${BUILDNOW}; then # ensure it exists touch "${BUILDORDER}" @@ -190,7 +194,7 @@ if [ ${DEPTH} -eq 0 ]; then # Run build command pushd "${BUILDDIR}/${_pkg}" >/dev/null - sudo pacman -Syu + sudo pacman -Syu --noconfirm ${FULLBUILDCMD} # Run local release hook with $1 = $repo -- cgit v1.2.3-54-g00ecf From a0890dedb0ed2ba5c959ddd8cddf52fe4695baa9 Mon Sep 17 00:00:00 2001 From: Nicolás Reynolds Date: Sun, 8 Apr 2012 10:42:17 -0300 Subject: Use system clean file --- chcleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chcleanup b/chcleanup index d1c71c7..7074b84 100755 --- a/chcleanup +++ b/chcleanup @@ -1,6 +1,6 @@ #!/bin/bash -[ ! -f ~/cleansystem ] && exit 1 +[ ! -f /etc/libretools.d/cleansystem ] && exit 1 packages=($(comm -23 <(pacman -Qq | sort) <(sort /etc/libretools.d/cleansystem))) -- cgit v1.2.3-54-g00ecf From 7db00e8e126f890cca5f742238ef438b7d418d28 Mon Sep 17 00:00:00 2001 From: Nicolás Reynolds Date: Sun, 8 Apr 2012 12:01:26 -0300 Subject: treepkg documentation --- doc/treepkg.markdown | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 doc/treepkg.markdown diff --git a/doc/treepkg.markdown b/doc/treepkg.markdown new file mode 100644 index 0000000..2808599 --- /dev/null +++ b/doc/treepkg.markdown @@ -0,0 +1,126 @@ +# TreePKG + +`treepkg` is a tool for recursively building packages from an ABS tree. It has +been tested while building packages for the mips64el port and it has proven to +be useful. + +It was written having in mind the experience of `fullpkg`, which converted to +`fullpkg-ng` and then splitted into `fullpkg-build` and `fullpkg-find`. It's +aim is to simplify algorithms implemented on the fullpkg family, while solving +some design issues that made fullpkg miss some packages sometimes. + + +## Requirements + +`treepkg` needs the help of `toru-path` for "indexing" an ABS tree. `toru-path` +stores a plain text database of "pkgname:path" pairs, where pkgname is replaced +by the "pkgbase", "pkgname", and "provides" fields of a PKGBUILD, followed by +the path of the current PKGBUILD. + +This information is then used by `treepkg` to know where to find the PKGBUILD +of a package. The fullpkg family needed to guess this by traversing the full +ABS tree, and therefore it was unable to find pkgnames that differ from +pkgbase. + +So, to use `treepkg` you need to run `toru-path` after the ABS tree update. + +> Currently `toru-path` doesn't remove duplicated or updated pairs, but it +> picks the last ones and only processes new PKGBUILDs. This means `toru-path` +> works correctly but it's database will grow up slowly. + +## How does it work + +`treepkg` must be run from the path of the PKGBUILD you're going to build (this +may change over time). This will be DEPTH=0 and it will create a temporary +directory to work on in the format /tmp/pkgbase-treepkg-random-string. Inside +this directory, it will copy the needed PKGBUILDs prefixed with their depth +number. The first package will always be copied to 000\_pkgbase. + +From then on, treepkg sources the PKGBUILD and runs itself over all pkgnames on +the depends and makedepends array, only if it detects their versions aren't +already built or deprecated by newer ones, using the `is_built` utility. + +While processing this info, it will increase the depth of the packages. It'll +also write a CSV file with the knowledge it acquires from the ABS tree (useful +for debugging). This file is called BUILDORDER. + +When this process ends (no more needed dependencies are found), the temporary +work dir is traversed in inverse order (from higher depths to 0) running the +FULLBUILDCMD from libretools.conf and then the HOOKLOCALRELEASE variable, which +*must* provide a way to `repo-add` the packages to an available repository, so +the next build will find and install the newer version using pacman. + +For instance, having this as the first pacman repository (on /etc/pacman.conf): + + [stage3] + Server = /var/cache/pacman/pkg + +And this on /etc/makepkg.conf: + + PKGDEST=/var/cache/pacman/pkg + +Your HOOKLOCALRELEASE script should look like this: + + # Get needed vars + source /etc/makepkg.conf + source /etc/libretools.conf + source PKGBUILD + + unset build package check + + fullver=$(full_version ${epoch:-0} ${pkgver} ${pkgrel}) + pkgs=() + + # Generate canonical package paths + msg "Adding packages to [stage3]..." + for name in ${pkgname[@]}; do + msg2 "${name} ${fullver}" + pkgs+=("${PKGDEST}/${name}-${fullver}-*.pkg.tar.*") + done + + # Add the packages to a local + repo-add ${PKGDEST}/stage3.db.tar.gz ${pkgs[@]} + + # Stage the packages for later releasing + librestage $1 + +> Note the first HOOKLOCALRELEASE argument is the remote repository name (core, +> extra, etc.) + +There's a special case when a dependency depends on another that was put on +a depth level lower than itself. In this case the build order will be wrongly +assumed and you may end up with broken packages. + +To explain it with an example: + + ghostscript (0) - fontconfig (1) + \ cups (1) - fontconfig (ignored) + +The second time fontconfig appears, it will be ignored. In this case cups will +build against an fontconfig version that will be outdated by the fontconfig +version at depth 1. In this cases, `treepkg` will detect it cached the +dependency on a lower depth, and will "bury" it to a depth higher than the +current one. Thus this will become the build path: + + ghostscript (0) - fontconfig (buried) + \ cups (1) - fontconfig (2) + + +## Tips + +`treepkg` accepts two arguments: 1) the temporary work dir and 2) the next +depth level it should use (if current equals 0, it'll pass 1). You don't need +to pass this arguments when running it manually, they're used internally to +automatically construct the build path. + +But if a build failed, `treepkg` will cancel itself immediately informing you +where the leftovers files where left. If you pass this path to `treepkg` as the +first argument, it will resume the build, skipping to the last package being +packaged. + +You can take the opportunity given by this to modify the build path or the +PKGBUILDs, without having to re-run `treepkg` twice. For instance you can +remove a package from the build order, or move it manually, or update the +PKGBUILD that made `treepkg` fail in the first place. + +You don't probably want to mess with the second argument though. -- cgit v1.2.3-54-g00ecf From f03b4712e3db0a7bc28b63483a0314d509783779 Mon Sep 17 00:00:00 2001 From: Nicolás Reynolds Date: Sun, 8 Apr 2012 12:23:26 -0300 Subject: Added license --- treepkg | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/treepkg b/treepkg index 7d210cf..67740ae 100755 --- a/treepkg +++ b/treepkg @@ -1,5 +1,20 @@ #!/bin/bash #set -x +# (c) 2012 Nicolás Reynolds +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + source /etc/libretools.conf source $XDG_CONFIG_HOME/libretools/libretools.conf >/dev/null 2>&1|| true -- cgit v1.2.3-54-g00ecf From 1873a2f71e77ffbd46258474851600bdb1ecc297 Mon Sep 17 00:00:00 2001 From: Nicolás Reynolds Date: Sun, 8 Apr 2012 16:55:08 -0300 Subject: Minor improvements --- toru-path | 2 +- toru-utils | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/toru-path b/toru-path index 71dc85c..5065a30 100755 --- a/toru-path +++ b/toru-path @@ -17,7 +17,7 @@ for _pkgbuild in ${pkgbuilds[@]}; do continue } - fullpath=$(dirname $(readlink -f ${_pkgbuild})) + fullpath=$(dirname ${_pkgbuild}) for _pkg in ${pkgbase} ${pkgname[@]} ${provides[@]}; do paths+=(${_pkg/[<>=]*}:${fullpath}) diff --git a/toru-utils b/toru-utils index 2488091..9a7918e 100755 --- a/toru-utils +++ b/toru-utils @@ -78,6 +78,6 @@ trap_exit() { # Trap signals from makepkg set -E -trap 'trap_exit "(prfullpkg:${level}) TERM signal caught. Exiting..."' TERM HUP QUIT -trap 'trap_exit "(prfullpkg:${level}) Aborted by user! Exiting..."' INT -trap 'trap_exit "(prfullpkg:${level}) An unknown error has occurred. Exiting..."' ERR +trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT +trap 'trap_exit "Aborted by user! Exiting..."' INT +trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR -- cgit v1.2.3-54-g00ecf From e65a1c3238933e4f2909d34f11dc4fce3509a6fb Mon Sep 17 00:00:00 2001 From: Nicolás Reynolds Date: Mon, 9 Apr 2012 13:28:32 -0300 Subject: Copy makepkg logs only --- libremakepkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libremakepkg b/libremakepkg index 44c0d29..8678e9a 100755 --- a/libremakepkg +++ b/libremakepkg @@ -24,7 +24,7 @@ function copy_log { # copy logs if they exist - find "${CHROOTDIR}/${CHROOT}/build/" -name "*\.log" -exec cp {} ./ \; + find "${CHROOTDIR}/${CHROOT}/build/" -maxdepth 1 -name "*\.log" -exec cp {} ./ \; } -- cgit v1.2.3-54-g00ecf From f6ea811fbc6f449c225a0ddd0de5dbce07696c0b Mon Sep 17 00:00:00 2001 From: Nicolás Reynolds Date: Tue, 10 Apr 2012 15:42:45 -0300 Subject: Run db-update on the server after uploading packages --- librerelease | 3 +++ 1 file changed, 3 insertions(+) diff --git a/librerelease b/librerelease index bac5b88..c113572 100755 --- a/librerelease +++ b/librerelease @@ -147,4 +147,7 @@ rsync --recursive \ msg "Removing packages from local [staging]" clean +msg "Running db-update on repos" +ssh ${PARABOLAHOST} dbscripts/db-update + exit 0 -- cgit v1.2.3-54-g00ecf From 29307a3f26a1ec40bf008209ac0fdcc252596cfa Mon Sep 17 00:00:00 2001 From: Michał Masłowski Date: Sun, 29 Apr 2012 14:34:02 +0200 Subject: librebasebuilder: Avoid mkarchroot's configuration replacements, fixes #94. --- librebasebuilder | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) mode change 100644 => 100755 librebasebuilder diff --git a/librebasebuilder b/librebasebuilder old mode 100644 new mode 100755 index 3eaadd8..8ba1315 --- a/librebasebuilder +++ b/librebasebuilder @@ -45,7 +45,7 @@ done tempdir=/home/chroot/base outdir=$(pwd) -mkarchroot $tempdir mkinitcpio base sudo parted nano zile vi ed openssh +mkarchroot -n $tempdir mkinitcpio base sudo parted nano zile vi ed openssh cd $tempdir @@ -61,6 +61,24 @@ cat > etc/resolv.conf < etc/pacman.d/mirrorlist < Date: Thu, 3 May 2012 20:48:50 +0200 Subject: Fix base mirrorlist. --- librebasebuilder | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/librebasebuilder b/librebasebuilder index 8ba1315..f7d3a25 100755 --- a/librebasebuilder +++ b/librebasebuilder @@ -61,21 +61,21 @@ cat > etc/resolv.conf < etc/pacman.d/mirrorlist < Date: Sat, 5 May 2012 04:30:00 -0300 Subject: Fixed bury function. Now it actually buries packages and skips if the current level is deeper than the requested --- treepkg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/treepkg b/treepkg index 67740ae..d21377d 100755 --- a/treepkg +++ b/treepkg @@ -62,7 +62,7 @@ add_order() { bury() { # Bury only if we are going to build the dep # Get it's current depth and dir name - local current_depth=$(egrep "build;[0-9]\+;${1};" "${BUILDORDER}" | cut -d ';' -f 2) + local current_depth=$(grep "build;[0-9]\+;${1};" "${BUILDORDER}" | cut -d ';' -f 2) local current_name="$(printf "%03d" ${current_depth})_${1}" # If there's a depth or the package is not the root of the build tree (which @@ -71,6 +71,7 @@ bury() { if [ -z "${current_depth}" ]; then return; fi if [ -z "${current_name}" ]; then return; fi if [ ${current_depth} -eq 0 ]; then return; fi + if [ ${current_depth} -ge $2 ]; then return; fi ${VERBOSE} && msg "Burying ${1} from ${current_depth} to ${2}" -- cgit v1.2.3-54-g00ecf