summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJoshua I. Haase H. (xihh) <hahj87@gmail.com>2012-05-08 01:58:02 -0500
committerJoshua I. Haase H. (xihh) <hahj87@gmail.com>2012-05-08 01:58:02 -0500
commit3dcf773de928a0a587f0409d2bd1d7ecad80b549 (patch)
treebf66d492479bb9f454b9700b58fc53c114ae4e7d
parente13b2d4ae996c164b8082bfa18a7351ddb4467d7 (diff)
parent112b6427bf299b9db8fc34e5b771fecf1958db07 (diff)
Merge branch 'master' of gitpar:libretools
Conflicts: toru-utils
-rwxr-xr-xchcleanup8
-rw-r--r--doc/treepkg.markdown126
-rwxr-xr-x[-rw-r--r--]librebasebuilder20
-rwxr-xr-xlibremakepkg2
-rwxr-xr-xlibrerelease3
-rwxr-xr-xmips64el/mipsrelease8
-rwxr-xr-xtoru-path8
-rwxr-xr-xtoru-utils15
-rwxr-xr-xtreepkg233
9 files changed, 407 insertions, 16 deletions
diff --git a/chcleanup b/chcleanup
index 83c9f3f..7074b84 100755
--- a/chcleanup
+++ b/chcleanup
@@ -1,7 +1,11 @@
#!/bin/bash
-[ ! -f ~/cleansystem ] && exit 1
+[ ! -f /etc/libretools.d/cleansystem ] && exit 1
-sudo pacman --noconfirm -Rcs $(comm -23 <(pacman -Qq | sort) <(sort ~/cleansystem))
+packages=($(comm -23 <(pacman -Qq | sort) <(sort /etc/libretools.d/cleansystem)))
+
+echo "Removing: ${packages[@]}"
+
+sudo pacman --noconfirm -Rcs ${packages[@]}
exit $?
diff --git a/doc/treepkg.markdown b/doc/treepkg.markdown
new file mode 100644
index 0000000..2808599
--- /dev/null
+++ b/doc/treepkg.markdown
@@ -0,0 +1,126 @@
+# TreePKG
+
+`treepkg` is a tool for recursively building packages from an ABS tree. It has
+been tested while building packages for the mips64el port and it has proven to
+be useful.
+
+It was written having in mind the experience of `fullpkg`, which converted to
+`fullpkg-ng` and then splitted into `fullpkg-build` and `fullpkg-find`. It's
+aim is to simplify algorithms implemented on the fullpkg family, while solving
+some design issues that made fullpkg miss some packages sometimes.
+
+
+## Requirements
+
+`treepkg` needs the help of `toru-path` for "indexing" an ABS tree. `toru-path`
+stores a plain text database of "pkgname:path" pairs, where pkgname is replaced
+by the "pkgbase", "pkgname", and "provides" fields of a PKGBUILD, followed by
+the path of the current PKGBUILD.
+
+This information is then used by `treepkg` to know where to find the PKGBUILD
+of a package. The fullpkg family needed to guess this by traversing the full
+ABS tree, and therefore it was unable to find pkgnames that differ from
+pkgbase.
+
+So, to use `treepkg` you need to run `toru-path` after the ABS tree update.
+
+> Currently `toru-path` doesn't remove duplicated or updated pairs, but it
+> picks the last ones and only processes new PKGBUILDs. This means `toru-path`
+> works correctly but it's database will grow up slowly.
+
+## How does it work
+
+`treepkg` must be run from the path of the PKGBUILD you're going to build (this
+may change over time). This will be DEPTH=0 and it will create a temporary
+directory to work on in the format /tmp/pkgbase-treepkg-random-string. Inside
+this directory, it will copy the needed PKGBUILDs prefixed with their depth
+number. The first package will always be copied to 000\_pkgbase.
+
+From then on, treepkg sources the PKGBUILD and runs itself over all pkgnames on
+the depends and makedepends array, only if it detects their versions aren't
+already built or deprecated by newer ones, using the `is_built` utility.
+
+While processing this info, it will increase the depth of the packages. It'll
+also write a CSV file with the knowledge it acquires from the ABS tree (useful
+for debugging). This file is called BUILDORDER.
+
+When this process ends (no more needed dependencies are found), the temporary
+work dir is traversed in inverse order (from higher depths to 0) running the
+FULLBUILDCMD from libretools.conf and then the HOOKLOCALRELEASE variable, which
+*must* provide a way to `repo-add` the packages to an available repository, so
+the next build will find and install the newer version using pacman.
+
+For instance, having this as the first pacman repository (on /etc/pacman.conf):
+
+ [stage3]
+ Server = /var/cache/pacman/pkg
+
+And this on /etc/makepkg.conf:
+
+ PKGDEST=/var/cache/pacman/pkg
+
+Your HOOKLOCALRELEASE script should look like this:
+
+ # Get needed vars
+ source /etc/makepkg.conf
+ source /etc/libretools.conf
+ source PKGBUILD
+
+ unset build package check
+
+ fullver=$(full_version ${epoch:-0} ${pkgver} ${pkgrel})
+ pkgs=()
+
+ # Generate canonical package paths
+ msg "Adding packages to [stage3]..."
+ for name in ${pkgname[@]}; do
+ msg2 "${name} ${fullver}"
+ pkgs+=("${PKGDEST}/${name}-${fullver}-*.pkg.tar.*")
+ done
+
+ # Add the packages to a local
+ repo-add ${PKGDEST}/stage3.db.tar.gz ${pkgs[@]}
+
+ # Stage the packages for later releasing
+ librestage $1
+
+> Note the first HOOKLOCALRELEASE argument is the remote repository name (core,
+> extra, etc.)
+
+There's a special case when a dependency depends on another that was put on
+a depth level lower than itself. In this case the build order will be wrongly
+assumed and you may end up with broken packages.
+
+To explain it with an example:
+
+ ghostscript (0) - fontconfig (1)
+ \ cups (1) - fontconfig (ignored)
+
+The second time fontconfig appears, it will be ignored. In this case cups will
+build against an fontconfig version that will be outdated by the fontconfig
+version at depth 1. In this cases, `treepkg` will detect it cached the
+dependency on a lower depth, and will "bury" it to a depth higher than the
+current one. Thus this will become the build path:
+
+ ghostscript (0) - fontconfig (buried)
+ \ cups (1) - fontconfig (2)
+
+
+## Tips
+
+`treepkg` accepts two arguments: 1) the temporary work dir and 2) the next
+depth level it should use (if current equals 0, it'll pass 1). You don't need
+to pass this arguments when running it manually, they're used internally to
+automatically construct the build path.
+
+But if a build failed, `treepkg` will cancel itself immediately informing you
+where the leftovers files where left. If you pass this path to `treepkg` as the
+first argument, it will resume the build, skipping to the last package being
+packaged.
+
+You can take the opportunity given by this to modify the build path or the
+PKGBUILDs, without having to re-run `treepkg` twice. For instance you can
+remove a package from the build order, or move it manually, or update the
+PKGBUILD that made `treepkg` fail in the first place.
+
+You don't probably want to mess with the second argument though.
diff --git a/librebasebuilder b/librebasebuilder
index 3eaadd8..f7d3a25 100644..100755
--- a/librebasebuilder
+++ b/librebasebuilder
@@ -45,7 +45,7 @@ done
tempdir=/home/chroot/base
outdir=$(pwd)
-mkarchroot $tempdir mkinitcpio base sudo parted nano zile vi ed openssh
+mkarchroot -n $tempdir mkinitcpio base sudo parted nano zile vi ed openssh
cd $tempdir
@@ -61,6 +61,24 @@ cat > etc/resolv.conf <<EOF
# End of file
EOF
+# From pacman-mirrorlist-libre-20120307-1. Remember to escape dollars.
+cat > etc/pacman.d/mirrorlist <<EOF
+# Parabola GNU/Linux - Last Updated: Wed Mar 7 17:33:36 GMT 2012
+
+# Atlanta, GA, USA
+# Responsible: belos
+Server = http://parabola.techno-geeks.org/\$repo/os/\$arch
+
+# Nuremberg, Germany
+# Responsible: johkra
+Server = http://parabolaweb.eu/\$repo/os/\$arch
+
+# UK
+# Responsible: Parabola Project
+Server = http://repo.parabolagnulinux.org/\$repo/os/\$arch
+
+EOF
+
rm .arch-chroot
tar cjf $outdir/parabola-mips64el-$(LC_ALL=C date -u +%Y%m%d).tar.bz2 .
diff --git a/libremakepkg b/libremakepkg
index 7db0e3c..f7924f6 100755
--- a/libremakepkg
+++ b/libremakepkg
@@ -24,7 +24,7 @@
function copy_log { # copy logs if they exist
- find "${CHROOTDIR}/${CHROOT}/build/" -name "*\.log" -exec cp {} ./ \;
+ find "${CHROOTDIR}/${CHROOT}/build/" -maxdepth 1 -name "*\.log" -exec cp {} ./ \;
}
diff --git a/librerelease b/librerelease
index bac5b88..c113572 100755
--- a/librerelease
+++ b/librerelease
@@ -147,4 +147,7 @@ rsync --recursive \
msg "Removing packages from local [staging]"
clean
+msg "Running db-update on repos"
+ssh ${PARABOLAHOST} dbscripts/db-update
+
exit 0
diff --git a/mips64el/mipsrelease b/mips64el/mipsrelease
index 1a4aade..f245c47 100755
--- a/mips64el/mipsrelease
+++ b/mips64el/mipsrelease
@@ -34,6 +34,11 @@ get_full_version() {
repo=$1; shift
+if [ -z "${repo}" ]; then
+ error "Empty repo"
+ exit 1
+fi
+
# Get all needed sources
source PKGBUILD
fullver=$(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})
@@ -48,6 +53,9 @@ done
repo-add ${PKGDEST}/stage3.db.tar.gz ${pkgs[@]}
+#sudo pacman -Sy
+librestage ${repo}
+
mkdir -p ${WORKDIR}/abs/${CARCH}/${repo} >/dev/null
diff --git a/toru-path b/toru-path
index 5081115..5065a30 100755
--- a/toru-path
+++ b/toru-path
@@ -17,16 +17,18 @@ for _pkgbuild in ${pkgbuilds[@]}; do
continue
}
- fullpath=$(dirname $(readlink -f ${_pkgbuild}))
+ fullpath=$(dirname ${_pkgbuild})
- for _pkg in ${pkgname[@]} ${provides[@]}; do
+ for _pkg in ${pkgbase} ${pkgname[@]} ${provides[@]}; do
paths+=(${_pkg/[<>=]*}:${fullpath})
done
- unset pkgname provides
+ unset pkgbase pkgname provides
done
# TODO remove old paths
+# by joining new paths to old paths and exclude the joining points from the
+# cache (sort of a sql join)
echo ${paths[@]} | tr ' ' "\n" | sort >> ${TORUPATH}/paths
lastsync ${LASTSYNCFILE}
diff --git a/toru-utils b/toru-utils
index 7e6d62d..0818aa6 100755
--- a/toru-utils
+++ b/toru-utils
@@ -59,17 +59,14 @@ get_pkgbuilds() {
$QUIET || warning "Forcing upgrade"
# Get all PKGBUILDs
- pkgbuilds=($(find $@ -mindepth 2 -maxdepth 3 -type f -name 'PKGBUILD'))
-
+ extra=""
else
-
# Only find newer than lastsyncfile and read everything else from cache
- pkgbuilds=($(find $@ -mindepth 2 -maxdepth 3 -type f -name 'PKGBUILD' -newer ${LASTSYNCFILE}))
-
+ extra=" -newer ${LASTSYNCFILE}"
fi
# Return all PKGBUILDs found
- echo ${pkgbuilds[@]}
+ find $@ -mindepth 2 -maxdepth 3 -type f -name 'PKGBUILD' ${extra}
}
# End inmediately but print a useful message
@@ -81,6 +78,6 @@ trap_exit() {
# Trap signals from makepkg
set -E
-trap 'trap_exit "($(basename $0):${level}) TERM signal caught. Exiting..."' TERM HUP QUIT
-trap 'trap_exit "($(basename $0):${level}) Aborted by user! Exiting..."' INT
-trap 'trap_exit "($(basename $0):${level}) An unknown error has occurred. Exiting..."' ERR
+trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT
+trap 'trap_exit "Aborted by user! Exiting..."' INT
+trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR
diff --git a/treepkg b/treepkg
new file mode 100755
index 0000000..d21377d
--- /dev/null
+++ b/treepkg
@@ -0,0 +1,233 @@
+#!/bin/bash
+#set -x
+# (c) 2012 Nicolás Reynolds <fauno@parabola.nu>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+source /etc/libretools.conf
+source $XDG_CONFIG_HOME/libretools/libretools.conf >/dev/null 2>&1|| true
+
+# Get system variables
+source /etc/makepkg.conf
+source $HOME/makepkg.conf >/dev/null 2>&1|| true
+
+# End inmediately but print an useful message
+trap_exit() {
+ error "($(basename $0)) $@ (leftovers on ${BUILDDIR})"
+ exit 1
+}
+
+# Trap signals from makepkg
+set -E
+trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT
+trap 'trap_exit "Aborted by user! Exiting..."' INT
+trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR
+
+# return : full version spec, including epoch (if necessary), pkgver, pkgrel
+# usage : get_fullver( ${epoch:-0}, $pkgver, $pkgrel )
+get_fullver() {
+ if [ $1 -eq 0 ]; then
+# zero epoch case, don't include it in version
+ echo $2-$3
+ else
+ echo $1:$2-$3
+ fi
+
+}
+
+# Add line to build order cache in CSV format
+# *must* be run from the PKGBUILD path
+# status;depth;pkgbase;[epoch:]pkgver-pkgrel;path;repo
+# $1 status
+# $2 pkgname
+add_order() {
+ echo "${1};${DEPTH};${2:-${pkgbase}};${fullver};${PWD};$(guess_repo "$PWD")" >> "${BUILDORDER}"
+ ${VERBOSE} && msg2 "%${DEPTH}s${2:-${pkgbase}} [${1}]" || true
+}
+
+# Bury a package deeper in the tree
+# $1 pkgbase
+# $2 nextdepth
+bury() {
+# Bury only if we are going to build the dep
+# Get it's current depth and dir name
+ local current_depth=$(grep "build;[0-9]\+;${1};" "${BUILDORDER}" | cut -d ';' -f 2)
+ local current_name="$(printf "%03d" ${current_depth})_${1}"
+
+# If there's a depth or the package is not the root of the build tree (which
+# can lead to funny chicken-and-egg problems), update the depth to the current
+# package next-depth and rename the dir too
+ if [ -z "${current_depth}" ]; then return; fi
+ if [ -z "${current_name}" ]; then return; fi
+ if [ ${current_depth} -eq 0 ]; then return; fi
+ if [ ${current_depth} -ge $2 ]; then return; fi
+
+ ${VERBOSE} && msg "Burying ${1} from ${current_depth} to ${2}"
+
+ {
+ sed -i "s|^\(build;\)\([0-9]\+\)\(;${1};.*\)$|\1${2}\3|" "${BUILDORDER}" && \
+ mv "${BUILDDIR}/${current_name}" "${BUILDDIR}/$(printf "%03d" ${2})_${1}"
+ } || return 1
+}
+
+# Finds a PKGBUILD on toru's path cache
+# Look in all caches but pick the first one
+# TODO move to a toru flag (-p?)
+where_is() {
+ grep -m1 "^${1}:" "${TORUPATH}/paths" 2>/dev/null| \
+ cut -d: -f2 2>/dev/null
+}
+
+# Guess the repo from the pkgbase path
+# $1 path, pwd or where_is
+guess_repo() {
+ basename "$(dirname "${1}")"
+}
+
+if [ ! -f PKGBUILD ]; then
+ error "Missing PKGBUILD ($PWD)"
+ exit 1
+fi
+
+if ! source PKGBUILD ; then
+ error "Can't source PKGBUILD"
+ exit 1
+fi
+
+# Save resources
+unset pkgdesc arch license groups backup install md5sums sha1sums \
+ sha256sums source options >/dev/null 2>&1
+
+unset build package >/dev/null 2>&1
+
+for _pkg in ${pkgname[@]}; do
+ unset package_${_pkg} >/dev/null 2>&1 || true
+done
+##
+
+# Get useful values
+pkgbase="${pkgbase:-${pkgname[0]}}"
+fullver=$(get_fullver ${epoch:-0} ${pkgver} ${pkgrel})
+
+# Get or set the work dir
+BUILDDIR="${1:-$(mktemp -d /tmp/${pkgbase}-treepkg-XXXx)}"
+BUILDORDER="${BUILDDIR}/BUILDORDER"
+DEPTH=${2:-0}
+NEXTDEPTH=$((${DEPTH} + 1))
+# This can be set as env vars (ie: $ V=false B=false treepkg)
+# TODO Turn into flags?
+VERBOSE=${V:-true}
+BUILD=${B:-true}
+CLEANUP=${C:-true}
+# Skip BUILDORDER creation and build anything on BUILDDIR
+BUILDNOW=${N:-false}
+
+if [ ! -z "${1}" -a ${DEPTH} -eq 0 ]; then
+ BUILDNOW=true
+fi
+
+if ! ${BUILDNOW}; then
+# ensure it exists
+ touch "${BUILDORDER}"
+
+# If this package is already built quit silently
+ if is_built "${pkgbase}" "${fullver}"; then
+ add_order "ignore"
+ exit 0
+ fi
+
+# Ignore if already in build order
+ egrep -q ";${pkgbase};" "${BUILDORDER}" && exit 0
+
+# Add pkgbase to build order
+ add_order "build"
+
+# Copy the directory to the build dir
+# TODO run makepkg --source to avoid moving garbage around?
+ cp -r "${PWD}" "${BUILDDIR}/$(printf "%03d" ${DEPTH})_${pkgbase}"
+
+# Cleanup dep versioning
+ deps=($(echo "${depends[@]} ${makedepends[@]}" | \
+ sed "s/[=<>]\+[^ ]\+//g" | \
+ tr ' ' "\n" | \
+ sort -u))
+
+# NOTE: getting depends from package() is a PITA
+ for _dep in ${deps[@]}; do
+# Ignore if already in build order
+# TODO move deps deeper in the tree if
+# pkgbase - dep1
+# \ dep2 - dep1
+# dep1 should be depth + 1
+# probably sed -i buildorder and increase dep's dep depth :D
+# remember to change dir name from xxx_pkgbase to xxx+1_pkgbase
+ egrep -q ";${_dep};" "${BUILDORDER}" && bury "${_dep}" ${NEXTDEPTH}
+
+# Ask toru where's a PKGBUILD
+ depdir="$(where_is ${_dep})"
+
+ if [ -z "${depdir}" -o ! -d "${depdir}" ]; then
+# We specify the pkgname because we can't source the dep PKGBUILD
+# Normally 'any' packages are missing from our work ABS
+ add_order "missing" "${_dep}"
+ continue
+ fi
+
+ pushd "${depdir}" >/dev/null
+
+# Run itself over dependencies
+ $0 "${BUILDDIR}" ${NEXTDEPTH}
+
+ done
+# End BUILD now
+fi
+
+# Only build at the end
+if [ ${DEPTH} -eq 0 ]; then
+ ${VERBOSE} && msg "Starting build" || true
+
+ if ${BUILD}; then
+ ${VERBOSE} && msg "Build tree stored in ${BUILDORDER}" || true
+
+# Build everything sorting the build dir
+# The reverse order ensures we start by the deepest packages
+ for _pkg in $(ls -r "${BUILDDIR}"); do
+# Ignore if there's no PKGBUILD
+ if [ ! -f "${BUILDDIR}/${_pkg}/PKGBUILD" ]; then continue; fi
+
+ ${VERBOSE} && msg "Building ${_pkg/_/ }" || true
+
+# Run build command
+ pushd "${BUILDDIR}/${_pkg}" >/dev/null
+ sudo pacman -Syu --noconfirm
+
+ ${FULLBUILDCMD}
+# Run local release hook with $1 = $repo
+ ${HOOKLOCALRELEASE} $(egrep ";${_pkg#*_};" "${BUILDORDER}" | cut -d';' -f6)
+ popd >/dev/null
+ done
+
+ else
+# Just print the working dir
+ ${VERBOSE} || echo "${BUILDORDER}" || true
+ fi
+
+if ${CLEANUP} ; then
+ msg2 "Removing ${BUILDDIR}"
+ rm -rf "${BUILDDIR}"
+fi
+
+fi
+
+exit $?