Merge branch 'master' of gitpar:libretools

Conflicts:
	toru-utils

9 files changed, 407 insertions, 16 deletions

diff --git a/chcleanup b/chcleanup
index 83c9f3f..7074b84 100755
--- a/chcleanup
+++ b/chcleanup
@@ -1,7 +1,11 @@
 #!/bin/bash
 
-[ ! -f ~/cleansystem ] && exit 1
+[ ! -f /etc/libretools.d/cleansystem ] && exit 1
 
-sudo pacman --noconfirm -Rcs $(comm -23 <(pacman -Qq | sort) <(sort ~/cleansystem))
+packages=($(comm -23 <(pacman -Qq | sort) <(sort /etc/libretools.d/cleansystem)))
+
+echo "Removing: ${packages[@]}"
+
+sudo pacman --noconfirm -Rcs ${packages[@]}
 
 exit $?
diff --git a/doc/treepkg.markdown b/doc/treepkg.markdown
new file mode 100644
index 0000000..2808599
--- /dev/null
+++ b/doc/treepkg.markdown
@@ -0,0 +1,126 @@
+# TreePKG
+
+`treepkg` is a tool for recursively building packages from an ABS tree. It has
+been tested while building packages for the mips64el port and it has proven to
+be useful.
+
+It was written having in mind the experience of `fullpkg`, which converted to
+`fullpkg-ng` and then splitted into `fullpkg-build` and `fullpkg-find`. It's
+aim is to simplify algorithms implemented on the fullpkg family, while solving
+some design issues that made fullpkg miss some packages sometimes.
+
+
+## Requirements
+
+`treepkg` needs the help of `toru-path` for "indexing" an ABS tree. `toru-path`
+stores a plain text database of "pkgname:path" pairs, where pkgname is replaced
+by the "pkgbase", "pkgname", and "provides" fields of a PKGBUILD, followed by
+the path of the current PKGBUILD.
+
+This information is then used by `treepkg` to know where to find the PKGBUILD
+of a package. The fullpkg family needed to guess this by traversing the full
+ABS tree, and therefore it was unable to find pkgnames that differ from
+pkgbase.
+
+So, to use `treepkg` you need to run `toru-path` after the ABS tree update.
+
+> Currently `toru-path` doesn't remove duplicated or updated pairs, but it
+> picks the last ones and only processes new PKGBUILDs. This means `toru-path`
+> works correctly but it's database will grow up slowly.
+
+## How does it work
+
+`treepkg` must be run from the path of the PKGBUILD you're going to build (this
+may change over time). This will be DEPTH=0 and it will create a temporary
+directory to work on in the format /tmp/pkgbase-treepkg-random-string. Inside
+this directory, it will copy the needed PKGBUILDs prefixed with their depth
+number. The first package will always be copied to 000\_pkgbase.
+
+From then on, treepkg sources the PKGBUILD and runs itself over all pkgnames on
+the depends and makedepends array, only if it detects their versions aren't
+already built or deprecated by newer ones, using the `is_built` utility.
+
+While processing this info, it will increase the depth of the packages. It'll
+also write a CSV file with the knowledge it acquires from the ABS tree (useful
+for debugging). This file is called BUILDORDER.
+
+When this process ends (no more needed dependencies are found), the temporary
+work dir is traversed in inverse order (from higher depths to 0) running the
+FULLBUILDCMD from libretools.conf and then the HOOKLOCALRELEASE variable, which
+*must* provide a way to `repo-add` the packages to an available repository, so
+the next build will find and install the newer version using pacman.
+
+For instance, having this as the first pacman repository (on /etc/pacman.conf):
+
+    [stage3]
+    Server = /var/cache/pacman/pkg
+
+And this on /etc/makepkg.conf:
+
+    PKGDEST=/var/cache/pacman/pkg
+
+Your HOOKLOCALRELEASE script should look like this:
+
+    # Get needed vars
+    source /etc/makepkg.conf
+    source /etc/libretools.conf
+    source PKGBUILD
+
+    unset build package check
+    
+    fullver=$(full_version ${epoch:-0} ${pkgver} ${pkgrel})
+    pkgs=()
+    
+    # Generate canonical package paths
+    msg "Adding packages to [stage3]..."
+    for name in ${pkgname[@]}; do
+        msg2 "${name} ${fullver}"
+        pkgs+=("${PKGDEST}/${name}-${fullver}-*.pkg.tar.*")
+    done
+    
+    # Add the packages to a local 
+    repo-add ${PKGDEST}/stage3.db.tar.gz ${pkgs[@]}
+
+    # Stage the packages for later releasing
+    librestage $1
+
+> Note the first HOOKLOCALRELEASE argument is the remote repository name (core,
+> extra, etc.)
+
+There's a special case when a dependency depends on another that was put on
+a depth level lower than itself. In this case the build order will be wrongly
+assumed and you may end up with broken packages.
+
+To explain it with an example:
+
+    ghostscript (0) - fontconfig (1)
+                    \ cups (1)       - fontconfig (ignored)
+
+The second time fontconfig appears, it will be ignored. In this case cups will
+build against an fontconfig version that will be outdated by the fontconfig
+version at depth 1. In this cases, `treepkg` will detect it cached the
+dependency on a lower depth, and will "bury" it to a depth higher than the
+current one. Thus this will become the build path:
+
+    ghostscript (0) - fontconfig (buried)
+                    \ cups (1)            - fontconfig (2)
+
+
+## Tips 
+
+`treepkg` accepts two arguments: 1) the temporary work dir and 2) the next
+depth level it should use (if current equals 0, it'll pass 1). You don't need
+to pass this arguments when running it manually, they're used internally to
+automatically construct the build path.
+
+But if a build failed, `treepkg` will cancel itself immediately informing you
+where the leftovers files where left. If you pass this path to `treepkg` as the
+first argument, it will resume the build, skipping to the last package being
+packaged.
+
+You can take the opportunity given by this to modify the build path or the
+PKGBUILDs, without having to re-run `treepkg` twice. For instance you can
+remove a package from the build order, or move it manually, or update the
+PKGBUILD that made `treepkg` fail in the first place.
+
+You don't probably want to mess with the second argument though.
diff --git a/librebasebuilder b/librebasebuilder
index 3eaadd8..f7d3a25 100644..100755
--- a/librebasebuilder
+++ b/librebasebuilder
@@ -45,7 +45,7 @@ done
 tempdir=/home/chroot/base
 outdir=$(pwd)
 
-mkarchroot $tempdir mkinitcpio base sudo parted nano zile vi ed openssh
+mkarchroot -n $tempdir mkinitcpio base sudo parted nano zile vi ed openssh
 
 cd $tempdir
 
@@ -61,6 +61,24 @@ cat > etc/resolv.conf <<EOF
 # End of file
 EOF
 
+# From pacman-mirrorlist-libre-20120307-1.  Remember to escape dollars.
+cat > etc/pacman.d/mirrorlist <<EOF
+# Parabola GNU/Linux - Last Updated: Wed Mar  7 17:33:36 GMT 2012
+
+# Atlanta, GA, USA
+# Responsible: belos
+Server = http://parabola.techno-geeks.org/\$repo/os/\$arch
+
+# Nuremberg, Germany
+# Responsible: johkra
+Server = http://parabolaweb.eu/\$repo/os/\$arch
+
+# UK
+# Responsible: Parabola Project
+Server = http://repo.parabolagnulinux.org/\$repo/os/\$arch
+
+EOF
+
 rm .arch-chroot
 
 tar cjf $outdir/parabola-mips64el-$(LC_ALL=C date -u +%Y%m%d).tar.bz2 .
diff --git a/libremakepkg b/libremakepkg
index 7db0e3c..f7924f6 100755
--- a/libremakepkg
+++ b/libremakepkg
@@ -24,7 +24,7 @@
 
 function copy_log {       # copy logs if they exist
 
-    find "${CHROOTDIR}/${CHROOT}/build/" -name "*\.log" -exec cp {} ./ \;
+    find "${CHROOTDIR}/${CHROOT}/build/" -maxdepth 1 -name "*\.log" -exec cp {} ./ \;
 
 }
 
diff --git a/librerelease b/librerelease
index bac5b88..c113572 100755
--- a/librerelease
+++ b/librerelease
@@ -147,4 +147,7 @@ rsync --recursive \
 msg "Removing packages from local [staging]"
 clean
 
+msg "Running db-update on repos"
+ssh ${PARABOLAHOST} dbscripts/db-update
+
 exit 0
diff --git a/mips64el/mipsrelease b/mips64el/mipsrelease
index 1a4aade..f245c47 100755
--- a/mips64el/mipsrelease
+++ b/mips64el/mipsrelease
@@ -34,6 +34,11 @@ get_full_version() {
 
 repo=$1; shift
 
+if [ -z "${repo}" ]; then
+    error "Empty repo"
+    exit 1
+fi
+
 # Get all needed sources
 source PKGBUILD
 fullver=$(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})
@@ -48,6 +53,9 @@ done
 
 repo-add ${PKGDEST}/stage3.db.tar.gz ${pkgs[@]}
 
+#sudo pacman -Sy
+librestage ${repo}
+
 
 mkdir -p ${WORKDIR}/abs/${CARCH}/${repo} >/dev/null
 
diff --git a/toru-path b/toru-path
index 5081115..5065a30 100755
--- a/toru-path
+++ b/toru-path
@@ -17,16 +17,18 @@ for _pkgbuild in ${pkgbuilds[@]}; do
     continue
   }
 
-  fullpath=$(dirname $(readlink -f ${_pkgbuild}))
+  fullpath=$(dirname ${_pkgbuild})
 
-  for _pkg in ${pkgname[@]} ${provides[@]}; do
+  for _pkg in ${pkgbase} ${pkgname[@]} ${provides[@]}; do
     paths+=(${_pkg/[<>=]*}:${fullpath})
   done
 
-  unset pkgname provides
+  unset pkgbase pkgname provides
 done
   
 # TODO remove old paths
+# by joining new paths to old paths and exclude the joining points from the
+# cache (sort of a sql join)
 echo ${paths[@]} | tr ' ' "\n" | sort >> ${TORUPATH}/paths
 
 lastsync ${LASTSYNCFILE}
diff --git a/toru-utils b/toru-utils
index 7e6d62d..0818aa6 100755
--- a/toru-utils
+++ b/toru-utils
@@ -59,17 +59,14 @@ get_pkgbuilds() {
 
     $QUIET || warning "Forcing upgrade"
 # Get all PKGBUILDs
-    pkgbuilds=($(find $@ -mindepth 2 -maxdepth 3 -type f -name 'PKGBUILD'))
-
+    extra=""
   else
-
 # Only find newer than lastsyncfile and read everything else from cache
-    pkgbuilds=($(find $@ -mindepth 2 -maxdepth 3 -type f -name 'PKGBUILD' -newer ${LASTSYNCFILE}))
-
+    extra=" -newer ${LASTSYNCFILE}"
   fi
 
 # Return all PKGBUILDs found
-  echo ${pkgbuilds[@]}
+  find $@ -mindepth 2 -maxdepth 3 -type f -name 'PKGBUILD' ${extra}
 }
 
 # End inmediately but print a useful message
@@ -81,6 +78,6 @@ trap_exit() {
 
 # Trap signals from makepkg
 set -E
-trap 'trap_exit "($(basename $0):${level}) TERM signal caught. Exiting..."' TERM HUP QUIT
-trap 'trap_exit "($(basename $0):${level}) Aborted by user! Exiting..."' INT
-trap 'trap_exit "($(basename $0):${level}) An unknown error has occurred. Exiting..."' ERR
+trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT
+trap 'trap_exit "Aborted by user! Exiting..."' INT
+trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR
diff --git a/treepkg b/treepkg
new file mode 100755
index 0000000..d21377d
--- /dev/null
+++ b/treepkg
@@ -0,0 +1,233 @@
+#!/bin/bash
+#set -x
+# (c) 2012  Nicolás Reynolds <fauno@parabola.nu>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+source /etc/libretools.conf
+source $XDG_CONFIG_HOME/libretools/libretools.conf >/dev/null 2>&1|| true
+
+# Get system variables
+source /etc/makepkg.conf
+source $HOME/makepkg.conf >/dev/null 2>&1|| true
+
+# End inmediately but print an useful message
+trap_exit() {
+  error "($(basename $0)) $@ (leftovers on ${BUILDDIR})"
+  exit 1
+}
+
+# Trap signals from makepkg
+set -E
+trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT
+trap 'trap_exit "Aborted by user! Exiting..."' INT
+trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR
+
+# return : full version spec, including epoch (if necessary), pkgver, pkgrel
+#  usage : get_fullver( ${epoch:-0}, $pkgver, $pkgrel )
+get_fullver() {
+    if [ $1 -eq 0 ]; then
+# zero epoch case, don't include it in version
+        echo $2-$3
+    else
+        echo $1:$2-$3
+    fi
+
+}
+
+# Add line to build order cache in CSV format
+# *must* be run from the PKGBUILD path
+# status;depth;pkgbase;[epoch:]pkgver-pkgrel;path;repo
+# $1 status
+# $2 pkgname
+add_order() {
+  echo "${1};${DEPTH};${2:-${pkgbase}};${fullver};${PWD};$(guess_repo "$PWD")" >> "${BUILDORDER}"
+  ${VERBOSE} && msg2 "%${DEPTH}s${2:-${pkgbase}} [${1}]" || true
+}
+
+# Bury a package deeper in the tree
+# $1 pkgbase
+# $2 nextdepth
+bury() {
+# Bury only if we are going to build the dep
+# Get it's current depth and dir name
+    local current_depth=$(grep "build;[0-9]\+;${1};" "${BUILDORDER}" | cut -d ';' -f 2)
+    local current_name="$(printf "%03d" ${current_depth})_${1}"
+
+# If there's a depth or the package is not the root of the build tree (which
+# can lead to funny chicken-and-egg problems), update the depth to the current
+# package next-depth and rename the dir too
+    if [ -z "${current_depth}" ]; then return; fi
+    if [ -z "${current_name}" ]; then return; fi
+    if [ ${current_depth} -eq 0 ]; then return; fi
+    if [ ${current_depth} -ge $2 ]; then return; fi
+
+    ${VERBOSE} && msg "Burying ${1} from ${current_depth} to ${2}"
+
+    {
+        sed -i "s|^\(build;\)\([0-9]\+\)\(;${1};.*\)$|\1${2}\3|" "${BUILDORDER}" && \
+        mv "${BUILDDIR}/${current_name}" "${BUILDDIR}/$(printf "%03d" ${2})_${1}"
+    } || return 1
+}
+
+# Finds a PKGBUILD on toru's path cache
+# Look in all caches but pick the first one
+# TODO move to a toru flag (-p?)
+where_is() {
+  grep -m1 "^${1}:" "${TORUPATH}/paths" 2>/dev/null| \
+    cut -d: -f2 2>/dev/null
+}
+
+# Guess the repo from the pkgbase path
+# $1 path, pwd or where_is
+guess_repo() {
+    basename "$(dirname "${1}")"
+}
+
+if [ ! -f PKGBUILD ]; then
+  error "Missing PKGBUILD ($PWD)"
+  exit 1
+fi
+
+if ! source PKGBUILD ; then
+  error "Can't source PKGBUILD"
+  exit 1
+fi
+
+# Save resources
+unset pkgdesc arch license groups backup install md5sums sha1sums \
+      sha256sums source options >/dev/null 2>&1
+
+unset build package >/dev/null 2>&1
+
+for _pkg in ${pkgname[@]}; do
+  unset package_${_pkg} >/dev/null 2>&1 || true
+done
+## 
+
+# Get useful values
+pkgbase="${pkgbase:-${pkgname[0]}}"
+fullver=$(get_fullver ${epoch:-0} ${pkgver} ${pkgrel})
+
+# Get or set the work dir
+BUILDDIR="${1:-$(mktemp -d /tmp/${pkgbase}-treepkg-XXXx)}"
+BUILDORDER="${BUILDDIR}/BUILDORDER"
+DEPTH=${2:-0}
+NEXTDEPTH=$((${DEPTH} + 1))
+# This can be set as env vars (ie: $ V=false B=false treepkg)
+# TODO Turn into flags?
+VERBOSE=${V:-true}
+BUILD=${B:-true}
+CLEANUP=${C:-true}
+# Skip BUILDORDER creation and build anything on BUILDDIR
+BUILDNOW=${N:-false}
+
+if [ ! -z "${1}" -a ${DEPTH} -eq 0 ]; then
+    BUILDNOW=true
+fi
+
+if ! ${BUILDNOW}; then
+# ensure it exists
+    touch "${BUILDORDER}"
+
+# If this package is already built quit silently
+    if is_built "${pkgbase}" "${fullver}"; then
+      add_order "ignore"
+      exit 0
+    fi
+
+# Ignore if already in build order
+    egrep -q ";${pkgbase};" "${BUILDORDER}" && exit 0
+
+# Add pkgbase to build order
+    add_order "build"
+
+# Copy the directory to the build dir
+# TODO run makepkg --source to avoid moving garbage around?
+    cp -r "${PWD}" "${BUILDDIR}/$(printf "%03d" ${DEPTH})_${pkgbase}"
+
+# Cleanup dep versioning
+    deps=($(echo "${depends[@]} ${makedepends[@]}" | \
+           sed "s/[=<>]\+[^ ]\+//g" | \
+           tr ' ' "\n" | \
+           sort -u))
+
+# NOTE: getting depends from package() is a PITA
+    for _dep in ${deps[@]}; do
+# Ignore if already in build order
+# TODO move deps deeper in the tree if
+# pkgbase - dep1
+#         \ dep2 - dep1
+# dep1 should be depth + 1
+# probably sed -i buildorder and increase dep's dep depth :D
+# remember to change dir name from xxx_pkgbase to xxx+1_pkgbase
+      egrep -q ";${_dep};" "${BUILDORDER}" && bury "${_dep}" ${NEXTDEPTH}
+
+# Ask toru where's a PKGBUILD
+      depdir="$(where_is ${_dep})"
+
+      if [ -z "${depdir}" -o ! -d "${depdir}" ]; then
+# We specify the pkgname because we can't source the dep PKGBUILD
+# Normally 'any' packages are missing from our work ABS
+        add_order "missing" "${_dep}"
+        continue
+      fi
+
+      pushd "${depdir}" >/dev/null
+
+# Run itself over dependencies
+      $0 "${BUILDDIR}" ${NEXTDEPTH}
+
+    done
+# End BUILD now
+fi
+
+# Only build at the end
+if [ ${DEPTH} -eq 0 ]; then
+  ${VERBOSE} && msg "Starting build" || true
+
+  if ${BUILD}; then
+    ${VERBOSE} && msg "Build tree stored in ${BUILDORDER}" || true
+
+# Build everything sorting the build dir
+# The reverse order ensures we start by the deepest packages
+    for _pkg in $(ls -r "${BUILDDIR}"); do
+# Ignore if there's no PKGBUILD
+      if [ ! -f "${BUILDDIR}/${_pkg}/PKGBUILD" ]; then continue; fi
+
+      ${VERBOSE} && msg "Building ${_pkg/_/ }" || true
+
+# Run build command
+      pushd "${BUILDDIR}/${_pkg}" >/dev/null
+        sudo pacman -Syu --noconfirm
+
+        ${FULLBUILDCMD}
+# Run local release hook with $1 = $repo
+        ${HOOKLOCALRELEASE} $(egrep ";${_pkg#*_};" "${BUILDORDER}" | cut -d';' -f6)
+      popd >/dev/null
+    done
+
+  else
+# Just print the working dir
+    ${VERBOSE} || echo "${BUILDORDER}" || true
+  fi
+
+if ${CLEANUP} ; then
+      msg2 "Removing ${BUILDDIR}"
+      rm -rf "${BUILDDIR}"
+fi
+
+fi
+
+exit $?