diff options
author | Nicolás Reynolds <apoyosis@correo.inta.gob.ar> | 2012-11-05 11:44:26 -0300 |
---|---|---|
committer | Nicolás Reynolds <apoyosis@correo.inta.gob.ar> | 2012-11-05 11:44:26 -0300 |
commit | 369245e8dbee7ad97acec087de77081627a255db (patch) | |
tree | 8e128b0bad84d042b0d504e3e5ed726b0ffe9457 | |
parent | 3d3156b97f679a84210c08ca7e7563691d988a3b (diff) | |
parent | 9983b0c757834d61b9099a5953dec9d92330ac76 (diff) |
Merge branch 'master' of git://ponape.local/libretools
-rwxr-xr-x | chcleanup | 54 | ||||
-rwxr-xr-x | fullpkg | 1 | ||||
-rwxr-xr-x | fullpkg-find | 58 | ||||
-rwxr-xr-x | libremessages | 6 | ||||
-rwxr-xr-x | libremkchroot | 13 | ||||
-rwxr-xr-x | librerelease | 13 | ||||
-rw-r--r-- | libretools.conf | 23 | ||||
-rwxr-xr-x | mips64el/mipsrelease | 5 | ||||
-rwxr-xr-x | toru-info | 28 | ||||
-rwxr-xr-x | toru-path | 34 | ||||
-rwxr-xr-x | toru-utils | 7 | ||||
-rwxr-xr-x | toru-where | 7 | ||||
-rwxr-xr-x | treepkg | 40 | ||||
-rwxr-xr-x | update-cleansystem | 66 |
14 files changed, 259 insertions, 96 deletions
@@ -1,11 +1,59 @@ #!/bin/bash +# (c) Nicolás Reynolds <fauno@parabola.nu> +# Released under GPLv3 +# +# Performs chroot cleanup smartly, it only removes the unneeded packages or +# leaves you with a cleansystem +# +# See: HOOKPREBUILD + +set -e [ ! -f /etc/libretools.d/cleansystem ] && exit 1 +[ ! -d "${DB:-/var/lib/libretools/clean}"/sync ] && exit 1 + +source $(dirname $0)/libremessages +source /etc/makepkg.conf +source ${HOME}/.makepkg.conf 2>/dev/null|| true + +msg "Cleaning chroot..." + +cleanup_log=/tmp/libretools-cleanup.log +touch ${cleanup_log} + +# If we're running makepkg +if [ -f PKGBUILD ]; then + source PKGBUILD || true + + check=(${depends[@]} ${makedepends[@]} ${checkdepends[@]}) + + if [ ${#check[@]} -ne 0 ]; then + +# Update the cleansystem database + sudo pacman -b "${BD:-/var/lib/libretools/clean}" -Sy +# Get the full list of packages needed by dependencies + sudo pacman -b "${BD:-/var/lib/libretools/clean}" \ + -Sp \ + --print-format "%n" \ + ${check[@]} \ + >${cleanup_log} + fi +fi + +# Diff installed packages against a clean chroot and needed packages, +# then remove leftovers +packages=($(comm -23 <(pacman -Qq | sort) \ + <(cat /etc/libretools.d/cleansystem ${cleanup_log} | sort -u) + )) + +[ ${#packages[@]} -eq 0 ] && exit 0 -packages=($(comm -23 <(pacman -Qq | sort) <(sort /etc/libretools.d/cleansystem))) +msg2 "Removing %d packages" ${#packages[@]} -echo "Removing: ${packages[@]}" +# Only remove leftovers, -Rcs removes too much +sudo pacman --noconfirm -Rn ${packages[@]} -sudo pacman --noconfirm -Rcs ${packages[@]} +# Cleanup +rm -f ${cleanup_log} exit $? @@ -15,6 +15,7 @@ usage() { echo " -h : this message." echo "" echo "Wrapper for \`fullpkg-find' and \`fullpkg-build'" + echo "" exit 1 } diff --git a/fullpkg-find b/fullpkg-find index 64c1790..e188ec8 100755 --- a/fullpkg-find +++ b/fullpkg-find @@ -11,14 +11,6 @@ guess_repo() { basename $(dirname $(pwd)) } -# Finds a PKGBUILD on toru's path cache -# Look in all caches but pick the first one -# TODO move to a toru flag (-p?) -where_is() { - grep -m1 "^${1}:" "${TORUPATH}/paths" 2>/dev/null| \ - cut -d: -f2 2>/dev/null -} - # return : full version spec, including epoch (if necessary), pkgver, pkgrel # usage : get_fullver( ${epoch:-0}, $pkgver, $pkgrel ) get_fullver() { @@ -31,6 +23,36 @@ get_fullver() { } +copy_files() { + + local copydir="$build_dir/${pkgbase:-${pkgname[0]}}" + mkdir -p "$copydir" + + # Copy PKGBUILD and sources + cp PKGBUILD "$copydir" + ( + source PKGBUILD + for file in "${source[@]}"; do + file="${file%%::*}" + file="${file##*://*/}" + if [[ -f $file ]]; then + cp "$file" "$copydir/" + elif [[ -f $SRCDEST/$file ]]; then + cp "$SRCDEST/$file" "$copydir/" + fi + done + + # Find all changelog and install files, even inside functions + for i in 'changelog' 'install'; do + while read -r file; do + # evaluate any bash variables used + eval file=\"$(sed 's/^\(['\''"]\)\(.*\)\1$/\2/' <<< "$file")\" + [[ -f $file ]] && cp "$file" "$copydir" + done < <(sed -n "s/^[[:space:]]*$i=//p" PKGBUILD) + done + ) +} + # Checks ABSROOT and look for target pkg deps. Adds them if not built or outdated. find_deps() { # Check this level @@ -63,7 +85,7 @@ find_deps() { exit 0 # Copy dir to build_dir else - cp -r ../${pkgbase}/ ${build_dir}/ + copy_files # to identify repo later echo "repo=$repo" > "${build_dir}/${pkgbase}/.INFO" @@ -84,7 +106,6 @@ find_deps() { for _dep in ${deps[@]}; do local found=false -# TODO change for where_is or toru-path # May fail, e.g. since abslibre-mips64el doesn't include # arch=any packages. local pkgdir=$(toru -p ${_dep}) || true @@ -133,7 +154,7 @@ usage() { echo "" echo "OPTIONS:" echo " -h : this message." -# echo " -A <absroot> : use this ABSROOT." # Is it needed anymore? + echo " -A <absroot> : use this ABSROOT." echo " -c : clean <build_dir> before working." echo " -m <max_level> : check deps until this level" echo " -n : don't update pacman db." @@ -145,7 +166,7 @@ usage() { while getopts 'hA:l:cmn' arg; do case "$arg" in h) usage ;; -# A) ABSROOT="$OPTARG" ;; + A) ABSROOT="$OPTARG" ;; l) LEVEL="$OPTARG" ;; # hidden option to know dep level. c) CLEANFIRST='true' ;; m) MAXLEVEL="$OPTARG" ;; @@ -185,18 +206,7 @@ if [ "$LEVEL" -eq 0 ]; then msg "Checking dependencies" fi -# Probable circular deps -[ "$LEVEL" -gt "$MAXLEVEL" ] && exit 20 - # Find the dependencies on the ABS itself -find_deps || { - -# Probable circular deps - if [ "$?" -eq 20 -a "$LEVEL" -eq 0 ]; then - error "Check for circular deps on $build_dir/BUILDORDER"; - fi -# Pass message 20 - exit 20 -} +find_deps exit 0 diff --git a/libremessages b/libremessages index 5d817dd..68badb8 100755 --- a/libremessages +++ b/libremessages @@ -86,6 +86,12 @@ stat_done() { printf "${BOLD}done${ALL_OFF}\n" >&2 } +# Set the terminal title +# TODO test on several terms - it works on screen/tmux +term_title() { + printf "\033k%s\033\\" "$@" +} + # usage : in_array( $needle, $haystack ) in_array() { [[ $2 ]] || return 1 # Not found diff --git a/libremkchroot b/libremkchroot index c1f2072..b576209 100755 --- a/libremkchroot +++ b/libremkchroot @@ -2,7 +2,7 @@ # LibreMkChroot # Creates a chroot -# Copyright 2011 Luke Shumaker +# Copyright 2011, 2012 Luke Shumaker # ---------- GNU General Public License 3 ---------- @@ -27,10 +27,7 @@ if [ -e "$XDG_CONFIG_HOME/libretools/libretools.conf" ]; then source "$XDG_CONFIG_HOME/libretools/libretools.conf" fi -#CHROOTNAME="${CHROOT:-${SUDO_USER:-root}}" - cmd=${0##*/} - function usage { echo "Usage: $cmd [OPTIONS]" echo 'This script will create a chroot to build packages in.' @@ -47,12 +44,12 @@ function usage { echo ' -M <file> Location of makepkg config file.' } -mkchroot_args=''; +mkchroot_args=(); while getopts 'hfd:c:C:M:' arg; do case "$arg" in h) usage; exit 0 ;; - f) mkchroot_args="$mkchroot_args -$arg" ;; - c|C|M) mkchroot_args="$mkchroot_args -$arg $OPTARG" ;; + f) mkchroot_args+=("-$arg");; + c|C|M) mkchroot_args+=("-$arg" "$OPTARG");; d) CHROOTDIR=$OPTARG ;; ?) usage; exit 1 ;; esac @@ -64,4 +61,4 @@ if (( EUID )); then fi mkdir -p "${CHROOTDIR}" -xargs -d'\n' mkarchroot $mkchroot_args "${CHROOTDIR}/root" < /etc/libretools.d/cleansystem +xargs -d'\n' mkarchroot "${mkchroot_args[@]}" "${CHROOTDIR}/root" < /etc/libretools.d/cleansystem diff --git a/librerelease b/librerelease index c113572..c4d97c1 100755 --- a/librerelease +++ b/librerelease @@ -66,8 +66,6 @@ function sign_packages { if ! gpg --quiet --verify "${package}${SIGEXT}" >/dev/null 2>&1; then error "Failed! Resigning..." rm -f "${package}${SIGEXT}" - else - continue fi fi @@ -91,7 +89,7 @@ function clean_non_packages { # Clean everything if not on dry-run mode function clean { [ -z ${dryrun} ] && \ - find ${WORKDIR}/staging/ -type f -delete + rm -f $@ } if [ -w / ]; then @@ -116,7 +114,7 @@ clean_non_packages if [ ! -z "${SIGID}" ]; then sign_packages else - error "Package signing is *required*" + error "Package signing is *required*, please set SIGID on your libretools.conf" exit 1 fi @@ -126,6 +124,9 @@ fi find ${WORKDIR}/staging -type f -exec chmod 644 {} \; find ${WORKDIR}/staging -type d -exec chmod 755 {} \; +# Get the synced files +SYNCED_FILES=($(find ${WORKDIR}/staging -type f)) + msg "Uploading packages..." rsync --recursive \ ${dryrun} \ @@ -144,8 +145,8 @@ rsync --recursive \ exit 1 } -msg "Removing packages from local [staging]" -clean +msg "Removing ${#SYNCED_FILES[@]} files from local [staging]" +clean ${SYNCED_FILES[@]} msg "Running db-update on repos" ssh ${PARABOLAHOST} dbscripts/db-update diff --git a/libretools.conf b/libretools.conf index 514c37d..88ba515 100644 --- a/libretools.conf +++ b/libretools.conf @@ -7,10 +7,12 @@ DIFFTOOL=vimdiff # The dir where you work on WORKDIR=/home/$USER/packages # The repos you'll be packaging for +# # Tip: As early repos take precedence on $REPOS loops, you can use this as # inverted order of precedence. Put testing repos first so fullpkg find new -# PKGBUILDs first, for instance. -REPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar') +# PKGBUILDs first, for instance. Toru-path uses reverse order to enforce repo +# precedence on the path cache (the last path added replaces the rest) +REPOS=('core' 'libre' 'extra' 'community' 'libre-testing' 'social' 'sugar') # The architectures ARCHES=('i686' 'x86_64' 'mips64el' 'any') @@ -30,9 +32,6 @@ CACHEDIR=`grep "^#\?CacheDir" /etc/pacman.conf | cut -d'=' -f2` ## Parabola hostname (should be the same used on ssh_config PARABOLAHOST=parabola -## Run a command before releasing a package (ie. SSH connection, SSH tunnel, etc.) -HOOKPRERELEASE="ssh -fN parabola" - ## Server destination of libre packages # Don't change unless you know what you're doing and you won't screw # anything ;) @@ -52,6 +51,20 @@ ABSLIBREGIT=http://projects.parabolagnulinux.org/abslibre.git FULLBUILDCMD="sudo libremakepkg -cuN" # Cross compiling fullkpg # FULLBUILDCMD="sudo libremakepkg -cuN -d '/path/to/cross-compiling/chroot'" +# Build from within the chroot (or host system) +# FULLBUILDCMD="makepkg -sL --noconfirm" + +# Run a command before releasing a package (ie. SSH connection, SSH tunnel, etc.) +HOOKPRERELEASE="ssh -fN parabola" + +# Run a command before running FULLBUILDCMD, usually to cleanup uneeded packages +# Note! chcleanup *is not* chroot aware, if you run it as it is it will cleanup +# your system +# HOOKPREBUILD="chcleanup" + +# Locally release the package or any other action after running FULLBUILDCMD +# succesfully +# HOOKLOCALRELEASE="" ## Toru # Section for toru's vars diff --git a/mips64el/mipsrelease b/mips64el/mipsrelease index f245c47..4d7a7de 100755 --- a/mips64el/mipsrelease +++ b/mips64el/mipsrelease @@ -43,7 +43,7 @@ fi source PKGBUILD fullver=$(get_full_version ${epoch:-0} ${pkgver} ${pkgrel}) pkgs=() -makepkg --source -f +makepkg --source -f --skippgpcheck msg "Adding packages to [stage3]..." for name in ${pkgname[@]}; do @@ -53,7 +53,6 @@ done repo-add ${PKGDEST}/stage3.db.tar.gz ${pkgs[@]} -#sudo pacman -Sy librestage ${repo} @@ -63,6 +62,4 @@ pushd ${WORKDIR}/abs/${CARCH}/${repo} >/dev/null tar xvf $SRCPKGDEST/${pkgbase:-${pkgname[0]}}-${fullver}${SRCEXT} popd >/dev/null -$libretoolsdir/chcleanup || true - exit $? diff --git a/toru-info b/toru-info new file mode 100755 index 0000000..523f682 --- /dev/null +++ b/toru-info @@ -0,0 +1,28 @@ +#!/bin/bash +# Prints info about a given pkgname +source /etc/libretools.conf + +for _pkg in $@; do + _pkgbuild="$(toru-where $_pkg)" + + if [ -f "$_pkgbuild/PKGBUILD" ]; then + source "$_pkgbuild/PKGBUILD" 2>/dev/null || { + warning "Errors on %s" $_pkg + continue + } + + deps="${depends[@]} ${makedepends[@]} ${checkdepends[@]}" + repo="$(basename $(dirname "$_pkgbuild"))" + + unset build package depends makedepends checkdepends optdepends source md5sums + + msg "%s/%s %s-%s" $repo $_pkg $pkgver $pkgrel + msg2 "$pkgdesc" + msg2 "$url" + msg2 "Depends: ${deps}" + else + warning "%s doesn't exist" $_pkg + fi + + unset pkgname pkgver pkgrel pkgdesc url +done @@ -2,17 +2,35 @@ source $(dirname $(command -v $0))/toru-utils +TORUPATH=${T:-${TORUPATH}} +VERBOSE=${V:-false} + +if [ ! -w "$TORUPATH" ]; then + error "Toru's path isn't writable. Please check $TORUPATH" + exit 1 +fi + LASTSYNCFILE=${TORUPATH}/lastsync.paths +PATHFILE=${TORUPATH}/paths.tch + +if [ ! -e "${PATHFILE}" ]; then + tcamgr create "${PATHFILE}" +fi # TODO pass other paths via flags -pkgbuilds=($(get_pkgbuilds ${ABSROOT})) -paths=() +# ABSROOT has trailing slash +fullrepos=() +for (( i = ${#REPOS[@]}-1 ; i >= 0 ; i-- )); do + ${VERBOSE} && msg "Processing [%s]" ${REPOS[$i]} + fullrepos+=("${ABSROOT}${REPOS[$i]}") +done +pkgbuilds=($(get_pkgbuilds ${fullrepos[@]})) msg "Updating path cache" msg2 "${#pkgbuilds[@]} PKGBUILDs to update" for _pkgbuild in ${pkgbuilds[@]}; do - $DEBUG && plain "$_pkgbuild" - source ${_pkgbuild} || { +# plain "$_pkgbuild" + source ${_pkgbuild} >/dev/null 2>&1 || { error "${_pkgbuild} contains errors, skipping" continue } @@ -20,15 +38,11 @@ for _pkgbuild in ${pkgbuilds[@]}; do fullpath=$(dirname ${_pkgbuild}) for _pkg in ${pkgbase} ${pkgname[@]} ${provides[@]}; do - paths+=(${_pkg/[<>=]*}:${fullpath}) + $VERBOSE && msg2 "${_pkg} -> ${fullpath}" + tcamgr put ${PATHFILE} ${_pkg/[<>=]*} ${fullpath} done unset pkgbase pkgname provides done - -# TODO remove old paths -# by joining new paths to old paths and exclude the joining points from the -# cache (sort of a sql join) -echo ${paths[@]} | tr ' ' "\n" | sort >> ${TORUPATH}/paths lastsync ${LASTSYNCFILE} @@ -1,15 +1,8 @@ #!/bin/bash -#!/bin/bash - source /etc/abs.conf source /etc/libretools.conf -if [ ! -w "$TORUPATH" ]; then - error "Toru's path isn't writable. Please check $TORUPATH" - exit 1 -fi - LASTSYNCFILE=${TORUPATH}/lastsync FORCE=false QUIET=false diff --git a/toru-where b/toru-where new file mode 100755 index 0000000..e9ab29d --- /dev/null +++ b/toru-where @@ -0,0 +1,7 @@ +#!/bin/bash +# Locates a PKGBUILD dir on toru's path cache +source /etc/libretools.conf + +PATHFILE=${TORUPATH}/paths.tch + +tcamgr get ${PATHFILE} $1 2>/dev/null || echo "" @@ -18,12 +18,15 @@ source /etc/libretools.conf source $XDG_CONFIG_HOME/libretools/libretools.conf >/dev/null 2>&1|| true +term_title "$(basename $0)" + # Get system variables source /etc/makepkg.conf source $HOME/makepkg.conf >/dev/null 2>&1|| true # End inmediately but print an useful message trap_exit() { + term_title "error!" error "($(basename $0)) $@ (leftovers on ${BUILDDIR})" exit 1 } @@ -81,16 +84,8 @@ bury() { } || return 1 } -# Finds a PKGBUILD on toru's path cache -# Look in all caches but pick the first one -# TODO move to a toru flag (-p?) -where_is() { - grep -m1 "^${1}:" "${TORUPATH}/paths" 2>/dev/null| \ - cut -d: -f2 2>/dev/null -} - # Guess the repo from the pkgbase path -# $1 path, pwd or where_is +# $1 path, pwd or toru-where guess_repo() { basename "$(dirname "${1}")" } @@ -142,10 +137,12 @@ if ! ${BUILDNOW}; then touch "${BUILDORDER}" # If this package is already built quit silently - if is_built "${pkgbase}" "${fullver}"; then - add_order "ignore" - exit 0 - fi + for _pkg in ${pkgname[@]}; do + if is_built "${_pkg}" "${fullver}"; then + add_order "ignore" + exit 0 + fi + done # Ignore if already in build order egrep -q ";${pkgbase};" "${BUILDORDER}" && exit 0 @@ -165,17 +162,14 @@ if ! ${BUILDNOW}; then # NOTE: getting depends from package() is a PITA for _dep in ${deps[@]}; do -# Ignore if already in build order -# TODO move deps deeper in the tree if +# Move deps deeper in the tree if # pkgbase - dep1 # \ dep2 - dep1 # dep1 should be depth + 1 -# probably sed -i buildorder and increase dep's dep depth :D -# remember to change dir name from xxx_pkgbase to xxx+1_pkgbase egrep -q ";${_dep};" "${BUILDORDER}" && bury "${_dep}" ${NEXTDEPTH} # Ask toru where's a PKGBUILD - depdir="$(where_is ${_dep})" + depdir="$(toru-where ${_dep})" if [ -z "${depdir}" -o ! -d "${depdir}" ]; then # We specify the pkgname because we can't source the dep PKGBUILD @@ -205,16 +199,25 @@ if [ ${DEPTH} -eq 0 ]; then for _pkg in $(ls -r "${BUILDDIR}"); do # Ignore if there's no PKGBUILD if [ ! -f "${BUILDDIR}/${_pkg}/PKGBUILD" ]; then continue; fi +# Skip if already built (faster than calling is_build again) + if [ -f "${BUILDDIR}/${_pkg}/built_ok" ]; then continue; fi ${VERBOSE} && msg "Building ${_pkg/_/ }" || true +# Remove leading zeros and space if any + term_title "$(echo ${_pkg/_/ } | sed "s/^0\+ \?//")" + # Run build command pushd "${BUILDDIR}/${_pkg}" >/dev/null sudo pacman -Syu --noconfirm + ${HOOKPREBUILD} + ${FULLBUILDCMD} # Run local release hook with $1 = $repo ${HOOKLOCALRELEASE} $(egrep ";${_pkg#*_};" "${BUILDORDER}" | cut -d';' -f6) + + touch built_ok popd >/dev/null done @@ -230,4 +233,5 @@ fi fi +term_title "done" exit $? diff --git a/update-cleansystem b/update-cleansystem index dd3ff6c..6bec742 100755 --- a/update-cleansystem +++ b/update-cleansystem @@ -1,24 +1,68 @@ #!/bin/bash # Updates the cleansystem file -# Creates a fake Parabola root and prints all packages installable from base -# and base-devel plus extras +# Creates a fake Parabola root and writes to cleansystem all +# packages installable from base and base-devel plus extras. -set -E +set -e +# Copyright 2012 Nicolás Reynolds, Luke Shumaker -if [ ! -w / ]; then - echo "Run as root." - exit 1 +# ---------- GNU General Public License 3 ---------- + +# This file is part of Parabola. + +# Parabola is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Parabola is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Parabola. If not, see <http://www.gnu.org/licenses/>. + +set -e + +# libretools.conf gives us libremessages +source /etc/libretools.conf + +cleansystem=/etc/libretools.d/cleansystem + +cmd=${0##*/} +usage() { + echo "Usage: $cmd [<EXTRA_PACKAGES>]" + echo " $cmd -h" + echo "Creates a fake Parabola root and writes to \`$cleansystem' all" + echo "packages installable from base and base-devel plus extras." + echo '' + echo 'Options:' + echo ' -h Show this message' +} + +if [ "$1" == '-h' ]; then + usage + exit 0 fi -tmpdir=/tmp/cleansystem.${RANDOM} +if [ ! -w "$cleansystem" ]; then + error 'This script must be run as root' + exit 1 +fi -mkdir -p ${tmpdir}/var/lib/pacman +# Maintain a clean database in the system +db_dir="${DB:-/var/lib/libretools/clean}" +[ ! -d "${db_dir}" ] && mkdir -p "${db_dir}" # We sync first because updating info gets printed to stdout too -pacman -r ${tmpdir} --config /etc/pacman.conf -Sy -pacman -r ${tmpdir} \ +pacman -b "${db_dir}" --config /etc/pacman.conf -Sy 2>/dev/null +pacman -b "${db_dir}" \ --config /etc/pacman.conf \ -Sp --print-format "%n" \ - base base-devel sudo ${@} | sort > /etc/libretools.d/cleansystem + base base-devel sudo $@ | sort > "$cleansystem" + +# Ensures everything's installed +pacman -Sy --needed --noconfirm base base-devel sudo $@ exit $? |