fullpkg-ng -> fullpkg

1 files changed, 294 insertions, 185 deletions

diff --git a/fullpkg b/fullpkg
index a23a448..a80f962 100755
--- a/fullpkg
+++ b/fullpkg
@@ -1,15 +1,16 @@
 #!/bin/bash
-# TODO
-# * Do version checking
-# * Detect circular builds
+# TO TEST: (on find_deps)
 # * Detect pkgnames by provides, replaces, etc. instead of dir tree
 
 source /etc/makepkg.conf
 source /etc/abs.conf
 source /etc/libretools.conf
+source /usr/bin/libremessages
+
+[ -r $XDG_CONFIG_HOME/libretools/libretools.conf ] && \
+    source $XDG_CONFIG_HOME/libretools/libretools.conf
 
-[[ -r ~/.config/libretools/libretools.conf ]] && \
-    source ~/.config/libretools/libretools.conf
+## START FUNCTIONS ##
 
 function usage {
     echo "cd to a dir containing a PKGBUILD and run:"
@@ -18,50 +19,26 @@ function usage {
     printf "and stage the packages on it's repo."
     echo
     echo "OPTIONS:"
-    echo " -h          : this message."
-    echo " -f          : build even when a package has been built."
-    echo " -n absdir   : set ABSROOT to this dir"
-    echo " -r reponame : set repo name to reponame"
-    echo " -R pkgname  : build pkgname if it is a dep"
-    echo
-}
-
-force_build='n'
-force_array=()
-_fullpkgargs=""
-failed=()
-missing=()
-
-while getopts 'hfn:r:R:' arg; do
-    case $arg in
-	h) usage; exit 0 ;;
-	f) force_build='y' ;;
-	R) force_array=(${force_array[@]} $OPTARG); _fullpkgargs+="-R $OPTARG ";;
-	n) ABSROOT="$OPTARG" ;;
-	r) repo="$OPTARG" ;;
-    esac
-done
-
-[[ ! -r PKGBUILD ]] && {
-    error "This isn't a build directory"
+    echo " -h           : this message."
+    echo " -a absdir    : set absdir as ABSROOT."
+    echo " -b           : do not check deps but build. Needs -d"
+    echo " -c           : check deps only, do not build."
+    echo " -d build_dir : use this dir to build. Defaults to mktemp."
+    echo " -n           : don't update pacman db."
+    echo " -m max_level : check deps until this level"
+    echo " -r \"command\" : use this instead of \"$FULLBUILDCMD\""
+    # printf " -f pkgname  : build even when a package has been built. "
+    # printf " Use it as many times as needed\n"
     echo
-    usage
-    exit 1
 }
 
-tmp_dir=$(mktemp -d /tmp/$(basename $PWD).XXXXXX)
-queue_file=$(mktemp /tmp/queue.XXXXXX)
-ban_file=$(mktemp /tmp/ban.XXXXXX)
-
-## START FUNCTIONS ##
-
 # Queue Management
 # * Always get the queue list from the server
 # * Add/Remove from queue
 # * Check if a package is listed
 
 # TODO
-# * Check for concurrence
+# * Check for concurrence # @ fauno: What do you mean?
 
 # Get the queue list from the server
 get_queue() {
@@ -81,30 +58,28 @@ put_queue() {
 
 # Add packages to the queue
 update_queue() {
-    get_queue || return $?
-
-    basename $PWD | sed "s/$/:$PACKAGER/" >> $queue_file || return 2
-
+    get_queue
+    echo "$(basename $PWD):$PACKAGER" >> $queue_file
     put_queue || return $?
 }
 
 # Remove a package from the queue
 remove_queue() {
-    get_queue || return $?
+    get_queue
 
     grep -vw "^$(basename $PWD)" $queue_file > $queue_file.2
-    cat $queue_file.2 > $queue_file
+    mv $queue_file.2 $queue_file
 
-    put_queue && rm $queue_file{,.2}  && return 0 || return $?
+    put_queue && return 0 || return $?
 }
 
 # Checks if a package is listed
 check_queue() {
-    get_queue || return $?
+    get_queue
 
-    packager=$(grep -w "$(basename $PWD)" ${queue_file} | cut -d ':' -f2)
+    local packager=$(grep -w "$(basename $PWD)" ${queue_file} | cut -d ':' -f2)
 
-    [[ ! -z $packager ]] && [[ "$packager" != "$PACKAGER" ]] && {
+    [ -n "$packager" -a "$packager" != "$PACKAGER" ] && {
         warning "$(basename $PWD) is being packaged by $packager. Please wait."
         return 1
     }
@@ -114,160 +89,294 @@ check_queue() {
 
 # END Queue Management #
 
-# Checks if the package is banned from building
-is_banned() {
-    rsync -e ssh -aq $PARABOLAHOST:mips64el/ban $ban_file >/dev/null 2>&1 || {
-        plain "Failed to get ban list"
-        return 1
-    }
-    grep -w $1 $ban_file >/dev/null 2>&1
-    return $?
-}
-
 guess_repo() {
     basename $(dirname $(pwd))
 }
 
-#  usage : in_array( $needle, $haystack )
-# return : 0 - found
-#          1 - not found
-function in_array {
-    [[ $2 ]] || return 1
-    local needle=$1; shift
-    local item
-    for item in "$@"; do
-	[[ ${item#@} = $needle ]] && return 0
-    done
-    return 1 # Not Found
-}
-
-function quit {
-    remove_queue
-    exit 1
-}
-
+# Usage: cleanup [ $(basename $PWD) ] from PKGBUILD dir
+# cleans the build_dir
 function cleanup {
-    rm $ban_file $queue_file
-    rm -rf $tmp_dir
+    if [ ! -d $build_dir ]; then
+	return 1
+    elif [ -d $build_dir -a ${#@} -gt 0 ]; then
+	for _dir in $@; do
+	    rm -rf $build_dir/$_dir/
+	done
+    else
+	rm -rf $build_dir/*
+    fi
 }
 
-# TODO keep track of spawned fullpkgs
-
-## END FUNCTIONS ##
-
-source PKGBUILD
-repo=${repo:-$(guess_repo)}
-msg "Building ${repo:-missing repo}/${pkgbase:-${pkgname[@]}}: $pkgdesc"
-
-# Pre build tests
-if [ $force_build == 'n' ]; then
-
-    # Be able to write files
-    if [[ ! -w $queue_file ]]; then
-	error "can't write queue file"
-	exit 1
-    elif [[ ! -w $ban_file ]] ; then
-	error "can't write ban file"
-	exit 1
-    fi
+# Check PKGBUILD and find non built or outdated deps
+# on ABSROOT which should be abslibre-misp64el
+function find_deps {
+    ## Check this level.
+    source PKGBUILD
+    local repo=${repo:-$(guess_repo)}
 
+    # If package is built exit
+    # TODO?: If this package is in force_build: skip this step
     if is_built "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}"; then
-	msg2 "This package is built."
 	exit 0
     fi
 
-    if is_banned ${pkgbase:-$pkgname}; then 
-	error "This package is banned from building. Check the ban list"
-	exit 1
+    # Tell which packages are deeper in deps (even if they are on build_dir)
+    # so we can build them first.
+    echo "${level}:$(basename $PWD)" >> "${build_dir}/BUILDORDER"
+
+    # if pkgbuild directory is on build_dir, do not copy and exit
+    if [ -d "${build_dir}/$(basename $PWD)" ]; then
+	exit 0
+    else
+	cp -r ../$(basename $PWD) ${build_dir}/
+	# Info to eval later
+	echo "repo=$repo" > "${build_dir}/$(basename $PWD)/.INFO"
     fi
+    
+    msg2 "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}"
+
+    ## Check next levels
+    # Clean version checking
+    deps=$(echo "${depends[@]} ${makedepends[@]}" | \
+	sed "s/[=<>]\+[^ ]\+//g" | \
+	tr ' ' "\n" | \
+	sort -u)
+    declare -i next_level=$level+1
+    
+    for _dep in ${deps[@]}; do
+	for _repo in ${REPOS[@]}; do
+	    # try to find $_dep on each repo from dirname
+	    if [ -e "$ABSROOT/${_repo}/$_dep/PKGBUILD" ]; then
+		pushd "$ABSROOT/${_repo}/$_dep" > /dev/null
+		$0 -c -d ${build_dir} -l ${next_level}
+		# Circular deps must fail
+		[ $? -eq 20 ] && return 20
+		popd > /dev/null
+		break 1 # found, go to next dep
+	    # if search pkgname in repo doesn't work
+	    # this should find pkgsplits
+	    elif _dir=($(find "$ABSROOT/${_repo}" -type f -name PKGBUILD -print0 | \
+		"xargs" -0 -e grep -HEw "pkgname|pkgbase|provides" | grep $_dep 2>&1));
+	    then
+		pushd $(dirname $(echo $_dir | cut -d: -f1)) > /dev/null
+		$0 -c -d ${build_dir} -l ${next_level}
+		# Circular deps must fail
+		[ $? -eq 20 ] && return 20
+		popd > /dev/null
+		break 1 # found, go to next dep
+	    else
+		echo "dep_not_found:$_dep:$_repo" >> $build_dir/log
+	    fi
+	done
+    done
+    unset next_level dir
+    # unset PKGBUILD variables
+    unset pkgname pkgver pkgrel epoch pkgdesc arch url license groups depends \
+	makedepens checkdepends optdepends provides conflicts replaces backup \
+	options install changelog source noextract md5sums build check package
+}
 
-    check_queue || exit 1
-
-fi
-
-# This will be executed at exit for any reason.
-trap "quit" EXIT INT QUIT TERM KILL HUP
-
-if ! grep mips64el PKGBUILD >/dev/null; then
-  msg "Adding mips64el arch"
-  sed -i "s/^\(arch=([^)anym]\+\))/\1 'mips64el')/" "PKGBUILD"
-fi
-
-# Clean version checking
-deps=$(echo "${depends[@]} ${makedepends[@]} ${pkgdeps[@]}" | \
-    sed "s/[=<>]\+[^ ]\+//g" | \
-    tr ' ' "\n" | \
-    sort -u)
-
-msg "Checking dependencies"
-for _dep in ${deps[@]}; do
-    is_banned $_dep && continue
-
-    for _repo in ${REPOS[@]}; do
-        # TODO find split packages
-        [[ -e "$ABSROOT/${_repo}/$_dep/PKGBUILD" ]] && {
-            source "$ABSROOT/${_repo}/$_dep/PKGBUILD"
-            msg2 "Checking for $_dep>=$pkgver-$pkgrel"
-	    
-            if ! in_array $_dep ${force_array[@]}; then
-              if is_built "$_dep>=$pkgver-$pkgrel"; then
-                  plain "this package is built"
-                  break
-              fi
-            else
-              _fullpkgargs+="-f "
-              _fullpkgargs="$(echo $_fullpkgargs | sed s/"-R $_dep "//)"
-              force_array=( $(echo ${forcearray[@]} | tr " " "\n" | grep -vw "^$_dep") )
-            fi
-	    
-            cp -r "$ABSROOT/$_repo/$_dep" $tmp_dir/ || {
-                error "Can't copy $_dep to the work dir."
-                exit 1
-            }
-
-            # Enter the work dir and run this command in it
-            pushd $tmp_dir/$_dep >/dev/null
-
-	    $0 -r $_repo $_fullpkgargs
-
-            [[ $? -ne 0 ]] && {
-                failed=(${failed[@]} $_dep)
-            }
-
-            popd >/dev/null
-        }
+function _pkg_build () {
+    pushd ${build_dir}  > /dev/null
+    # packages to build are on $buildorder
+    # greater levels must be built first
+    build_packages=($(sort -gr $buildorder | cut -d: -f2))
+    while [ ${#build_packages[@]} -ge 1 ]; do
+	pushd $build_dir/${build_packages[0]} > /dev/null
+	source PKGBUILD 
+	msg2 "${pkgbase:-${pkgname[0]}} $pkgver-$pkgrel"
+	# Add mips64el
+	if ! grep mips64el PKGBUILD >/dev/null; then
+	    plain "Adding mips64el arch"
+	    sed -i "s/^\(arch=([^)anym]\+\))/\1 'mips64el')/" "PKGBUILD"
+	fi
+        # Check if pkg is being built
+	msg2 "Checking build queue"
+	check_queue || {
+	    echo "someone_is_building:$(basename $PWD)" >> $build_dir/log
+	    grep -vwh "$(basename $PWD)" $buildorder > $buildorder.2
+	    mv $buildorder.2 $buildorder
+	    continue
+	}
+	# Let everybody know we're building this.
+	msg2 "Updating build queue"
+	update_queue || {
+	    warning "Couldn't update the queue, let your partners know about this."
+	}
+	msg2 "Checking for non free deps"
+	pkgbuild-check-nonfree || {
+	    # pkgbuild-check-nonfree fails with 15 if there are nonfree deps,
+	    # fails with something else if blacklist can't be retrieved
+	    if [ $? -eq 15 ]; then
+		# log they have nonfree deps and so didn't build
+		echo "nonfree:$(basename $PWD)" >> $build_dir/log
+		# take out package from $buildorder
+		grep -vwh "$(basename $PWD)" $buildorder > $buildorder.2
+		mv $buildorder.2 $buildorder
+		# continue building next package
+		continue
+	    fi
+	}
+	msg2 "Building $(basename $PWD)"
+	# this buildcmd is on libretools.conf
+	$FULLBUILDCMD; r=$?
+	case $r in
+	    0) plain "The build was succesful."
+		source .INFO && [ -n $repo ] && {
+		    librestage $repo || {
+			echo "unstaged:$(basename $PWD)" >> $build_dir/log
+		    }
+		}
+		echo "built:$(basename $PWD)" >> $build_dir/log
+		cleanup "$(basename $PWD)" 
+		;;
+	    1) error "There were errors while trying to build the package." 
+		echo "failed:$(basename $PWD)" >> $build_dir/log
+		;;
+	    2) error "The build failed."
+		echo "failed:$(basename $PWD)" >> $build_dir/log
+		;;
+	esac
+	# Package was built or failed: take it out of $buildorder
+	grep -vwh "${build_packages[0]}" $buildorder > $buildorder.2
+	mv $buildorder.2 $buildorder
+	# Take package out from queue
+	remove_queue
+	# Set build_packages before next cycle run
+	build_packages=($(sort -gr $buildorder | cut -d: -f2))
+	popd > /dev/null
     done
+    pkgs=$(cat $build_dir/log | grep "nonfree:") && {
+	error "Those packages contain nonfree deps:"
+	echo ${pkgs[@]} | tr " " "\n" | cut -d: -f2
+    }
+    pkgs=$(cat $build_dir/log | tr " " "\n" | grep "built:") && {
+	error "Those packages were built and installed:"
+	echo ${pkgs[@]} | tr " " "\n" | cut -d: -f2
+	msg "Uploading packages to the server"
+    }
+    pkgs=$(cat $build_dir/log | tr " " "\n" | grep "failed:") && {
+	error "Those packages failed to build:"
+	echo ${pkgs[@]} | tr " " "\n" | cut -d: -f2
+    }
+    pkgs=$(cat $build_dir/log | tr " " "\n" | grep "unstaged:") && {
+	error "Those packages couldn't be staged because of missing reponame:"
+	echo ${pkgs[@]} | tr " " "\n" | cut -d: -f2
+    }
+    popd > /dev/null
+}
+## END FUNCTIONS ##
+
+force_build=""
+level=0
+noupdate='n'
+build_only='n'
+check_deps_only='n'
+do_cleanup='n'
+max_level=21
+while getopts 'ha:bcCd:l:nm:r:' arg; do
+    case $arg in
+	h) usage; exit 0 ;;
+	a) ABSROOT="$OPTARG" ;;
+	b) build_only='y' ;;
+	c) check_deps_only='y' ;;
+	C) do_cleanup='y';;
+	# f) force_build+="-f pkgname " ;;
+	d) build_dir="$OPTARG" ;;
+	# hidden option to know what to build first.
+	# if $level > 0 it will not build
+	l) level=$OPTARG ;;
+	n) noupdate='y';;
+	m) max_level=$OPTARG ;;
+	r) FULLBUILDCMD="$OPTARG" ;;
+    esac
 done
 
-# TODO probably not elegant enough
-# TODO only the last fullpkg should show this message
-#      and it should contain all failed pkgs
-[[ ${#failed[@]} -gt 0 ]] && {
-    error "This packages failed to build: ${failed[@]}"
-    exit 1
+# Only on level 0
+[ $level -eq 0 ] && {
+    # if build_dir exist use it, else make a build_dir
+    build_dir=${build_dir:-$(mktemp -d /tmp/fullpkg.XXXXXX)}
+    # cleanup if the option was used.
+    [ $do_cleanup == 'y' ] && {
+	cleanup
+	[ $build_only == 'y' ] && exit 0
+    }
+    # set queue_file and ban_file
+    mkdir -p $XDG_CONFIG_HOME/libretools
+    queue_file=$XDG_CONFIG_HOME/libretools/queue
+    ban_file=$XDG_CONFIG_HOME/libretools/ban
+    # make files for log and buildorder
+    touch $build_dir/{log,BUILDORDER}
+    buildorder=$build_dir/BUILDORDER
+
+    [ $noupdate = 'n' ] && {
+	msg "Updating pacman db and packages"
+	sudo pacman -Syu --noconfirm
+    }
+    # Build only
+    [ $build_only == 'y' ] && {
+	# Exit loop on Ctrl+C
+	trap "break" INT
+        # Remove from queue package being built on error
+	trap "remove_queue" EXIT INT QUIT TERM KILL
+	_pkg_build
+	exit 0
+    }
+    msg "Checking dependencies"
 }
 
-# Let everybody know we're building this
-update_queue || {
-    warning "Couldn't update the queue, let your partners know about this."
+[ ! -r PKGBUILD ] && {
+    error "This isn't a build directory"
+    usage && exit 1
 }
 
-cp -r ../$(basename $PWD) $tmp_dir/
-pushd $tmp_dir/$(basename $PWD) >/dev/null
-
-msg "Syncing database"
-sudo pacman -Syu --noconfirm
-makepkg --noconfirm  --nocheck -sLcr ; r=$?
-case $r in
-    0) msg "The build was succesful."
-       mipsrelease *.pkg.tar.*
-       librestage $repo
-       sudo pacman -Sy
-       # cleanup is only on succesfull build so failed can be inspected
-       cleanup;;
-    1) error "There were errors while trying to build the package." ;;
-    2) error "The build failed." ;;
-esac
-
-exit $r
+## if $level = 20 there is highly likely there are circular deps
+[ $level -eq $max_level -o $level -gt $max_level ] && exit 20
+
+
+find_deps || { 
+    # if find_deps finds circular deps
+    # it should exit with status 20
+    [ $? -eq 20 ] && {
+	# only show message on level 0
+	[ $level -eq 0 ] && error "Check for circular deps on $build_dir/BUILDORDER";
+    }
+    exit 20
+}
+
+# levels greater than 0 must only check deps
+[ $check_deps_only = 'y' -o $level -gt 0 ] && exit 0
+
+# check BUILDORDER to not include banned deps and
+[ $level -eq 0 -a -d $build_dir ] && {
+    # Check for banned deps
+    if [ -w $ban_file -a -r $ban_file ]; then
+	chmod o+rw $ban_file || error "Ban file is not readable/writable ($ban_file)"
+    else
+	rsync -e ssh -aq $PARABOLAHOST:mips64el/ban >/dev/null 2>&1 || {
+	    warning "Failed to get ban list" && [ -r $ban_file ] && {
+	        # continue if download failed but local copy
+		search=$(cat $ban_file | tr "\n" "|")
+		echo ${@} | tr " " "\n" | egrep -w "$search" $buildorder >> $build_dir/banned
+		echo ${@} | tr " " "\n" | egrep -vw "$search" $buildorder > $buildorder.2
+		mv $buildorder.2 $buildorder
+		unset search
+	    }
+	}
+    fi
+}
+
+## START Building
+msg "Building packages:"
+
+[ ! -w $queue_file ] && error "can't write queue file"
+# Exit loop on Ctrl+C
+trap "break" INT
+# Remove from queue package being built on error
+trap "remove_queue" EXIT INT QUIT TERM KILL
+_pkg_build
+
+echo
+msg2 "Check if your system works fine and librerelease if it does"
 
+exit 0