Toru maintains a cache of already processed PKGBUILDs

2 files changed, 127 insertions, 51 deletions

diff --git a/libretools.conf b/libretools.conf
index fa5d4b2..a506af7 100644
--- a/libretools.conf
+++ b/libretools.conf
@@ -54,6 +54,10 @@ ABSLIBREGIT=http://projects.parabolagnulinux.org/abslibre.git
 FULLBUILDCMD="sudo libremakepkg -cumL -M --noconfirm -M --nocheck"
 #FULLBUILDCMD="here is a place for cross-compiling build cmd"
 
+## Toru
+# Section for toru's vars
+TORUPATH=/var/lib/libretools/toru
+
 # Checks if vars aren't empty
 
 for VAR in CHROOTDIR CHROOT CHCOPY CACHEDIR PARABOLAHOST LIBREDESTDIR \
diff --git a/toru b/toru
index 332f42d..3304491 100755
--- a/toru
+++ b/toru
@@ -5,7 +5,6 @@
 ## TODO
 # * Add license text
 # * Create symlinks from pkgbase to pkgname[@] for easy package finding
-# * Use lastsync to store processed packages
 
 ## GOALS
 # * Have a searchable database of PKGBUILD metadata
@@ -13,43 +12,83 @@
 # * Possibility to hook up ABS dirs besides ABSROOT (low priority)
 # * Tell updates and non available binary packages (working on this)
 
- source /etc/abs.conf
- source /etc/libretools.conf
+source /etc/abs.conf
+source /etc/libretools.conf
+
+if [ ! -w "$TORUPATH" ]; then
+ error "Toru's path isn't writable. Please check $TORUPATH"
+ exit 1 
+fi
+
+# TODO move to common functions
+function in_array {  #  usage : in_array( $needle, $haystack )
+
+    [[ $2 ]] || return 1  # Not found
+    local needle=$1; shift
+    local item
+    for item in "$@"; do
+	[[ ${item#@} = $needle ]] && return 0  # Found
+    done
+    return 1 # Not Found
+}
 
 # Stores the lastsync date
- lastsync() {
-     [ -e ${lastsyncfile} -a ! -w ${lastsyncfile} ] && {
-         error "The sync date can't be saved. ${lastsyncfile} isn't writable."
-         return 1
-     }
+lastsync() {
+    [ -e ${lastsyncfile} -a ! -w ${lastsyncfile} ] && {
+        error "The sync date can't be saved. ${lastsyncfile} isn't writable."
+        return 1
+    }
+
+    date +%s > ${lastsyncfile}
+    touch ${lastsyncfile}
+}
+
+# Saves contents on a named cache
+# $1 cache name (repo)
+# $2+ contents
+store_cache() {
+  cache=$1; shift
 
-     date +%s > ${lastsyncfile}
-     touch ${lastsyncfile}
- }
+  [ -z "$cache" ] && return 1
+
+  cat $@ > ${TORUPATH}/${cache}.cache
+
+  return $?
+}
+
+# Return cache contents
+# $1 cache name
+read_cache() {
+  [ ! -e "$1" ] && return 1
+
+  cat ${TORUPATH}/${1}.cache
+
+  return $?
+}
 
 ##
 #  usage : get_full_version( $epoch, $pkgver, $pkgrel )
 # return : full version spec, including epoch (if necessary), pkgver, pkgrel
 ##
- get_full_version() {
-     if [[ $1 -eq 0 ]]; then
-        # zero epoch case, don't include it in version
-         echo $2-$3
-     else
-         echo $1:$2-$3
-     fi
- }
+get_full_version() {
+   if [[ $1 -eq 0 ]]; then
+      # zero epoch case, don't include it in version
+       echo $2-$3
+   else
+       echo $1:$2-$3
+   fi
+}
 
 # Outputs an ordered package-fullpkgver array
- print_package_array() {
-     echo "$@" | tr " " "\n" | sort -V -u
- }
+print_package_array() {
+   echo "$@" | tr " " "\n" | sort -V -u
+}
 
 
 # Gets repo.db contents (unordered)
 # $1 repo
- get_db_contents() {
-     [ ! -r /var/lib/pacman/sync/$1.db ] && return 0
+get_db_contents() {
+  [ ! -r /var/lib/pacman/sync/$1.db ] && return 0
 
   bsdtar -tf /var/lib/pacman/sync/$1.db | cut -d'/' -f1
 }
@@ -81,13 +120,22 @@ extract_fullpkgver() {
      local needed_updates=()
      local old_versions=()
 
+# Traverse all specified repos
+     for _repo in $@; do
+       if ! in_array ${_repo} ${REPOS[@]}; then
+         warning "You don't have access to this repo (check REPOS at libretools.conf)"
+         continue
+       fi
+
 # Find all the PKGBUILDs newer than the last update
 # Update newer, otherwise everything
-     if [ $force ] || [ ! -e ${lastsyncfile} ]; then
+     if [[ $force = true || ! -e ${lastsyncfile} ]]; then
          $quiet || msg "Forcing upgrade"
-         pkgbuilds=($(find ${@} -maxdepth 2 -type f -name 'PKGBUILD'))
+         pkgbuilds=($(find ${_repo} -maxdepth 2 -type f -name 'PKGBUILD'))
      else
-         pkgbuilds=($(find ${@} -maxdepth 2 -type f -name 'PKGBUILD' -newer ${lastsyncfile}))
+         pkgbuilds=($(find ${_repo} -maxdepth 2 -type f -name 'PKGBUILD' -newer ${lastsyncfile}))
+         packages_to_sync=($(read_cache ${_repo}))
+         msg2 "Getting ${#packages_to_sync[@]} packages from cache"
      fi
 
 # Inform how many PKGBUILDS were found and quit immediately if none
@@ -97,17 +145,29 @@ extract_fullpkgver() {
          exit 0
      }
 
+# Traverse all found PKGBUILDs
      for _pkgbuild in ${pkgbuilds[@]}; do
 
 # The repo name is guessed
 # You *must* use repo/pkgbase structure
          _pkgpath=$(dirname "${_pkgbuild}")
          _pkgbase=$(basename "${_pkgpath}")
-         _pkgrepo=$(basename $(dirname "${_pkgpath}"))
 
+# Load PKGBUILD's metadata
          source ${_pkgbuild}
 
+# We won't need this
+         unset build package url md5sums install pkgdesc backup options
+# TODO fill a license list
+         unset license
+# TODO create source tarballs?
+         unset mksource
+# TODO solve dependency tree?
+         unset depends makedepends
+
          for _pkg in ${pkgname[@]}; do
+# Keep removing unneeded stuff
+             unset package_${_pkg} >/dev/null 2>&1
 # Fill the list of packages to find
              packages_to_sync+=($_pkg-$(get_full_version ${epoch:-0} $pkgver $pkgrel))
          done
@@ -116,33 +176,41 @@ extract_fullpkgver() {
      done
 
 # Get repo database contents
-     packages_in_sync=($(get_db_contents ${_pkgrepo}))
-     print_package_array "${packages_to_sync[@]}" > ${TMPDIR}/packages_to_sync
-     print_package_array "${packages_in_sync[@]}" > ${TMPDIR}/packages_in_sync
+    packages_in_sync=($(get_db_contents ${_repo}))
+    print_package_array "${packages_to_sync[@]}" > ${TMPDIR}/packages_to_sync
+    print_package_array "${packages_in_sync[@]}" > ${TMPDIR}/packages_in_sync
 
 # We've orderer the files!
-  needed_updates=($(comm --nocheck-order -32 ${TMPDIR}/packages_to_sync ${TMPDIR}/packages_in_sync))
-  old_versions=($(comm --nocheck-order -31 ${TMPDIR}/packages_to_sync ${TMPDIR}/packages_in_sync))
-  missing_packages=($(comm --nocheck-order -12 ${TMPDIR}/packages_to_sync ${TMPDIR}/packages_in_sync))
+    needed_updates=($(comm --nocheck-order -32 ${TMPDIR}/packages_to_sync ${TMPDIR}/packages_in_sync))
+    old_versions=($(comm --nocheck-order -31 ${TMPDIR}/packages_to_sync ${TMPDIR}/packages_in_sync))
+#    missing_packages=($(comm --nocheck-order -12 ${TMPDIR}/packages_to_sync ${TMPDIR}/packages_in_sync))
 
-     $quiet || msg "This packages are available to update"
-     for _update in ${needed_updates[@]}; do
-         pkg=$(extract_pkgname $_update)
+   $quiet || msg "This packages are available to update"
+   for _update in ${needed_updates[@]}; do
+      pkg=$(extract_pkgname $_update)
 
 # Only print pkgnames when in quiet mode (useful for scripts)
-    $quiet && echo $pkg
-    $quiet || {
-      ver=$(extract_fullpkgver $_update)
-      oldver=$(extract_fullpkgver $(grep -w $pkg ${TMPDIR}/packages_in_sync))
+      $quiet && echo $pkg
+      $quiet || {
+          ver=$(extract_fullpkgver $_update)
+          oldver=$(extract_fullpkgver $(grep -w $pkg ${TMPDIR}/packages_in_sync))
 
-             msg2 "$pkg $oldver => $ver"
-         }
+          msg2 "$pkg $oldver => $ver"
+      }
 
      done
 
-#  lastsync
+# Save the cache
+    store_cache ${_repo} ${TMPDIR}/packages_to_sync
+
+    unset packages_to_sync packages_in_sync needed_updates old_versions pkg \
+          ver oldver 
+
+  done
 
- }
+  lastsync
+
+}
 
 # Find all the packages that are missing from the repo dbs (aka not built)
 missing() {
@@ -169,14 +237,18 @@ while getopts 'hqfum' arg; do
 done
 
 # This is the syncfile, stores the last date as content and mtime
-#lastsyncfile=${ABSROOT}/toru.lastsync
+lastsyncfile=${TORUPATH}/lastsync
+
+TMPDIR=$(mktemp -d)
 
- TMPDIR=$(mktemp -d)
+[[ -z ${TMPDIR} ]] && exit 1
 
- [[ -z ${TMPDIR} ]] && exit 1
+# TODO this is all for debugging
+msg2 "force: $force quiet: $quiet repos: $@ commands: ${commands[@]}"
+msg2 "read lastsync: $([ -e $lastsyncfile ])"
 
- ${commands[0]} ${@}
+${commands[0]} ${@}
 
- rm -rf ${TMPDIR}
+rm -rf ${TMPDIR}
 
- exit $?
+exit $?