From c8beffa7904abe7e0ad01fed6113acf449df15cd Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Thu, 19 Feb 2009 19:12:34 +0100 Subject: Fix several issues with xdelta 1) The changes to sync.c look big but there are mostly caused by the indentation. Fix a bug where download_size == 0 because the packages and deltas are already in the cache, but we still need to build the deltas list and apply the deltas to create the final package. 2) Fix the gzip / md5sum issue by switching to xdelta3, disabling external recompression and using gzip -n in pacman, and disable bsdtar compression and using gzip -n in makepkg. Signed-off-by: Xavier Chantry --- doc/pacman.conf.5.txt | 2 +- lib/libalpm/delta.c | 5 ++- lib/libalpm/sync.c | 118 +++++++++++++++++++++++++++----------------------- scripts/makepkg.sh.in | 23 +++++++--- 4 files changed, 83 insertions(+), 65 deletions(-) diff --git a/doc/pacman.conf.5.txt b/doc/pacman.conf.5.txt index fa212947..2f1fe3df 100644 --- a/doc/pacman.conf.5.txt +++ b/doc/pacman.conf.5.txt @@ -144,7 +144,7 @@ Options *UseDelta*:: Download delta files instead of complete packages if possible. Requires - the xdelta program to be installed. + the xdelta3 program to be installed. *TotalDownload*:: When downloading, display the amount downloaded, download rate, ETA, diff --git a/lib/libalpm/delta.c b/lib/libalpm/delta.c index 9e4bcb42..de5dd601 100644 --- a/lib/libalpm/delta.c +++ b/lib/libalpm/delta.c @@ -21,6 +21,7 @@ #include #include +#include /* intmax_t */ #include #include #include @@ -215,13 +216,13 @@ off_t _alpm_shortest_delta_path(alpm_list_t *deltas, return(bestsize); } - _alpm_log(PM_LOG_DEBUG, "started delta shortest-path search\n"); + _alpm_log(PM_LOG_DEBUG, "started delta shortest-path search for '%s'\n", to); vertices = delta_graph_init(deltas); bestsize = delta_vert(vertices, to, &bestpath); - _alpm_log(PM_LOG_DEBUG, "delta shortest-path search complete\n"); + _alpm_log(PM_LOG_DEBUG, "delta shortest-path search complete : '%jd'\n", (intmax_t)bestsize); alpm_list_free_inner(vertices, _alpm_graph_free); alpm_list_free(vertices); diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index fca96d8f..49ea3c27 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -27,6 +27,7 @@ #include #include #include +#include /* intmax_t */ #include #include #include @@ -387,8 +388,8 @@ static int compute_download_size(pmpkg_t *newpkg) size = alpm_pkg_get_size(newpkg); } - _alpm_log(PM_LOG_DEBUG, "setting download size %lld for pkg %s\n", - (long long)size, alpm_pkg_get_name(newpkg)); + _alpm_log(PM_LOG_DEBUG, "setting download size %jd for pkg %s\n", + (intmax_t)size, alpm_pkg_get_name(newpkg)); newpkg->download_size = size; return(0); @@ -679,6 +680,12 @@ off_t SYMEXPORT alpm_pkg_download_size(pmpkg_t *newpkg) return(newpkg->download_size); } +static int endswith(char *filename, char *extension) +{ + char *s = filename + strlen(filename) - strlen(extension); + return (strcmp(s, extension) == 0); +} + /** Applies delta files to create an upgraded package file. * * All intermediate files are deleted, leaving only the starting and @@ -724,16 +731,18 @@ static int apply_deltas(pmtrans_t *trans) CALLOC(to, len, sizeof(char), RET_ERR(PM_ERR_MEMORY, 1)); snprintf(to, len, "%s/%s", cachedir, d->to); - /* an example of the patch command: (using /cache for cachedir) - * xdelta patch /path/to/pacman_3.0.0-1_to_3.0.1-1-i686.delta \ - * /path/to/pacman-3.0.0-1-i686.pkg.tar.gz \ - * /cache/pacman-3.0.1-1-i686.pkg.tar.gz - */ - /* build the patch command */ - snprintf(command, PATH_MAX, "xdelta patch %s %s %s", delta, from, to); + /* compression command */ + char *compress = "cat"; + if(endswith(to, ".gz")) { + compress = "gzip -n"; + } else if(endswith(to, ".bz2")) { + compress = "bzip"; + } + /* -R for disabling external recompression, -c for sending to stdout */ + snprintf(command, PATH_MAX, "xdelta3 -d -q -R -c -s %s %s | %s > %s", from, delta, compress, to); - _alpm_log(PM_LOG_DEBUG, _("command: %s\n"), command); + _alpm_log(PM_LOG_DEBUG, "command: %s\n", command); EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_START, d->to, d->delta); @@ -847,29 +856,31 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data) fname = alpm_pkg_get_filename(spkg); ASSERT(fname != NULL, RET_ERR(PM_ERR_PKG_INVALID_NAME, -1)); - if(spkg->download_size != 0) { - alpm_list_t *delta_path = spkg->delta_path; - if(delta_path) { - alpm_list_t *dlts = NULL; - - for(dlts = delta_path; dlts; dlts = dlts->next) { - pmdelta_t *d = dlts->data; - - if(d->download_size != 0) { - /* add the delta filename to the download list if - * it's not in the cache */ - files = alpm_list_add(files, strdup(d->delta)); - } - - /* keep a list of the delta files for md5sums */ - deltas = alpm_list_add(deltas, d); + alpm_list_t *delta_path = spkg->delta_path; + if(delta_path) { + /* using deltas */ + alpm_list_t *dlts = NULL; + + for(dlts = delta_path; dlts; dlts = dlts->next) { + pmdelta_t *d = dlts->data; + + if(d->download_size != 0) { + /* add the delta filename to the download list if needed */ + files = alpm_list_add(files, strdup(d->delta)); } - } else { - /* not using deltas, so add the file to the download list */ + /* keep a list of all the delta files for md5sums */ + deltas = alpm_list_add(deltas, d); + } + + } else { + /* not using deltas */ + if(spkg->download_size != 0) { + /* add the filename to the download list if needed */ files = alpm_list_add(files, strdup(fname)); } } + } } @@ -890,37 +901,34 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data) handle->totaldlcb(0); } - if(handle->usedelta) { + /* if we have deltas to work with */ + if(handle->usedelta && deltas) { int ret = 0; - - /* only output if there are deltas to work with */ - if(deltas) { - errors = 0; - /* Check integrity of deltas */ - EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_START, NULL, NULL); - - for(i = deltas; i; i = i->next) { - pmdelta_t *d = alpm_list_getdata(i); - const char *filename = alpm_delta_get_filename(d); - const char *md5sum = alpm_delta_get_md5sum(d); - - if(test_md5sum(trans, filename, md5sum) != 0) { - errors++; - *data = alpm_list_add(*data, strdup(filename)); - } + errors = 0; + /* Check integrity of deltas */ + EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_START, NULL, NULL); + + for(i = deltas; i; i = i->next) { + pmdelta_t *d = alpm_list_getdata(i); + const char *filename = alpm_delta_get_filename(d); + const char *md5sum = alpm_delta_get_md5sum(d); + + if(test_md5sum(trans, filename, md5sum) != 0) { + errors++; + *data = alpm_list_add(*data, strdup(filename)); } - if(errors) { - pm_errno = PM_ERR_DLT_INVALID; - goto error; - } - EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_DONE, NULL, NULL); + } + if(errors) { + pm_errno = PM_ERR_DLT_INVALID; + goto error; + } + EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_DONE, NULL, NULL); - /* Use the deltas to generate the packages */ - EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_START, NULL, NULL); - ret = apply_deltas(trans); - EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_DONE, NULL, NULL); + /* Use the deltas to generate the packages */ + EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_START, NULL, NULL); + ret = apply_deltas(trans); + EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_DONE, NULL, NULL); - } if(ret) { pm_errno = PM_ERR_DLT_PATCHFAILED; goto error; diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in index 0aa8a9b1..a41b0695 100644 --- a/scripts/makepkg.sh.in +++ b/scripts/makepkg.sh.in @@ -963,25 +963,34 @@ create_package() { # tar it up msg2 "$(gettext "Compressing package...")" - local TAR_OPT case "$PKGEXT" in - *tar.gz) TAR_OPT="z" ;; - *tar.bz2) TAR_OPT="j" ;; + *tar.gz) EXT=${PKGEXT%.gz} ;; + *tar.bz2) EXT=${PKGEXT%.bz2} ;; *) warning "$(gettext "'%s' is not a valid archive extension.")" \ - "$PKGEXT" ;; + "$PKGEXT" ; EXT=$PKGEXT ;; esac + local pkg_file="$PKGDEST/${nameofpkg}-${pkgver}-${pkgrel}-${CARCH}${EXT}" - local pkg_file="$PKGDEST/${nameofpkg}-${pkgver}-${pkgrel}-${CARCH}${PKGEXT}" + local ret=0 # when fileglobbing, we want * in an empty directory to expand to # the null string rather than itself shopt -s nullglob + bsdtar -cf - $comp_files * > "$pkg_file" || ret=$? + shopt -u nullglob - if ! bsdtar -c${TAR_OPT}f "$pkg_file" $comp_files *; then + if [ $ret -eq 0 ]; then + case "$PKGEXT" in + *tar.gz) gzip -f -n "$pkg_file" ;; + *tar.bz2) bzip2 -f "$pkg_file" ;; + esac + ret=$? + fi + + if [ $ret -ne 0 ]; then error "$(gettext "Failed to create package file.")" exit 1 # TODO: error code fi - shopt -u nullglob } create_srcpackage() { -- cgit v1.2.3-54-g00ecf From 9fa18d9a4b4ce5217842c71d8a45676e3fb9d3f4 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Wed, 25 Feb 2009 19:26:31 +0100 Subject: Add pkgdelta script to create deltas. This should obsolete the delta support in makepkg. Having a separate script should be more flexible. Example usage: $ pkgdelta repo/tzdata-2009a-1-x86_64.pkg.tar.gz repo/tzdata-2009b-1-x86_64.pkg.tar.gz ==> Generating delta from version 2009a-1 to version 2009b-1 ==> Generated delta : 'repo/tzdata-2009a-1_to_2009b-1-x86_64.delta' Signed-off-by: Xavier Chantry --- scripts/.gitignore | 1 + scripts/Makefile.am | 3 + scripts/pkgdelta.sh.in | 165 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 scripts/pkgdelta.sh.in diff --git a/scripts/.gitignore b/scripts/.gitignore index f2f19fd8..eafc4930 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -3,3 +3,4 @@ pacman-optimize rankmirrors repo-add repo-remove +pkgdelta diff --git a/scripts/Makefile.am b/scripts/Makefile.am index d6d9bb93..5a2b780c 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -8,12 +8,14 @@ bin_SCRIPTS = \ OURSCRIPTS = \ makepkg \ pacman-optimize \ + pkgdelta \ rankmirrors \ repo-add EXTRA_DIST = \ makepkg.sh.in \ pacman-optimize.sh.in \ + pkgdelta.sh.in \ rankmirrors.py.in \ repo-add.sh.in @@ -58,6 +60,7 @@ $(OURSCRIPTS): Makefile makepkg: $(srcdir)/makepkg.sh.in pacman-optimize: $(srcdir)/pacman-optimize.sh.in +pkgdelta: $(srcdir)/pkgdelta.sh.in rankmirrors: $(srcdir)/rankmirrors.py.in repo-add: $(srcdir)/repo-add.sh.in repo-remove: $(srcdir)/repo-add.sh.in diff --git a/scripts/pkgdelta.sh.in b/scripts/pkgdelta.sh.in new file mode 100644 index 00000000..588dc49d --- /dev/null +++ b/scripts/pkgdelta.sh.in @@ -0,0 +1,165 @@ +#!/bin/bash +# +# pkgdelta - create delta files for use with pacman and repo-add +# @configure_input@ +# +# Copyright (c) 2009 Xavier Chantry +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# bash options +set -o nounset +set -o errexit + +# gettext initialization +export TEXTDOMAIN='pacman' +export TEXTDOMAINDIR='@localedir@' + +myver='@PACKAGE_VERSION@' + +QUIET=0 + +# ensure we have a sane umask set +umask 0022 + +msg() { + [ $QUIET -ne 0 ] && return + local mesg=$1; shift + printf "==> ${mesg}\n" "$@" >&1 +} + +warning() { + local mesg=$1; shift + printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 +} + +# print usage instructions +usage() { + printf "pkgdelta (pacman) %s\n\n" "$myver" + printf "$(gettext "Usage: pkgdelta [-q] \n")" + printf "$(gettext "\ + pkgdelta will create a delta file between two packages\n\ +This delta file can then be added to a database using repo-add.\n\n")" + echo "$(gettext "Example: pkgdelta pacman-3.0.0.pkg.tar.gz pacman-3.0.1.pkg.tar.gz")" +} + +version() { + printf "pkgdelta (pacman) %s\n\n" "$myver" + printf "$(gettext "\ +Copyright (c) 2009 Xavier Chantry .\n\n\ +This is free software; see the source for copying conditions.\n\ +There is NO WARRANTY, to the extent permitted by law.\n")" +} + +read_pkginfo() +{ + pkgname= pkgver= arch= + local OLDIFS=$IFS + # IFS (field separator) is only the newline character + IFS=" +" + local line var val + for line in $(bsdtar -xOf "$1" .PKGINFO 2>/dev/null | + grep -v "^#" | sed 's|\(\w*\)\s*=\s*\(.*\)|\1="\2"|'); do + eval "$line" + if [ -n "$pkgname" -a -n "$pkgver" -a -n "$arch" ]; then + IFS=$OLDIFS + return 0 + fi + done + IFS=$OLDIFS + error "$(gettext "Invalid package '%s'")" "$1" + return 1 +} + +# $oldfile $oldmd5 $newfile $newmd5 $deltafile $deltamd5 $deltasize +create_xdelta() +{ + local oldfile=$1 + local newfile=$2 + local \ + oldname oldver oldarch \ + newname newver newarch \ + deltafile + + read_pkginfo "$oldfile" || return 1 + oldname="$pkgname" + oldver="$pkgver" + oldarch="$arch" + read_pkginfo "$newfile" || return 1 + newname="$pkgname" + newver="$pkgver" + newarch="$arch" + + if [ "$oldname" != "$newname" ]; then + error "$(gettext "The package names don't match : '%s' and '%s'")" "$oldname" "$newname" + return 1 + fi + + if [ "$oldarch" != "$newarch" ]; then + error "$(gettext "The package architectures don't match : '%s' and '%s'")" "$oldarch" "$newarch" + return 1 + fi + + if [ "$oldver" == "$newver" ]; then + error "$(gettext "Both packages have the same version : '%s'")" "$newver" + return 1 + fi + + msg "$(gettext "Generating delta from version %s to version %s")" "$oldver" "$newver" + deltafile="$(dirname $newfile)/$pkgname-${oldver}_to_${newver}-$arch.delta" + local ret=0 + + xdelta3 -q -f -s "$oldfile" "$newfile" "$deltafile" || ret=$? + if [ $ret -ne 0 ]; then + error "$(gettext "Delta could not be created.")" + return 1 + else + msg "$(gettext "Generated delta : '%s'")" "$deltafile" + [ $QUIET -eq 1 ] && echo "$deltafile" + fi + return 0 +} + +case "$1" in + -q|--quiet) QUIET=1; shift ;; +esac + +if [ $# -ne 2 ]; then + usage + exit 0 +fi + +if [ ! -f "$1" ]; then + error "$(gettext "File '%s' does not exist")" "$1" + exit 0 +fi + +if [ ! -f "$2" ]; then + error "$(gettext "File '%s' does not exist")" "$2" + exit 0 +fi + +if [ ! "$(type -p xdelta3)" ]; then + error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" + exit 1 +fi + +create_xdelta "$1" "$2" -- cgit v1.2.3-54-g00ecf From f8bb69c1d22473a3a82c6bab27799e3cada56a0a Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Thu, 26 Feb 2009 19:23:33 +0100 Subject: repo-add cleanup Refactor the main loop, which was difficult to read. Use case instead of if when appropriate. Signed-off-by: Xavier Chantry --- scripts/repo-add.sh.in | 118 +++++++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index b12188ce..4520dd4c 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -270,6 +270,48 @@ db_remove_entry() { popd 2>&1 >/dev/null } # end db_remove_entry +check_repo_db() +{ + if [ -f "$REPO_DB_FILE" ]; then + if ! (bsdtar -tf "$REPO_DB_FILE" | grep -q "/desc"); then + error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" + exit 1 + fi + msg "$(gettext "Extracting database to a temporary location...")" + bsdtar -xf "$REPO_DB_FILE" -C "$gstmpdir" + else + if [ "$cmd" == "repo-remove" ]; then + error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" + exit 1 + fi + fi +} + +add() +{ + pkgfile=$1 + if [ ! -f "$1" ]; then + error "$(gettext "Package '%s' not found.")" "$pkgfile" + return 1 + fi + + if ! bsdtar -tf "$pkgfile" .PKGINFO 2>&1 >/dev/null; then + error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" + return 1 + fi + + msg "$(gettext "Adding package '%s'")" "$pkgfile" + + db_write_entry "$pkgfile" +} + +remove() +{ + msg "$(gettext "Searching for package '%s'...")" "$arg" + + db_remove_entry "$arg" +} + # PROGRAM START # determine whether we have gettext; make it a no-op if we do not @@ -279,17 +321,10 @@ if [ ! $(type -t gettext) ]; then } fi -# check for help flags -if [ "$1" = "-h" -o "$1" = "--help" ]; then - usage - exit 0 -fi - -# check for version flags -if [ "$1" = "-V" -o "$1" = "--version" ]; then - version - exit 0 -fi +case "$1" in + -h|--help) usage; exit 0;; + -V|--version) version; exit 0;; +esac # check for correct number of args if [ $# -lt 2 ]; then @@ -312,52 +347,29 @@ fi success=0 # parse arguments for arg in "$@"; do - if [ "$arg" == "--force" -o "$arg" == "-f" ]; then - warning "$(gettext "the -f and --force options are no longer recognized")" - msg2 "$(gettext "use options=(force) in the PKGBUILD instead")" - elif [ "$arg" == "--quiet" -o "$arg" == "-q" ]; then - QUIET=1 - elif [ -z "$REPO_DB_FILE" ]; then - REPO_DB_FILE="$arg" - if [ -f "$REPO_DB_FILE" ]; then - if ! (bsdtar -tf "$REPO_DB_FILE" | grep -q "/desc"); then - error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" - exit 1 - fi - msg "$(gettext "Extracting database to a temporary location...")" - bsdtar -xf "$REPO_DB_FILE" -C "$gstmpdir" - elif [ "$cmd" == "repo-remove" ]; then - error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" - exit 1 - fi - else - if [ "$cmd" == "repo-add" ]; then - if [ -f "$arg" ]; then - if ! bsdtar -tf "$arg" .PKGINFO 2>&1 >/dev/null; then - error "$(gettext "'%s' is not a package file, skipping")" "$arg" - else - msg "$(gettext "Adding package '%s'")" "$arg" - - if db_write_entry "$arg"; then - success=1 - fi - fi + case "$arg" in + -q|--quiet) QUIET=1;; + + -f|--force) + warning "$(gettext "the -f and --force options are no longer recognized")" + msg2 "$(gettext "use options=(force) in the PKGBUILD instead")" + ;; + + *) + if [ -z "$REPO_DB_FILE" ]; then + REPO_DB_FILE="$arg" + check_repo_db else - error "$(gettext "Package '%s' not found.")" "$arg" + case "$cmd" in + repo-add) add $arg && success=1 ;; + repo-remove) remove $arg && success=1 ;; + esac fi - elif [ "$cmd" == "repo-remove" ]; then - msg "$(gettext "Searching for package '%s'...")" "$arg" - - if db_remove_entry "$arg"; then - success=1 - else - error "$(gettext "Package matching '%s' not found.")" "$arg" - fi - fi - fi + ;; + esac done -# if all operations were a success, re-zip database +# if at least one operation was a success, re-zip database if [ $success -eq 1 ]; then msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" -- cgit v1.2.3-54-g00ecf From 994804f20e51ea6dbbb236b919846a420b2d2369 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Thu, 26 Feb 2009 20:02:05 +0100 Subject: repo-add.sh.in : repo-remove improvements * report when a package entry to be removed is not found * backup and restore eventual "deltas" files * slight optimization when looking for an entry : only look at the entries starting with $pkgname Signed-off-by: Xavier Chantry --- scripts/repo-add.sh.in | 50 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index 4520dd4c..80a70de8 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -93,6 +93,20 @@ write_list_entry() { fi } +find_pkgentry() +{ + local pkgname=$1 + local pkgentry + for pkgentry in $gstmpdir/$pkgname*; do + name=${pkgentry##*/} + if [ "${name%-*-*}" = "$pkgname" ]; then + echo $pkgentry + return 0 + fi + done + return 1 +} + # write a delta entry to the pacman database # arg1 - path to delta db_write_delta() @@ -186,6 +200,9 @@ db_write_entry() mkdir "$pkgname-$pkgver" cd "$pkgname-$pkgver" + # restore an eventual deltas file + [ -f "../$pkgname.deltas" ] && mv "../$pkgname.deltas" deltas + # create desc entry msg2 "$(gettext "Creating 'desc' db entry...")" echo -e "%FILENAME%\n$(basename "$1")\n" >>desc @@ -256,18 +273,20 @@ db_write_entry() # remove existing entries from the DB # arg1 - package name db_remove_entry() { - pushd "$gstmpdir" 2>&1 >/dev/null - - # remove any other package in the DB with same name - local existing - for existing in *; do - if [ "${existing%-*-*}" = "$1" ]; then - msg2 "$(gettext "Removing existing package '%s'...")" "$existing" - rm -rf "$existing" + local pkgname=$1 + local notfound=1 + local pkgentry=$(find_pkgentry $pkgname) + while [ -n "$pkgentry" ]; do + notfound=0 + if [ -f "$pkgentry/deltas" ]; then + mv "$pkgentry/deltas" "$gstmpdir/$pkgname.deltas" fi + msg2 "$(gettext "Removing existing package '%s'...")" \ + "$(basename $pkgentry)" + rm -rf $pkgentry + pkgentry=$(find_pkgentry $pkgname) done - - popd 2>&1 >/dev/null + return $notfound } # end db_remove_entry check_repo_db() @@ -307,9 +326,16 @@ add() remove() { - msg "$(gettext "Searching for package '%s'...")" "$arg" + pkgname=$1 + msg "$(gettext "Searching for package '%s'...")" "$pkgname" - db_remove_entry "$arg" + if db_remove_entry "$pkgname"; then + rm -f "$gstmpdir/$pkgname.deltas" + return 0 + else + error "$(gettext "Package matching '%s' not found.")" "$pkgname" + return 1 + fi } # PROGRAM START -- cgit v1.2.3-54-g00ecf From 89685bdb2918c59a109cba2199ab25bc1bcfa784 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Thu, 26 Feb 2009 20:08:33 +0100 Subject: repo-add : drop delta support to rewrite it from scratch The current implementation has several problems : Wrong database format All the info is taken from the filename, which is a bit ugly It looks for .delta files in the current directory when adding a package, which is not very flexible Signed-off-by: Xavier Chantry --- scripts/repo-add.sh.in | 59 -------------------------------------------------- 1 file changed, 59 deletions(-) diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index 80a70de8..93416dc5 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -107,37 +107,6 @@ find_pkgentry() return 1 } -# write a delta entry to the pacman database -# arg1 - path to delta -db_write_delta() -{ - # blank out all variables - local deltafile="$1" - local filename=$(basename "$deltafile") - local deltavars pkgname fromver tover arch csize md5sum - - # format of the delta filename: - # (package)-(fromver)_to_(tover)-(arch).delta - deltavars=( $(echo "$filename" | sed -e 's/\(.*\)-\(.*-.*\)_to_\(.*-.*\)-\(.*\).delta/\1 \2 \3 \4/') ) - pkgname=${deltavars[0]} - fromver=${deltavars[1]} - tover=${deltavars[2]} - arch=${deltavars[3]} - - # get md5sum and size of delta - md5sum="$(openssl dgst -md5 "$deltafile" | awk '{print $NF}')" - csize=$(@SIZECMD@ "$deltafile") - - # ensure variables were found - if [ -z "$pkgname" -o -z "$fromver" -o -z "$tover" -o -z "$arch" ]; then - return 1 - fi - - # add the entry for this delta file - echo -e "$fromver $tover $csize $filename $md5sum" >>deltas -} # end db_write_delta - - # write an entry to the pacman database # arg1 - path to package db_write_entry() @@ -232,40 +201,12 @@ db_write_entry() write_list_entry "PROVIDES" "$_provides" "depends" write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - # create deltas entry if there are delta files - # Xav : why should deltas be in $startdir? - for delta in $startdir/$pkgname-*-*_to_*-*-$arch.delta; do - # This for loop also pulls in all files that start with the current package - # name and are followed by a -whatever. For instance, running this loop for - # gcc would also grab gcc-libs. To guard against this, compare the package - # name of the delta to the current package name. - local filename=$(basename "$delta") - local dpkgname="$(echo "$filename" | sed -e 's/\(.*\)-.*-.*_to_.*-.*-.*.delta/\1/')" - if [ "$pkgname" = "$dpkgname" -a -f "$delta" ]; then - # create deltas file if it does not already exist - if [ ! -f "deltas" ]; then - msg2 "$(gettext "Creating 'deltas' db entry...")" - echo -e "%DELTAS%" >>deltas - fi - - # write this delta entry - if db_write_delta "$delta"; then - msg2 "$(gettext "Added delta '%s'")" "$(basename "$delta")" - else - warning "$(gettext "Could not add delta '%s'")" "$(basename "$delta")" - fi - fi - done - # add the final newline - [ -f "deltas" ] && echo -e "" >>deltas - popd 2>&1 >/dev/null # preserve the modification time # Xav : what for? pkgdir="$gstmpdir/$pkgname-$pkgver" touch -r "$pkgfile" "$pkgdir/desc" "$pkgdir/depends" - [ -f "$pkgdir/deltas" ] && touch -r "$pkgfile" "$pkgdir/deltas" return 0 } # end db_write_entry -- cgit v1.2.3-54-g00ecf From bbcf96230ad16ad91f5ab8cb93c05fd15136d3e4 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Wed, 25 Feb 2009 19:06:16 +0100 Subject: repo-add : rewrite delta support Use the correct database format Use xdelta3 to get the source and destination files from the delta itself Allow delta files to be added with repo-add just like package files. delta files can also be removed with repo-remove. This is simply done by looking for a .delta extension in the arguments, and calling the appropriate db_write_delta or db_remove_delta functions. Example usage: repo-add repo/test.db.tar.gz repo/libx11-1.1.99.2-2-x86_64.pkg.tar.gz repo-add repo/test.db.tar.gz repo/libx11-1.1.5-2_to_1.1.99.2-2-x86_64.delta repo-remove repo/test.db.tar.gz libx11-1.1.5-2_to_1.1.99.2-2-x86_64.delta Signed-off-by: Xavier Chantry --- scripts/repo-add.sh.in | 102 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index 93416dc5..aa03ab78 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -57,8 +57,8 @@ error() { # print usage instructions usage() { printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "Usage: repo-add [-q] ...\n")" - printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" + printf "$(gettext "Usage: repo-add [-q] ...\n")" + printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" printf "$(gettext "\ repo-add will update a package database by reading a package file.\n\ Multiple packages to add can be specified on the command line.\n\n")" @@ -107,6 +107,73 @@ find_pkgentry() return 1 } +# Get the package name from the delta filename +get_delta_pkgname() { + local tmp + + tmp=${1##*/} + echo ${tmp%-*-*_to*} +} + +# write a delta entry +# arg1 - path to delta file +db_write_delta() +{ + deltafile="$1" + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [ -z "$pkgentry" ]; then + return 1 + fi + deltas="$pkgentry/deltas" + # create deltas file if it does not already exist + if [ ! -f "$deltas" ]; then + msg2 "$(gettext "Creating 'deltas' db entry...")" + echo -e "%DELTAS%" >>$deltas + fi + # get md5sum and compressed size of package + md5sum="$(openssl dgst -md5 "$deltafile" | awk '{print $NF}')" + csize=$(@SIZECMD@ "$deltafile") + + oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') + newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') + + if grep -q "$oldfile.*$newfile" $deltas; then + warning "$(gettext "An entry for '%s' already existed")" "$deltafile" + sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup + msg2 "$(gettext "Removing existing entry '%s'...")" "$deltafile" + fi + echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas + + return 0 +} # end db_write_delta + +# remove a delta entry +# arg1 - path to delta file +db_remove_delta() +{ + deltafile="$1" + filename=${deltafile##*/} + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [ -z "$pkgentry" ]; then + return 1 + fi + deltas="$pkgentry/deltas" + if [ ! -f "$deltas" ]; then + return 1 + fi + if grep -q "$filename" $deltas; then + sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup + msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" + return 0 + fi + + return 1 +} # end db_remove_delta + # write an entry to the pacman database # arg1 - path to package db_write_entry() @@ -222,7 +289,7 @@ db_remove_entry() { if [ -f "$pkgentry/deltas" ]; then mv "$pkgentry/deltas" "$gstmpdir/$pkgname.deltas" fi - msg2 "$(gettext "Removing existing package '%s'...")" \ + msg2 "$(gettext "Removing existing entry '%s'...")" \ "$(basename $pkgentry)" rm -rf $pkgentry pkgentry=$(find_pkgentry $pkgname) @@ -249,12 +316,26 @@ check_repo_db() add() { - pkgfile=$1 if [ ! -f "$1" ]; then - error "$(gettext "Package '%s' not found.")" "$pkgfile" + error "$(gettext "File '%s' not found.")" "$1" return 1 fi + if [ "${1##*.}" == "delta" ]; then + deltafile=$1 + msg "$(gettext "Adding delta '%s'")" "$deltafile" + if [ ! "$(type -p xdelta3)" ]; then + error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" + exit 1 + fi + if db_write_delta "$deltafile"; then + return 0 + else + return 1 + fi + fi + + pkgfile=$1 if ! bsdtar -tf "$pkgfile" .PKGINFO 2>&1 >/dev/null; then error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" return 1 @@ -267,6 +348,17 @@ add() remove() { + if [ "${1##*.}" == "delta" ]; then + deltafile=$1 + msg "$(gettext "Searching for delta '%s'...")" "$deltafile" + if db_remove_delta "$deltafile"; then + return 0 + else + error "$(gettext "Delta matching '%s' not found.")" "$deltafile" + return 1 + fi + fi + pkgname=$1 msg "$(gettext "Searching for package '%s'...")" "$pkgname" -- cgit v1.2.3-54-g00ecf From a556bc57fccf9b35f9b2eca08ddae595c3a75592 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Fri, 27 Feb 2009 15:18:10 +0100 Subject: repo-add : fix package variables declaration - arch was missing - backup is not used by repo-add. However makepkg still needs to put it in PKGINFO because pacman uses it - startdir is no longer used after the new delta implementation - the declaration of group, depend, backup, etc is not needed because these variables are always declared before being used : declare $var="$val" case "$var" in group) _groups="$_groups$group\n" ;; - reorder the variables declaration to follow the same order than they are written to the depends and desc file, for making future checks easier Signed-off-by: Xavier Chantry --- scripts/repo-add.sh.in | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index aa03ab78..5643bda3 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -180,10 +180,8 @@ db_write_entry() { # blank out all variables local pkgfile="$1" - local pkgname pkgver pkgdesc url builddate packager csize size \ - group depend backup license replaces provides conflict force \ - _groups _depends _backups _licenses _replaces _provides _conflicts \ - startdir optdepend _optdepends md5sum + local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager force \ + _groups _licenses _replaces _depends _conflicts _provides _optdepends local OLDIFS="$IFS" # IFS (field separator) is only the newline character @@ -200,12 +198,11 @@ db_write_entry() declare $var="$val" case "$var" in group) _groups="$_groups$group\n" ;; - depend) _depends="$_depends$depend\n" ;; - backup) _backups="$_backups$backup\n" ;; license) _licenses="$_licenses$license\n" ;; replaces) _replaces="$_replaces$replaces\n" ;; - provides) _provides="$_provides$provides\n" ;; + depend) _depends="$_depends$depend\n" ;; conflict) _conflicts="$_conflicts$conflict\n" ;; + provides) _provides="$_provides$provides\n" ;; optdepend) _optdepends="$_optdepends$optdepend\n" ;; esac done @@ -222,7 +219,6 @@ db_write_entry() return 1 fi - startdir=$(pwd) pushd "$gstmpdir" 2>&1 >/dev/null if [ -d "$pkgname-$pkgver" ]; then -- cgit v1.2.3-54-g00ecf From 59b4725bbb881140fea47dd2d1b09cef37e8a9dc Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Tue, 3 Mar 2009 17:05:14 +0100 Subject: repo-add : new locking system Weird things could happen if several repo-add were run concurrently on the same database. The introduced locking system will prevent this to happen. Signed-off-by: Xavier Chantry --- scripts/repo-add.sh.in | 58 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index 5643bda3..a760d1b1 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -28,7 +28,10 @@ myver='@PACKAGE_VERSION@' confdir='@sysconfdir@' QUIET=0 -REPO_DB_FILE="" +REPO_DB_FILE= +LOCKFILE= +CLEAN_LOCK=0 +startdir="$PWD" # ensure we have a sane umask set umask 0022 @@ -219,7 +222,7 @@ db_write_entry() return 1 fi - pushd "$gstmpdir" 2>&1 >/dev/null + cd "$gstmpdir" if [ -d "$pkgname-$pkgver" ]; then warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" @@ -264,7 +267,7 @@ db_write_entry() write_list_entry "PROVIDES" "$_provides" "depends" write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - popd 2>&1 >/dev/null + cd "$startdir" # preserve the modification time # Xav : what for? @@ -295,6 +298,15 @@ db_remove_entry() { check_repo_db() { + # check lock file + if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then + CLEAN_LOCK=1 + else + error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" + [ -f "$LOCKFILE" ] && error "$(gettext "Held by %s")" "$(cat $LOCKFILE)" + exit 1 + fi + if [ -f "$REPO_DB_FILE" ]; then if ! (bsdtar -tf "$REPO_DB_FILE" | grep -q "/desc"); then error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" @@ -367,6 +379,23 @@ remove() fi } +trap_exit() +{ + echo + error "$@" + exit 1 +} + +clean_up() { + local exit_code=$? + + cd "$startdir" + [ -d "$gstmpdir" ] && rm -rf "$gstmpdir" + [ $CLEAN_LOCK -eq 1 -a -f "$LOCKFILE" ] && rm -f "$LOCKFILE" + + exit $exit_code +} + # PROGRAM START # determine whether we have gettext; make it a no-op if we do not @@ -387,11 +416,6 @@ if [ $# -lt 2 ]; then exit 1 fi -# main routine -gstmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ - error "$(gettext "Cannot create temp directory for database building.")"; \ - exit 1) - # figure out what program we are cmd="$(basename $0)" if [ "$cmd" != "repo-add" -a "$cmd" != "repo-remove" ]; then @@ -399,6 +423,15 @@ if [ "$cmd" != "repo-add" -a "$cmd" != "repo-remove" ]; then exit 1 fi +gstmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ + error "$(gettext "Cannot create temp directory for database building.")"; \ + exit 1) + +trap 'clean_up' EXIT +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR + success=0 # parse arguments for arg in "$@"; do @@ -413,6 +446,7 @@ for arg in "$@"; do *) if [ -z "$REPO_DB_FILE" ]; then REPO_DB_FILE="$arg" + LOCKFILE="$REPO_DB_FILE.lck" check_repo_db else case "$cmd" in @@ -437,14 +471,14 @@ if [ $success -eq 1 ]; then filename=$(basename "$REPO_DB_FILE") - pushd "$gstmpdir" 2>&1 >/dev/null + cd "$gstmpdir" if [ -n "$(ls)" ]; then bsdtar -c${TAR_OPT}f "$filename" * else # the database will be moved to .old below, and there will be no new one to replace it error "$(gettext "All packages have been removed from the database. Deleting '%s'.")" "$REPO_DB_FILE" fi - popd 2>&1 >/dev/null + cd "$startdir" [ -f "$REPO_DB_FILE" ] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" [ -f "$gstmpdir/$filename" ] && mv "$gstmpdir/$filename" "$REPO_DB_FILE" @@ -452,7 +486,5 @@ else msg "$(gettext "No packages modified, nothing to do.")" fi -# remove the temp directory used to unzip -[ -d "$gstmpdir" ] && rm -rf "$gstmpdir" - +exit 0 # vim: set ts=2 sw=2 noet: -- cgit v1.2.3-54-g00ecf From 91d43ba4b44e0827c6d9b66c70485672953f5a6d Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Tue, 3 Mar 2009 17:10:26 +0100 Subject: repo-add : don't change the modification time on depends and desc The modification time on depends and desc file were changed to match the modification time of the package file. I don't see why and we are actualling losing information here. If we want to know the date of the package file, we can just look inside the depends file. If we want to know when the entry was created, we should not alter the modification time of depends and desc. Besides, this had the non-obvious and undocumented side effect that the depends file was always created, even if it was empty. And pacman actually does require that. So I added a "touch depends" to always create the file. Signed-off-by: Xavier Chantry --- scripts/repo-add.sh.in | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index a760d1b1..186d9fb8 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -262,6 +262,8 @@ db_write_entry() # create depends entry msg2 "$(gettext "Creating 'depends' db entry...")" + # create the file even if it will remain empty + touch "depends" write_list_entry "DEPENDS" "$_depends" "depends" write_list_entry "CONFLICTS" "$_conflicts" "depends" write_list_entry "PROVIDES" "$_provides" "depends" @@ -269,11 +271,6 @@ db_write_entry() cd "$startdir" - # preserve the modification time - # Xav : what for? - pkgdir="$gstmpdir/$pkgname-$pkgver" - touch -r "$pkgfile" "$pkgdir/desc" "$pkgdir/depends" - return 0 } # end db_write_entry -- cgit v1.2.3-54-g00ecf From 0c614c181efec31ea8d3948745e5746b642cb10c Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Tue, 3 Mar 2009 17:13:52 +0100 Subject: repo-add : rename gstmpdir to tmpdir What does gstmpdir mean? Signed-off-by: Xavier Chantry --- scripts/repo-add.sh.in | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index 186d9fb8..2390a92c 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -100,7 +100,7 @@ find_pkgentry() { local pkgname=$1 local pkgentry - for pkgentry in $gstmpdir/$pkgname*; do + for pkgentry in $tmpdir/$pkgname*; do name=${pkgentry##*/} if [ "${name%-*-*}" = "$pkgname" ]; then echo $pkgentry @@ -222,7 +222,7 @@ db_write_entry() return 1 fi - cd "$gstmpdir" + cd "$tmpdir" if [ -d "$pkgname-$pkgver" ]; then warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" @@ -283,7 +283,7 @@ db_remove_entry() { while [ -n "$pkgentry" ]; do notfound=0 if [ -f "$pkgentry/deltas" ]; then - mv "$pkgentry/deltas" "$gstmpdir/$pkgname.deltas" + mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" fi msg2 "$(gettext "Removing existing entry '%s'...")" \ "$(basename $pkgentry)" @@ -310,7 +310,7 @@ check_repo_db() exit 1 fi msg "$(gettext "Extracting database to a temporary location...")" - bsdtar -xf "$REPO_DB_FILE" -C "$gstmpdir" + bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" else if [ "$cmd" == "repo-remove" ]; then error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" @@ -368,7 +368,7 @@ remove() msg "$(gettext "Searching for package '%s'...")" "$pkgname" if db_remove_entry "$pkgname"; then - rm -f "$gstmpdir/$pkgname.deltas" + rm -f "$tmpdir/$pkgname.deltas" return 0 else error "$(gettext "Package matching '%s' not found.")" "$pkgname" @@ -387,7 +387,7 @@ clean_up() { local exit_code=$? cd "$startdir" - [ -d "$gstmpdir" ] && rm -rf "$gstmpdir" + [ -d "$tmpdir" ] && rm -rf "$tmpdir" [ $CLEAN_LOCK -eq 1 -a -f "$LOCKFILE" ] && rm -f "$LOCKFILE" exit $exit_code @@ -420,7 +420,7 @@ if [ "$cmd" != "repo-add" -a "$cmd" != "repo-remove" ]; then exit 1 fi -gstmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ +tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ error "$(gettext "Cannot create temp directory for database building.")"; \ exit 1) @@ -468,7 +468,7 @@ if [ $success -eq 1 ]; then filename=$(basename "$REPO_DB_FILE") - cd "$gstmpdir" + cd "$tmpdir" if [ -n "$(ls)" ]; then bsdtar -c${TAR_OPT}f "$filename" * else @@ -478,7 +478,7 @@ if [ $success -eq 1 ]; then cd "$startdir" [ -f "$REPO_DB_FILE" ] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" - [ -f "$gstmpdir/$filename" ] && mv "$gstmpdir/$filename" "$REPO_DB_FILE" + [ -f "$tmpdir/$filename" ] && mv "$tmpdir/$filename" "$REPO_DB_FILE" else msg "$(gettext "No packages modified, nothing to do.")" fi -- cgit v1.2.3-54-g00ecf