From 49a0260b06f9d24c790631cb4e2a4f63fce3a8c3 Mon Sep 17 00:00:00 2001 From: Parabola Date: Fri, 17 Sep 2010 09:55:19 -0700 Subject: Upload version --- TODO | 0 __init__.py | 0 changelog | 2 + get_license.sh | 55 ++++++++++++++ pato2.py | 232 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repo-state | 11 +++ 6 files changed, 300 insertions(+) create mode 100644 TODO create mode 100644 __init__.py create mode 100644 changelog create mode 100755 get_license.sh create mode 100644 pato2.py create mode 100755 repo-state diff --git a/TODO b/TODO new file mode 100644 index 0000000..e69de29 diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/changelog b/changelog new file mode 100644 index 0000000..0e89f2f --- /dev/null +++ b/changelog @@ -0,0 +1,2 @@ +repo-maintainer 1.0 + * Known working version diff --git a/get_license.sh b/get_license.sh new file mode 100755 index 0000000..a7241a1 --- /dev/null +++ b/get_license.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# -*- coding: utf-8 -*- + + # get_license.sh + # Copyright 2010 Joshua Ismael Haase Hernández + + # ---------- GNU General Public License 3 ---------- + + # This file is part of Parabola. + + # Parabola is free software: you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by + # the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + + # Parabola is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + + # You should have received a copy of the GNU General Public License + # along with Parabola. If not, see . + +docs="/home/parabolavnx/parabolagnulinux.org/docs" +repo="/home/parabolavnx/parabolagnulinux.org/repo" +dir="$docs/pending-licenses" + +echo "Cleaning $dir" +rm -rf $dir/* + +tempdir=$(mktemp -d) +cd $tempdir + +a=($(cut -d: -f1 $docs/pending*.txt)) +echo ${a[@]} + +for x in ${a[@]}; do + b=( $(ls $repo/*/os/*/$x*) ) + for y in ${b[@]}; do + echo "chmod +r $y" + chmod +r $y + echo "tar -xf $y usr/share/licenses" + bsdtar -xf $y usr/share/licenses + echo "chmod -r $y" + chmod -r $y + done +done + +mv usr/share/licenses/* $dir + +cd + +rm -rf $tempdir + +exit 0 \ No newline at end of file diff --git a/pato2.py b/pato2.py new file mode 100644 index 0000000..6d8a981 --- /dev/null +++ b/pato2.py @@ -0,0 +1,232 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" + parabola.py + Copyright 2009 Rafik Mas'ad + Copyright 2010 Joshua Ismael Haase Hernández + + ---------- GNU General Public License 3 ---------- + + This file is part of Parabola. + + Parabola is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Parabola is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Parabola. If not, see . + + +""" + +import tarfile, commands +from glob import glob +from user import home +from os.path import isdir, isfile, realpath +# ---------- Config Variables Start Here ---------- # + +time__ = commands.getoutput("date +%Y%m%d-%H:%M") + +# Mirror Parameters +mirror = "mirrors.kernel.org" +mirrorpath = "::mirrors/archlinux" + +# Directories and files +## Optionals +path = home + "/parabolagnulinux.org" +docs = path + "/docs" +logdir = path + "/log" +## Must be defined +logname= logdir + "/" + time__ + "-repo-maintainer.log" +repodir= path + "/repo" +tmp = home + "/tmp" +archdb = tmp + "/db" +emptydb= path + "/files/repo-empty.db.tar.gz" + +# Repo, arch, and other folders to use for repo +repo_list = ("core", "extra", "community","multilib") +dir_list = ("pool",) +arch_list = ("i686", "x86_64") +other = ("any",) + +# Output +output = True +verbose = False + +# Files +blacklist = docs + "/blacklist.txt" +whitelist = docs + "/whitelist.txt" +pending = docs + "/pending" +rsyncBlacklist = docs + "/rsyncBlacklist" + +# ---------- Config Variables End Here---------- # + +def printf(text,output_=output): + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(logname, 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output_: print (str(text) + "\n") + +def listado(filename_): + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename_,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0] for pkg in lista if pkg] + +def db(repo_,arch_): + """Construye un nombre para sincronizar una base de datos.""" + return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) + +def packages(repo_, arch_, expr="*"): + """ Get packages on a repo, arch folder """ + return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) + +def sync_all_repo(verbose_=verbose): + folders = ",".join(repo_list + dir_list) + cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + +def get_from_desc(desc, var,db_tar_file=False): + """ Get a var from desc file """ + desc = desc.split("\n") + return desc[desc.index(var)+1] + +def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): + """ Makes a list of package name, file and license """ + info=list() + # Extract DB tar.gz + commands.getoutput("mkdir -p " + archdb) + if not db_tar_file: + db_tar_file = repodir + db(repo_,arch_) + if isfile(db_tar_file): + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + pkg_desc_file = open(dir_ + "/desc", "r") + desc = pkg_desc_file.read() + pkg_desc_file.close() + info.append(( get_from_desc(desc,"%NAME%"), + dir_.split("/")[-1], + get_from_desc(desc,"%LICENSE%") )) + if verbose_: printf(info) + commands.getoutput("rm -r %s/*" % archdb) + return tuple(info) + +def make_pending(repo_,arch_,info_): + """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" + search = tuple( listado(blacklist) + listado (whitelist) ) + if verbose: printf("blaclist + whitelist= " + str(search) ) + lista_=list() + for (name,pkg_,license_) in info_: + if "custom" in license_: + if name not in search: + lista_.append( (name, license_ ) ) + elif not name: + printf( pkg_ + " package has no %NAME% attibute " ) + if verbose: printf( lista_ ) + a=open( pending + "-" + repo_ + ".txt", "w" ).write( + "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + +def remove_from_blacklist(repo_,arch_,info_,blacklist_): + """ Check the blacklist and remove packages on the db""" + lista_=list() + pack_=list() + for (name_, pkg_, license_) in info_: + if name_ in blacklist_: + lista_.append(name_) + for p in packages(repo_,arch_,pkg_ + "*"): + pack_.append(p) + if lista_: + lista_=" ".join(lista_) + com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ + printf(com_) + a = commands.getoutput(com_) + if verbose: printf(a) + if pack_: + pack_=" ".join(pack_) + com_="chmod a-r " + pack_ + printf(com_) + a=commands.getoutput(com_) + if verbose: printf(a) + +def link(repo_,arch_,file_): + """ Makes a link in the repo for the package """ + cmd_="ln -sf " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + a=commands.getoutput(cmd_) + if verbose: + printf(cmd_ + a) + +def add_free_repo(verbose_=verbose): + for repo_ in repo_list: + for arch_ in arch_list: + lista_=list() + for file_ in glob(repodir + "/free/" + repo_ + "/os/" + arch_ + "/*"): + lista_.append(file_) + link(repo_,arch_,file_) + for dir_ in other: + for file_ in glob(repodir + "/free/" + repo_ + "/os/" + dir_ + "/*"): + lista_.append(file_) + link(repo_,arch_,file_) + if lista_: + lista_=" ".join(lista_) + if verbose: printf(lista_) + cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose: printf(a) + +def get_licenses(verbose_=verbose): + """ Extract the license from packages in repo_,arch_ and in pending_ file""" + cmd_=home + "/usr/bin/get_license.sh" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + +if __name__ == "__main__": + from time import time + start_time = time() + def minute(): + return str(round((time() - start_time)/60, 1)) + + printf(" Cleaning %s folder " % (tmp) ) + commands.getoutput("rm -r %s/*" % tmp) + printf(" Syncing repo") + sync_all_repo(True) + + printf(" Updating databases and pending files lists: minute %s \n" % minute() ) + for repo in repo_list: + for arch in arch_list: + printf( "\n" + repo + "-" + arch + "\n" ) + printf( "Get info: minute %s " % minute() ) + info=get_info(repo,arch) + printf( "Make pending: minute %s" % minute() ) + make_pending(repo,arch,info) + printf( "Update DB: minute %s" % minute() ) + remove_from_blacklist( + repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) + + printf("Adding Parabola Packages: minute %s\n" % minute() ) + add_free_repo(True) + + printf("Extracting licenses in pending: minute %s" % minute() ) + get_licenses() + + printf("\n\nDelay: %s minutes \n" % minute()) + diff --git a/repo-state b/repo-state new file mode 100755 index 0000000..319db83 --- /dev/null +++ b/repo-state @@ -0,0 +1,11 @@ +#!/bin/sh + +a=$(date +%Y%m%d+%H%M) +mkdir -p $a +cd $a + +. /home/joshpar/programas/arch2parabola/repo-list-diff +# scp parabolavnx@parabolagnulinux.org:~/tmp/rsyncBlacklist.txt ./ + +cd .. +exit 0 \ No newline at end of file -- cgit v1.2.3 From 847679d7a5e594efce5f8df73ca6751fab9ce591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 17 Sep 2010 12:34:02 -0500 Subject: Added TODO list --- TODO | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/TODO b/TODO index e69de29..11ef51a 100644 --- a/TODO +++ b/TODO @@ -0,0 +1,6 @@ +* Test Suite + + - Review all repo efectively + - Remove all blacklisted packages + - Get pending list right + - Extract licenses all right \ No newline at end of file -- cgit v1.2.3 From c6787b4d56fc585aca5b5f5422aea0f2c85a5db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 28 Sep 2010 21:21:05 -0500 Subject: Added sources to the repo syncing command --- .gitignore | 1 + pato2.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e4e5f6c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ \ No newline at end of file diff --git a/pato2.py b/pato2.py index 6d8a981..df81b73 100644 --- a/pato2.py +++ b/pato2.py @@ -51,7 +51,7 @@ emptydb= path + "/files/repo-empty.db.tar.gz" # Repo, arch, and other folders to use for repo repo_list = ("core", "extra", "community","multilib") -dir_list = ("pool",) +dir_list = ("pool","sources") arch_list = ("i686", "x86_64") other = ("any",) -- cgit v1.2.3 From 6c1729bf9d94407b36ac5717c254dacf8dffef3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 12 Oct 2010 11:51:43 -0500 Subject: If it can't open db_tar_file, print a warning and continue. --- pato2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index df81b73..58e03f8 100644 --- a/pato2.py +++ b/pato2.py @@ -109,7 +109,11 @@ def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): if not db_tar_file: db_tar_file = repodir + db(repo_,arch_) if isfile(db_tar_file): - db_open_tar = tarfile.open(db_tar_file, 'r:gz') + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s" % db_tar_file) + return(tuple()) else: printf("No db_file %s" % db_tar_file) return(tuple()) -- cgit v1.2.3 From 473de5be358e3bc1c0c490622e9b2c47d2bfbdf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 12 Oct 2010 12:33:33 -0500 Subject: Deleted unused variables --- pato2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pato2.py b/pato2.py index 58e03f8..78d8c55 100644 --- a/pato2.py +++ b/pato2.py @@ -47,7 +47,6 @@ logname= logdir + "/" + time__ + "-repo-maintainer.log" repodir= path + "/repo" tmp = home + "/tmp" archdb = tmp + "/db" -emptydb= path + "/files/repo-empty.db.tar.gz" # Repo, arch, and other folders to use for repo repo_list = ("core", "extra", "community","multilib") -- cgit v1.2.3 From b8619985289cde1591aa40c7df2c948fdbb8c30a Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 31 Jan 2011 09:15:56 -0800 Subject: changed mirror --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 78d8c55..cf40809 100644 --- a/pato2.py +++ b/pato2.py @@ -34,7 +34,7 @@ from os.path import isdir, isfile, realpath time__ = commands.getoutput("date +%Y%m%d-%H:%M") # Mirror Parameters -mirror = "mirrors.kernel.org" +mirror = "mirrors.eu.kernel.org" mirrorpath = "::mirrors/archlinux" # Directories and files @@ -49,7 +49,7 @@ tmp = home + "/tmp" archdb = tmp + "/db" # Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community","multilib") +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") dir_list = ("pool","sources") arch_list = ("i686", "x86_64") other = ("any",) @@ -90,7 +90,7 @@ def packages(repo_, arch_, expr="*"): def sync_all_repo(verbose_=verbose): folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir + cmd_ = "rsync -av --progress --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) -- cgit v1.2.3 From a5872f3b259b97125089866169d06aa9a4d78e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 7 Feb 2011 12:30:30 -0600 Subject: moved config out from pato2 --- config.py | 32 ++++++++++++++++++++++++++++++++ pato2.py | 37 +------------------------------------ 2 files changed, 33 insertions(+), 36 deletions(-) create mode 100644 config.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..e2fc350 --- /dev/null +++ b/config.py @@ -0,0 +1,32 @@ +time__ = commands.getoutput("date +%Y%m%d-%H:%M") + +# Mirror Parameters +mirror = "mirrors.eu.kernel.org" +mirrorpath = "::mirrors/archlinux" + +# Directories and files +## Optionals +path = home + "/parabolagnulinux.org" +docs = path + "/docs" +logdir = path + "/log" +## Must be defined +logname= logdir + "/" + time__ + "-repo-maintainer.log" +repodir= path + "/repo" +tmp = home + "/tmp" +archdb = tmp + "/db" + +# Repo, arch, and other folders to use for repo +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") +dir_list = ("pool","sources") +arch_list = ("i686", "x86_64") +other = ("any",) + +# Output +output = True +verbose = False + +# Files +blacklist = docs + "/blacklist.txt" +whitelist = docs + "/whitelist.txt" +pending = docs + "/pending" +rsyncBlacklist = docs + "/rsyncBlacklist" diff --git a/pato2.py b/pato2.py index cf40809..3ed57b2 100644 --- a/pato2.py +++ b/pato2.py @@ -24,47 +24,12 @@ """ +from repo_maintainer.config import * import tarfile, commands from glob import glob from user import home from os.path import isdir, isfile, realpath -# ---------- Config Variables Start Here ---------- # - -time__ = commands.getoutput("date +%Y%m%d-%H:%M") - -# Mirror Parameters -mirror = "mirrors.eu.kernel.org" -mirrorpath = "::mirrors/archlinux" - -# Directories and files -## Optionals -path = home + "/parabolagnulinux.org" -docs = path + "/docs" -logdir = path + "/log" -## Must be defined -logname= logdir + "/" + time__ + "-repo-maintainer.log" -repodir= path + "/repo" -tmp = home + "/tmp" -archdb = tmp + "/db" - -# Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") -other = ("any",) - -# Output -output = True -verbose = False - -# Files -blacklist = docs + "/blacklist.txt" -whitelist = docs + "/whitelist.txt" -pending = docs + "/pending" -rsyncBlacklist = docs + "/rsyncBlacklist" - -# ---------- Config Variables End Here---------- # def printf(text,output_=output): """Guarda el texto en la variable log y puede imprimir en pantalla.""" -- cgit v1.2.3 From 44dc101393495d876a4fab40689c1d6c8b307c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 7 Feb 2011 12:32:16 -0600 Subject: Added headers to config.py --- config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.py b/config.py index e2fc350..df837d4 100644 --- a/config.py +++ b/config.py @@ -1,3 +1,6 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + time__ = commands.getoutput("date +%Y%m%d-%H:%M") # Mirror Parameters -- cgit v1.2.3 From 937bf24bacdd29629b5efd2c949c0a6e4d17b163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 7 Feb 2011 17:38:33 -0600 Subject: Making hardlinks instead of symbolic --- config.py | 2 ++ pato2.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index df837d4..8a55cf7 100644 --- a/config.py +++ b/config.py @@ -8,10 +8,12 @@ mirror = "mirrors.eu.kernel.org" mirrorpath = "::mirrors/archlinux" # Directories and files + ## Optionals path = home + "/parabolagnulinux.org" docs = path + "/docs" logdir = path + "/log" + ## Must be defined logname= logdir + "/" + time__ + "-repo-maintainer.log" repodir= path + "/repo" diff --git a/pato2.py b/pato2.py index 3ed57b2..cd08ca0 100644 --- a/pato2.py +++ b/pato2.py @@ -24,7 +24,7 @@ """ -from repo_maintainer.config import * +from repm.config import * import tarfile, commands from glob import glob @@ -136,7 +136,7 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ - cmd_="ln -sf " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ a=commands.getoutput(cmd_) if verbose: printf(cmd_ + a) -- cgit v1.2.3 From 100970b35ed77a8f617a7071aba2618e038923ba Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Mon, 7 Feb 2011 16:51:06 -0800 Subject: Use free repo on new location --- pato2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index cf40809..9cbd3cb 100644 --- a/pato2.py +++ b/pato2.py @@ -40,6 +40,7 @@ mirrorpath = "::mirrors/archlinux" # Directories and files ## Optionals path = home + "/parabolagnulinux.org" +free_path = path + "/free/" docs = path + "/docs" logdir = path + "/log" ## Must be defined @@ -56,7 +57,7 @@ other = ("any",) # Output output = True -verbose = False +verbose = True # Files blacklist = docs + "/blacklist.txt" @@ -180,11 +181,11 @@ def add_free_repo(verbose_=verbose): for repo_ in repo_list: for arch_ in arch_list: lista_=list() - for file_ in glob(repodir + "/free/" + repo_ + "/os/" + arch_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*"): lista_.append(file_) link(repo_,arch_,file_) for dir_ in other: - for file_ in glob(repodir + "/free/" + repo_ + "/os/" + dir_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*"): lista_.append(file_) link(repo_,arch_,file_) if lista_: -- cgit v1.2.3 From 8c4813edd80c08e03c70ada2cb46a3597e493aaf Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 7 Feb 2011 16:54:47 -0800 Subject: 'Configuration out of pato' --- pato2.py | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/pato2.py b/pato2.py index 9cbd3cb..dd32403 100644 --- a/pato2.py +++ b/pato2.py @@ -24,48 +24,12 @@ """ +from repm.config import * import tarfile, commands from glob import glob from user import home from os.path import isdir, isfile, realpath -# ---------- Config Variables Start Here ---------- # - -time__ = commands.getoutput("date +%Y%m%d-%H:%M") - -# Mirror Parameters -mirror = "mirrors.eu.kernel.org" -mirrorpath = "::mirrors/archlinux" - -# Directories and files -## Optionals -path = home + "/parabolagnulinux.org" -free_path = path + "/free/" -docs = path + "/docs" -logdir = path + "/log" -## Must be defined -logname= logdir + "/" + time__ + "-repo-maintainer.log" -repodir= path + "/repo" -tmp = home + "/tmp" -archdb = tmp + "/db" - -# Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") -other = ("any",) - -# Output -output = True -verbose = True - -# Files -blacklist = docs + "/blacklist.txt" -whitelist = docs + "/whitelist.txt" -pending = docs + "/pending" -rsyncBlacklist = docs + "/rsyncBlacklist" - -# ---------- Config Variables End Here---------- # def printf(text,output_=output): """Guarda el texto en la variable log y puede imprimir en pantalla.""" @@ -172,7 +136,7 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ - cmd_="ln -sf " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ a=commands.getoutput(cmd_) if verbose: printf(cmd_ + a) -- cgit v1.2.3 From 317c0b412642b5268e8408f9d87e19261370cd32 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 7 Feb 2011 18:25:37 -0800 Subject: fixed errors --- config.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 config.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..9889c3f --- /dev/null +++ b/config.py @@ -0,0 +1,41 @@ + +#!/usr/bin/python +# -*- coding: utf-8 -*- +from user import home +import commands + +time__ = commands.getoutput("date +%Y%m%d-%H:%M") + +# Mirror Parameters +mirror = "mirrors.eu.kernel.org" +mirrorpath = "::mirrors/archlinux" + +# Directories and files + +## Optionals +path = home + "/parabolagnulinux.org" +docs = path + "/docs" +logdir = path + "/log" + +## Must be defined +logname= logdir + "/" + time__ + "-repo-maintainer.log" +repodir= path + "/repo" +tmp = home + "/tmp" +archdb = tmp + "/db" +free_path= path + "/free/" + +# Repo, arch, and other folders to use for repo +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") +dir_list = ("pool","sources") +arch_list = ("i686", "x86_64") +other = ("any",) + +# Output +output = True +verbose = False + +# Files +blacklist = docs + "/blacklist.txt" +whitelist = docs + "/whitelist.txt" +pending = docs + "/pending" +rsyncBlacklist = docs + "/rsyncBlacklist" -- cgit v1.2.3 From f837844242142599d1dee84309468624f6dd2048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 00:31:56 -0600 Subject: Added classes and exceptions --- config.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/config.py b/config.py index 223257d..6dcd348 100644 --- a/config.py +++ b/config.py @@ -39,3 +39,27 @@ blacklist = docs + "/blacklist.txt" whitelist = docs + "/whitelist.txt" pending = docs + "/pending" rsyncBlacklist = docs + "/rsyncBlacklist" + +# Classes and Exceptions +class NonValidFile(ValueError): pass +class NonValidDir(ValueError): pass +class NonValidCommand(ValueError): pass + +class Package: + """ An object that has information about a package. """ + package_info={ "name" : False, + "version" : False, + "arch" : False, + "license" : False, + "location": False} + + def __setitem__(self, key, item): + return self.package_info.__setitem__(key, item) + + def __getitem__(self, key): + return self.package_info.__getitem__(key) + + def __unicode__(self): + return str(self.package_info) + + -- cgit v1.2.3 From 039c18a9ab6259a13285a22d84390bba2c35642e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 00:37:29 -0600 Subject: Removed extra import commands: * commands and home were loaded on repm.config * realpath is not used --- pato2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index dd32403..236f2ea 100644 --- a/pato2.py +++ b/pato2.py @@ -26,10 +26,9 @@ """ from repm.config import * -import tarfile, commands +import tarfile from glob import glob -from user import home -from os.path import isdir, isfile, realpath +from os.path import isdir, isfile def printf(text,output_=output): """Guarda el texto en la variable log y puede imprimir en pantalla.""" -- cgit v1.2.3 From 80402467cf38f32373ac14d992d2daee4736ad82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 00:42:33 -0600 Subject: Added primitive testcase --- test/directory_list | 2 ++ test/link_list | 2 ++ test/package_list | 3 +++ test/test1.py | 22 ++++++++++++++++++++++ 4 files changed, 29 insertions(+) create mode 100644 test/directory_list create mode 100644 test/link_list create mode 100644 test/package_list create mode 100644 test/test1.py diff --git a/test/directory_list b/test/directory_list new file mode 100644 index 0000000..c0f7f47 --- /dev/null +++ b/test/directory_list @@ -0,0 +1,2 @@ +drwxrwxr-x 15 2010/09/11 11:28:50 community-staging +drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os \ No newline at end of file diff --git a/test/link_list b/test/link_list new file mode 100644 index 0000000..d015364 --- /dev/null +++ b/test/link_list @@ -0,0 +1,2 @@ +lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz +lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz \ No newline at end of file diff --git a/test/package_list b/test/package_list new file mode 100644 index 0000000..6ffa1de --- /dev/null +++ b/test/package_list @@ -0,0 +1,3 @@ +-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz +-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz +-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz \ No newline at end of file diff --git a/test/test1.py b/test/test1.py new file mode 100644 index 0000000..8e5cd1e --- /dev/null +++ b/test/test1.py @@ -0,0 +1,22 @@ +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +import repm.filter +import unittest +import commands + +class KnownValues(unittest.TestCase): + + def testDirectoryOutput(self): + """get_file_list_from_rsync_output should ignore directories""" + output=commands.getoutput("cat ./directory_list") + result=get_file_list_from_rsync_output(output) + self.assertEqual(tuple(), result) + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3 From 0bdbdeaf186a9ab73f596b60748b6ae8a25ebaeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 01:18:18 -0600 Subject: Added some testcases. --- test/test1.py | 48 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/test/test1.py b/test/test1.py index 8e5cd1e..a42e9a5 100644 --- a/test/test1.py +++ b/test/test1.py @@ -6,17 +6,59 @@ __date__ = "$Date: 2011/02/08 $" __copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" __license__ = "GPL3+" -import repm.filter +from repm.config import * +from repm.filter import * import unittest -import commands class KnownValues(unittest.TestCase): + directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", + "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") + # (output, name, version, arch) + link_list=( + ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686"), + ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686"), + ) + package_list=( + ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", + "abuse","0.7.1","x86_64"), + ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", + "acetoneiso2","2.3","i686"), + ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", + "acetoneiso2","2.3","x86_64") + ) + + def generate_results(example_tuple): + a=list() + for output, name, version, arch in example_tuple: + pkg=Packages() + pkg["name"] = name + pkg["version"] = version + pkg["arch"] = arch + a.append(pkg) + return tuple(a) def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" - output=commands.getoutput("cat ./directory_list") + rsync_out="\n".join(directory_list) result=get_file_list_from_rsync_output(output) self.assertEqual(tuple(), result) + def testLinkOutput(self): + """get_file_list_from_rsync_output should make a Package Object + from links """ + correct_result=generate_results(link_list) + rsync_out="\n".join([a for a,b,c,d in link_list]) + result=get_file_list_from_rsync_output(rsync_out) + self.assertEqual(correct_result, result) + + + def testPackageOutput(self): + """get_file_list_from_rsync_output should make a Package Object + from links """ + correct_result=generate_results(package_list) + rsync_out="\n".join([a for a,b,c,d in package_list]) + result=get_file_list_from_rsync_output(rsync_out) + self.assertEqual(correct_result, result) + if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From 3633ea3c4ec93436b2344dba79b06ff00c7ae528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 01:30:35 -0600 Subject: Corrected first testcases. --- test/test1.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test1.py b/test/test1.py index a42e9a5..495ba05 100644 --- a/test/test1.py +++ b/test/test1.py @@ -1,3 +1,4 @@ +# -*- encoding: utf-8 -*- """ """ __author__ = "Joshua Ismael Haase Hernández " @@ -27,10 +28,10 @@ class KnownValues(unittest.TestCase): "acetoneiso2","2.3","x86_64") ) - def generate_results(example_tuple): + def generate_results(self, example_tuple): a=list() for output, name, version, arch in example_tuple: - pkg=Packages() + pkg=Package() pkg["name"] = name pkg["version"] = version pkg["arch"] = arch @@ -39,24 +40,23 @@ class KnownValues(unittest.TestCase): def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" - rsync_out="\n".join(directory_list) + rsync_out="\n".join(self.directory_list) result=get_file_list_from_rsync_output(output) self.assertEqual(tuple(), result) def testLinkOutput(self): """get_file_list_from_rsync_output should make a Package Object from links """ - correct_result=generate_results(link_list) - rsync_out="\n".join([a for a,b,c,d in link_list]) + correct_result=self.generate_results(self.link_list) + rsync_out="\n".join([a for a,b,c,d in self.link_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) - def testPackageOutput(self): """get_file_list_from_rsync_output should make a Package Object from links """ - correct_result=generate_results(package_list) - rsync_out="\n".join([a for a,b,c,d in package_list]) + correct_result=self.generate_results(self.package_list) + rsync_out="\n".join([a for a,b,c,d in self.package_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) -- cgit v1.2.3 From 61f5ecfa6b57f31c14430a7de5099640fcbe3525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 03:13:39 -0600 Subject: * Updated Package class * Corrected test1.py -> for get_package_list_from_rsync_output --- config.py | 1 + test/test1.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/config.py b/config.py index 6dcd348..a7087c4 100644 --- a/config.py +++ b/config.py @@ -49,6 +49,7 @@ class Package: """ An object that has information about a package. """ package_info={ "name" : False, "version" : False, + "release" : False, "arch" : False, "license" : False, "location": False} diff --git a/test/test1.py b/test/test1.py index 495ba05..db87f51 100644 --- a/test/test1.py +++ b/test/test1.py @@ -14,41 +14,43 @@ import unittest class KnownValues(unittest.TestCase): directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") - # (output, name, version, arch) + # (output, name, version, arch, release, location) link_list=( - ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686"), - ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686"), + ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), + ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), ) package_list=( ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", - "abuse","0.7.1","x86_64"), + "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", - "acetoneiso2","2.3","i686"), + "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", - "acetoneiso2","2.3","x86_64") + "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") ) def generate_results(self, example_tuple): a=list() - for output, name, version, arch in example_tuple: + for output, name, version, arch, release, location in example_tuple: pkg=Package() pkg["name"] = name pkg["version"] = version pkg["arch"] = arch + pkg["release"] = release + pkg["location"] = location a.append(pkg) return tuple(a) def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" rsync_out="\n".join(self.directory_list) - result=get_file_list_from_rsync_output(output) + result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(tuple(), result) def testLinkOutput(self): """get_file_list_from_rsync_output should make a Package Object from links """ correct_result=self.generate_results(self.link_list) - rsync_out="\n".join([a for a,b,c,d in self.link_list]) + rsync_out="\n".join([a for a,b,c,d,e,f in self.link_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) @@ -56,7 +58,7 @@ class KnownValues(unittest.TestCase): """get_file_list_from_rsync_output should make a Package Object from links """ correct_result=self.generate_results(self.package_list) - rsync_out="\n".join([a for a,b,c,d in self.package_list]) + rsync_out="\n".join([a for a,b,c,d,e,f in self.package_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) -- cgit v1.2.3 From 8fcf2f7f622de21b580ee808de53cffcc9200639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 03:41:12 -0600 Subject: Better testcases --- test/test1.py | 60 +++++++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/test/test1.py b/test/test1.py index db87f51..4ec72d3 100644 --- a/test/test1.py +++ b/test/test1.py @@ -14,12 +14,10 @@ import unittest class KnownValues(unittest.TestCase): directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") - # (output, name, version, arch, release, location) - link_list=( + # (rsync_out, name, version, arch, release, location) + examples=( ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), - ) - package_list=( ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", @@ -29,16 +27,8 @@ class KnownValues(unittest.TestCase): ) def generate_results(self, example_tuple): - a=list() - for output, name, version, arch, release, location in example_tuple: - pkg=Package() - pkg["name"] = name - pkg["version"] = version - pkg["arch"] = arch - pkg["release"] = release - pkg["location"] = location - a.append(pkg) - return tuple(a) + rsync_out="\n".join([a for a,b,c,d,e,f in example_tuple]) + return get_file_list_from_rsync_output(rsync_out) def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" @@ -46,21 +36,35 @@ class KnownValues(unittest.TestCase): result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(tuple(), result) - def testLinkOutput(self): - """get_file_list_from_rsync_output should make a Package Object - from links """ - correct_result=self.generate_results(self.link_list) - rsync_out="\n".join([a for a,b,c,d,e,f in self.link_list]) - result=get_file_list_from_rsync_output(rsync_out) - self.assertEqual(correct_result, result) + def testNames(self): + results=self.generate_results(self.examples) + var =[name for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) - def testPackageOutput(self): - """get_file_list_from_rsync_output should make a Package Object - from links """ - correct_result=self.generate_results(self.package_list) - rsync_out="\n".join([a for a,b,c,d,e,f in self.package_list]) - result=get_file_list_from_rsync_output(rsync_out) - self.assertEqual(correct_result, result) + def testVersions(self): + results=self.generate_results(self.examples) + var = [version for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) + + def testArchs(self): + results=self.generate_results(self.examples) + var = [arch for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) + + def testReleases(self): + results=self.generate_results(self.examples) + var = [release for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) + + def testLocations(self): + results=self.generate_results(self.examples) + var = [location for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From 748ef10d89e5898c1ebffb08165f18cd3829119e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 03:43:07 -0600 Subject: Fixed stupid error. --- test/test1.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test1.py b/test/test1.py index 4ec72d3..ea488e9 100644 --- a/test/test1.py +++ b/test/test1.py @@ -46,25 +46,25 @@ class KnownValues(unittest.TestCase): results=self.generate_results(self.examples) var = [version for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["version"], var[i]) def testArchs(self): results=self.generate_results(self.examples) var = [arch for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["arch"], var[i]) def testReleases(self): results=self.generate_results(self.examples) var = [release for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["release"], var[i]) def testLocations(self): results=self.generate_results(self.examples) var = [location for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["location"], var[i]) if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From df79a07c092880259a851d4ebf9febe9a8e19880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 10:47:55 -0600 Subject: Fixed test1.py --- test/test1.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/test/test1.py b/test/test1.py index ea488e9..61b0651 100644 --- a/test/test1.py +++ b/test/test1.py @@ -26,9 +26,9 @@ class KnownValues(unittest.TestCase): "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") ) - def generate_results(self, example_tuple): - rsync_out="\n".join([a for a,b,c,d,e,f in example_tuple]) - return get_file_list_from_rsync_output(rsync_out) + def generate_results(self, example_tuple, attr): + rsync_out, name, version, arch, release, location = example_tuple + return get_file_list_from_rsync_output(rsync_out)[0][attr], locals()[attr] def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" @@ -37,34 +37,29 @@ class KnownValues(unittest.TestCase): self.assertEqual(tuple(), result) def testNames(self): - results=self.generate_results(self.examples) - var =[name for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) def testVersions(self): - results=self.generate_results(self.examples) - var = [version for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["version"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) def testArchs(self): - results=self.generate_results(self.examples) - var = [arch for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["arch"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) def testReleases(self): - results=self.generate_results(self.examples) - var = [release for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["release"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) def testLocations(self): - results=self.generate_results(self.examples) - var = [location for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["location"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From 9c8b6615d537a9c7bb483457cd8a300d7f9c597e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 10:55:33 -0600 Subject: * Added filter.py * Added __init__.py for test module * Removed unused files --- filter.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ test/__init__.py | 0 test/link_list | 2 -- test/package_list | 3 -- 4 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 filter.py create mode 100644 test/__init__.py delete mode 100644 test/link_list delete mode 100644 test/package_list diff --git a/filter.py b/filter.py new file mode 100644 index 0000000..5229e63 --- /dev/null +++ b/filter.py @@ -0,0 +1,90 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +import commands +import os +import re +from repm.config import * +from repm.pato2 import * + +rsync_list_command="rsync -av --no-motd --list-only " + +def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, + mirror_path=mirrorpath, blacklist_file=False): + """ Generates an rsync command for executing it by combining all parameters. + + Parameters: + ---------- + base_command -> str + mirror_name -> str + mirror_path -> str + dir_list -> list or tuple + destdir -> str Must be a dir + blacklist_file -> False or str File must exist + + Return: + ---------- + rsync_command -> str """ + from os.path import isfile, isdir + + if blacklist_file and not isfile(blacklist_file): + print(blacklist_file + " is not a file") + raise NonValidFile + + if not os.path.isdir(destdir): + print(destdir + " is not a directory") + raise NonValidDir + + dir_list="{" + ",".join(dir_list) + "}" + + if blacklist_file: + return " ".join((base_command, "--exclude-from-file="+blacklist_file, + mirror_name + mirror_path + dir_list, destdir)) + return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + +def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), + debug=verbose): + """ Runs rsync and gets returns it's output """ + cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + if debug: + printf("rsync_command" + cmd) + return commands.getoutput(cmd) + +def get_file_list_from_rsync_output(rsync_output): + """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. + + Parameters: + ---------- + rsync_output -> str containing output from rsync + + Returns: + ---------- + package_list -> tuple of Package objects. """ + a=list() + + def directory(line): + pass + + def package_or_link(line): + """ Take info out of filename """ + location_field = 4 + pkg = Package() + pkg["location"] = line.rsplit()[location_field] + fileattrs = pkg["location"].split("/")[-1].split("-") + pkg["arch"] = fileattrs.pop(-1).split(".")[0] + pkg["release"] = fileattrs.pop(-1) + pkg["version"] = fileattrs.pop(-1) + pkg["name"] = "-".join(fileattrs) + return pkg + + options = { "d": directory, + "l": package_or_link, + "-": package_or_link} + + for line in rsync_output.split("\n"): + if ".pkg.tar.gz" or ".pkg.tar.xz" in line: + pkginfo=options[line[0]](line) + if pkginfo: + a.append(pkginfo) + + return tuple(a) + diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/link_list b/test/link_list deleted file mode 100644 index d015364..0000000 --- a/test/link_list +++ /dev/null @@ -1,2 +0,0 @@ -lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz -lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz \ No newline at end of file diff --git a/test/package_list b/test/package_list deleted file mode 100644 index 6ffa1de..0000000 --- a/test/package_list +++ /dev/null @@ -1,3 +0,0 @@ --rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz --rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz --rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz \ No newline at end of file -- cgit v1.2.3 From 1c178e144ae404c6ee332e25c9f3ae37fb9abb23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 10:56:31 -0600 Subject: * Modified gitignore for ignoring pyc files --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e4e5f6c..e645833 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -*~ \ No newline at end of file +*~ +*.pyc \ No newline at end of file -- cgit v1.2.3 From 5f08ca1edd02121f950a829745a512a2bc58b016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 11:05:26 -0600 Subject: * Changed variable name for consistency. --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index a7087c4..6206607 100644 --- a/config.py +++ b/config.py @@ -38,7 +38,7 @@ verbose = False blacklist = docs + "/blacklist.txt" whitelist = docs + "/whitelist.txt" pending = docs + "/pending" -rsyncBlacklist = docs + "/rsyncBlacklist" +rsync_blacklist = docs + "/rsyncBlacklist" # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3 From d9ac9d17a92607b09fb9afa91ff96aeae38dbdf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 11:55:25 -0600 Subject: Delete unneeded file. --- test/directory_list | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 test/directory_list diff --git a/test/directory_list b/test/directory_list deleted file mode 100644 index c0f7f47..0000000 --- a/test/directory_list +++ /dev/null @@ -1,2 +0,0 @@ -drwxrwxr-x 15 2010/09/11 11:28:50 community-staging -drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os \ No newline at end of file -- cgit v1.2.3 From 2cae61ff561561e405f0cd0c150dbcd77ce25b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 13:26:54 -0600 Subject: config py: * Make Package class init, restrict keys. filter.py * Corrected function * Added generate_rsync_exclude --- config.py | 31 ++++++++++++++++++++++++------- filter.py | 53 ++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index 6206607..bd8ef1b 100644 --- a/config.py +++ b/config.py @@ -47,15 +47,21 @@ class NonValidCommand(ValueError): pass class Package: """ An object that has information about a package. """ - package_info={ "name" : False, - "version" : False, - "release" : False, - "arch" : False, - "license" : False, - "location": False} + package_info=dict() + def __init__(self): + self.package_info={ "name" : False, + "version" : False, + "release" : False, + "arch" : False, + "license" : False, + "location": False} + def __setitem__(self, key, item): - return self.package_info.__setitem__(key, item) + if key in self.package_info.keys(): + return self.package_info.__setitem__(key, item) + else: + raise ValueError("Package has no %s attribute" % key) def __getitem__(self, key): return self.package_info.__getitem__(key) @@ -63,4 +69,15 @@ class Package: def __unicode__(self): return str(self.package_info) + def __repr__(self): + return str(self.package_info) + + def __eq__(self,x): + if not isinstance(x, Package): + return False + for key in self.package_info.keys(): + if x[key] != self[key]: + return False + else: + return True diff --git a/filter.py b/filter.py index 5229e63..817a228 100644 --- a/filter.py +++ b/filter.py @@ -6,7 +6,7 @@ import re from repm.config import * from repm.pato2 import * -rsync_list_command="rsync -av --no-motd --list-only " +rsync_list_command="rsync -a --no-motd --list-only " def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, mirror_path=mirrorpath, blacklist_file=False): @@ -18,8 +18,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name= mirror_name -> str mirror_path -> str dir_list -> list or tuple - destdir -> str Must be a dir - blacklist_file -> False or str File must exist + destdir -> str Path to dir, dir must exist. + blacklist_file -> False or str Path to file, file must exist. Return: ---------- @@ -54,11 +54,11 @@ def get_file_list_from_rsync_output(rsync_output): Parameters: ---------- - rsync_output -> str containing output from rsync + rsync_output -> str Contains output from rsync Returns: ---------- - package_list -> tuple of Package objects. """ + package_list -> tuple Contains Package objects. """ a=list() def directory(line): @@ -81,10 +81,45 @@ def get_file_list_from_rsync_output(rsync_output): "-": package_or_link} for line in rsync_output.split("\n"): - if ".pkg.tar.gz" or ".pkg.tar.xz" in line: - pkginfo=options[line[0]](line) - if pkginfo: - a.append(pkginfo) + if ".pkg.tar" in line: + pkginfo=options[line[0]](line) + if pkginfo: + a.append(pkginfo) return tuple(a) +def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, + blacklist_file=rsync_blacklist, debug=verbose): + """ Generate an exclude list for rsync + + Parameters: + ---------- + package_iterable -> list or tuple Contains Package objects + blacklisted_names-> list or tuple Contains blacklisted names + blacklist_file -> str Path to file + debug -> bool + + Output: + ---------- + if debug == False -> None + if debug == True -> blacklist """ + a=list() + + for package in packages_iterable: + if not isinstance(package, Package): + raise ValueError(" %s is not a Package object " % package) + if package["name"] in blacklisted_names: + a.append(package["location"]) + + if debug: + printf(a) + + try: + fsock = open(blacklist_file,"w") + try: + fsock.write("\n".join(a)) + finally: + fsock.close() + except IOError: + printf("%s wasnt written" % blacklist_file) + -- cgit v1.2.3 From 072787627a2339c3d5aca541086c2e4545bbad8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 14:06:58 -0600 Subject: * Changed variable name in filter.py * Added example in test1.py --- filter.py | 15 +++++++-------- test/test1.py | 5 +++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/filter.py b/filter.py index 817a228..ffc0965 100644 --- a/filter.py +++ b/filter.py @@ -89,20 +89,19 @@ def get_file_list_from_rsync_output(rsync_output): return tuple(a) def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, - blacklist_file=rsync_blacklist, debug=verbose): + exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync Parameters: ---------- - package_iterable -> list or tuple Contains Package objects - blacklisted_names-> list or tuple Contains blacklisted names - blacklist_file -> str Path to file - debug -> bool + package_iterable -> list or tuple Contains Package objects + blacklisted_names-> list or tuple Contains blacklisted names + exclude_file -> str Path to file + debug -> bool If True, file list gets logged Output: ---------- - if debug == False -> None - if debug == True -> blacklist """ + None """ a=list() for package in packages_iterable: @@ -115,7 +114,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, printf(a) try: - fsock = open(blacklist_file,"w") + fsock = open(exclude_file,"w") try: fsock.write("\n".join(a)) finally: diff --git a/test/test1.py b/test/test1.py index 61b0651..13b5660 100644 --- a/test/test1.py +++ b/test/test1.py @@ -13,7 +13,8 @@ import unittest class KnownValues(unittest.TestCase): directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", - "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") + "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", + 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') # (rsync_out, name, version, arch, release, location) examples=( ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), @@ -60,6 +61,6 @@ class KnownValues(unittest.TestCase): for i in self.examples: k,v = self.generate_results(example_tuple=i,attr="location") self.assertEqual(k, v) - + if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From 6a37d25e51228a0a83d004b9acda63c528972a26 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 13 Feb 2011 14:26:40 -0800 Subject: Fixed glob expression --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 236f2ea..cfdd859 100644 --- a/pato2.py +++ b/pato2.py @@ -54,7 +54,7 @@ def packages(repo_, arch_, expr="*"): def sync_all_repo(verbose_=verbose): folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --progress --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir + cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) @@ -144,11 +144,11 @@ def add_free_repo(verbose_=verbose): for repo_ in repo_list: for arch_ in arch_list: lista_=list() - for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): lista_.append(file_) link(repo_,arch_,file_) for dir_ in other: - for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): lista_.append(file_) link(repo_,arch_,file_) if lista_: -- cgit v1.2.3 From 2321a53d35a9736b8c5c715ca40a9af69b1bf64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:14:47 -0600 Subject: Rearranged functions --- config.py | 5 +++++ filter.py | 43 ------------------------------------------- pato2.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/config.py b/config.py index bd8ef1b..4854e0d 100644 --- a/config.py +++ b/config.py @@ -40,6 +40,11 @@ whitelist = docs + "/whitelist.txt" pending = docs + "/pending" rsync_blacklist = docs + "/rsyncBlacklist" +# Rsync commands + +rsync_list_command="rsync -a --no-motd --list-only " + + # Classes and Exceptions class NonValidFile(ValueError): pass class NonValidDir(ValueError): pass diff --git a/filter.py b/filter.py index ffc0965..b51dba8 100644 --- a/filter.py +++ b/filter.py @@ -6,49 +6,6 @@ import re from repm.config import * from repm.pato2 import * -rsync_list_command="rsync -a --no-motd --list-only " - -def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, - mirror_path=mirrorpath, blacklist_file=False): - """ Generates an rsync command for executing it by combining all parameters. - - Parameters: - ---------- - base_command -> str - mirror_name -> str - mirror_path -> str - dir_list -> list or tuple - destdir -> str Path to dir, dir must exist. - blacklist_file -> False or str Path to file, file must exist. - - Return: - ---------- - rsync_command -> str """ - from os.path import isfile, isdir - - if blacklist_file and not isfile(blacklist_file): - print(blacklist_file + " is not a file") - raise NonValidFile - - if not os.path.isdir(destdir): - print(destdir + " is not a directory") - raise NonValidDir - - dir_list="{" + ",".join(dir_list) + "}" - - if blacklist_file: - return " ".join((base_command, "--exclude-from-file="+blacklist_file, - mirror_name + mirror_path + dir_list, destdir)) - return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) - -def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), - debug=verbose): - """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) - if debug: - printf("rsync_command" + cmd) - return commands.getoutput(cmd) - def get_file_list_from_rsync_output(rsync_output): """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. diff --git a/pato2.py b/pato2.py index cfdd859..125ae8c 100644 --- a/pato2.py +++ b/pato2.py @@ -166,6 +166,46 @@ def get_licenses(verbose_=verbose): a=commands.getoutput(cmd_) if verbose_: printf(a) +def generate_rsync_command(base_command, dir_list, destdir=repodir, + source=mirror+mirrorpath, blacklist_file=False): + """ Generates an rsync command for executing it by combining all parameters. + + Parameters: + ---------- + base_command -> str + dir_list -> list or tuple + destdir -> str Path to dir, dir must exist. + source -> str The source for rsync + blacklist_file -> False or str Path to file, file must exist. + + Return: + ---------- + rsync_command -> str """ + from os.path import isfile, isdir + + if blacklist_file and not isfile(blacklist_file): + print(blacklist_file + " is not a file") + raise NonValidFile + + if not os.path.isdir(destdir): + print(destdir + " is not a directory") + raise NonValidDir + + dir_list="{" + ",".join(dir_list) + "}" + + if blacklist_file: + return " ".join((base_command, "--exclude-from-file="+blacklist_file, + mirror_name + mirror_path + dir_list, destdir)) + return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + +def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), + debug=verbose): + """ Runs rsync and gets returns it's output """ + cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + if debug: + printf("rsync_command" + cmd) + return commands.getoutput(cmd) + if __name__ == "__main__": from time import time start_time = time() -- cgit v1.2.3 From 6203dc2fc926781db694ca2383b1e44e2c5469c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:29:33 -0600 Subject: - filter.py: * Makes a blacklist for rsync - pato2.py: * Has more functions --- filter.py | 6 +++++- pato2.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/filter.py b/filter.py index b51dba8..14ae31e 100644 --- a/filter.py +++ b/filter.py @@ -1,4 +1,4 @@ -#! /usr/bin/python + #! /usr/bin/python #-*- encoding: utf-8 -*- import commands import os @@ -79,3 +79,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) +if name == "__main__": + a=run_rsync(rsync_list_command) + packages=get_file_list_from_rsync_output(a) + generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/pato2.py b/pato2.py index 125ae8c..97eec9d 100644 --- a/pato2.py +++ b/pato2.py @@ -198,7 +198,7 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name + mirror_path + dir_list, destdir)) return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) -def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), +def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): """ Runs rsync and gets returns it's output """ cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) -- cgit v1.2.3 From b27cbc08447fe5605bf877ee0991888d5ecf6382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:32:48 -0600 Subject: Corrected filter.py to run --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.py b/filter.py index 14ae31e..c789cd5 100644 --- a/filter.py +++ b/filter.py @@ -79,7 +79,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) -if name == "__main__": +if __name__ == "__main__": a=run_rsync(rsync_list_command) packages=get_file_list_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) -- cgit v1.2.3 From 931daf34033714e64fb4f98a101dd804e4e6363e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:34:35 -0600 Subject: Added os module to config file --- config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config.py b/config.py index 4854e0d..bd46688 100644 --- a/config.py +++ b/config.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from user import home import commands +import os time__ = commands.getoutput("date +%Y%m%d-%H:%M") -- cgit v1.2.3 From 7f40b89e8cd0e9b2cfd92a8b74c4063d8b9aa3e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:38:59 -0600 Subject: Corrected funcions --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 97eec9d..55e63e7 100644 --- a/pato2.py +++ b/pato2.py @@ -195,13 +195,13 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - mirror_name + mirror_path + dir_list, destdir)) - return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + source + dir_list, destdir)) + return " ".join((base_command, source + dir_list, destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: printf("rsync_command" + cmd) return commands.getoutput(cmd) -- cgit v1.2.3 From cbe877f2ba25b398ad32b92d22dc6f5a108ef59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 02:28:01 -0600 Subject: * Changed get_file_list_ for pkginfo in function names * Added pkginfo_from_... funcions --- filter.py | 57 ++++++++++++++++++++++++++++++++++++++------------------- test/test1.py | 8 ++++---- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/filter.py b/filter.py index c789cd5..232319f 100644 --- a/filter.py +++ b/filter.py @@ -1,13 +1,34 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- import commands -import os import re from repm.config import * from repm.pato2 import * -def get_file_list_from_rsync_output(rsync_output): - """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. +def pkginfo_from_filename(filename): + """ Generates a Package object with info from a filename, + filename can be relative or absolute + + Parameters: + ---------- + filename -> str + + Returns: + ---------- + pkg -> Package object""" + pkg = Package() + pkg["location"] = filename + fileattrs = os.path.basename(filename).split("-") + pkg["arch"] = fileattrs.pop(-1).split(".")[0] + pkg["release"] = fileattrs.pop(-1) + pkg["version"] = fileattrs.pop(-1) + pkg["name"] = "-".join(fileattrs) + return pkg + + +def pkginfo_from_rsync_output(rsync_output): + """ Generates a list of packages and versions from an rsync output + wich uses --list-only and --no-motd options. Parameters: ---------- @@ -18,33 +39,30 @@ def get_file_list_from_rsync_output(rsync_output): package_list -> tuple Contains Package objects. """ a=list() - def directory(line): - pass - def package_or_link(line): """ Take info out of filename """ location_field = 4 - pkg = Package() - pkg["location"] = line.rsplit()[location_field] - fileattrs = pkg["location"].split("/")[-1].split("-") - pkg["arch"] = fileattrs.pop(-1).split(".")[0] - pkg["release"] = fileattrs.pop(-1) - pkg["version"] = fileattrs.pop(-1) - pkg["name"] = "-".join(fileattrs) - return pkg - + pkginfo_from_filename(line.rsplit()[location_field]) + + def directory(line): + pass + options = { "d": directory, "l": package_or_link, "-": package_or_link} for line in rsync_output.split("\n"): if ".pkg.tar" in line: - pkginfo=options[line[0]](line) - if pkginfo: - a.append(pkginfo) + pkginfo_=options[line[0]](line) + if pkginfo_: + a.append(pkginfo_) return tuple(a) +def pkginfo_from_files_in_dir(directory): + """ Returns pkginfo from filenames of packages in dir + wich has .pkg.tar.{xz,gz} on them """ + def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync @@ -78,8 +96,9 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, fsock.close() except IOError: printf("%s wasnt written" % blacklist_file) + if __name__ == "__main__": a=run_rsync(rsync_list_command) - packages=get_file_list_from_rsync_output(a) + packages=pkginfo_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/test/test1.py b/test/test1.py index 13b5660..9e94352 100644 --- a/test/test1.py +++ b/test/test1.py @@ -29,12 +29,12 @@ class KnownValues(unittest.TestCase): def generate_results(self, example_tuple, attr): rsync_out, name, version, arch, release, location = example_tuple - return get_file_list_from_rsync_output(rsync_out)[0][attr], locals()[attr] + return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] def testDirectoryOutput(self): - """get_file_list_from_rsync_output should ignore directories""" + """pkginfo_from_rsync_output should ignore directories""" rsync_out="\n".join(self.directory_list) - result=get_file_list_from_rsync_output(rsync_out) + result=pkginfo_from_rsync_output(rsync_out) self.assertEqual(tuple(), result) def testNames(self): @@ -62,5 +62,5 @@ class KnownValues(unittest.TestCase): k,v = self.generate_results(example_tuple=i,attr="location") self.assertEqual(k, v) -if __name__ == "__main__": +if __name__ == "__mpain__": unittest.main() -- cgit v1.2.3 From 4e7508d9a61c3653b46da8fa513513756bb3b6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 16:46:33 -0600 Subject: * pkginfo functions --- filter.py | 44 +++++++++++++++++++++++++++++++------------- test/test1.py | 2 +- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/filter.py b/filter.py index 232319f..ad3b33d 100644 --- a/filter.py +++ b/filter.py @@ -1,7 +1,6 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- -import commands -import re +import glob from repm.config import * from repm.pato2 import * @@ -11,11 +10,13 @@ def pkginfo_from_filename(filename): Parameters: ---------- - filename -> str + filename -> str Must contain .pkg.tar. Returns: ---------- pkg -> Package object""" + if ".pkg.tar." not in filename: + raise NonValidFile pkg = Package() pkg["location"] = filename fileattrs = os.path.basename(filename).split("-") @@ -37,31 +38,49 @@ def pkginfo_from_rsync_output(rsync_output): Returns: ---------- package_list -> tuple Contains Package objects. """ - a=list() + package_list=list() def package_or_link(line): """ Take info out of filename """ location_field = 4 pkginfo_from_filename(line.rsplit()[location_field]) - def directory(line): + def do_nothing(): pass - - options = { "d": directory, + + options = { "d": do_nothing, "l": package_or_link, - "-": package_or_link} + "-": package_or_link, + " ": do_nothing} for line in rsync_output.split("\n"): if ".pkg.tar" in line: - pkginfo_=options[line[0]](line) + pkginfo=options[line[0]](line) if pkginfo_: - a.append(pkginfo_) + package_list.append(pkginfo) - return tuple(a) + return tuple(package_list) def pkginfo_from_files_in_dir(directory): """ Returns pkginfo from filenames of packages in dir - wich has .pkg.tar.{xz,gz} on them """ + wich has .pkg.tar. on them + + Parameters: + ---------- + directory -> str Directory must exist + + Returns: + ---------- + package_list -> tuple Contains Package objects """ + package_list=list() + + if not os.path.isdir(directory): + raise NonValidDir + + for filename in glob(os.path.join(directory,"*")): + if ".pkg.tar." in filename: + package_list.append(pkginfo_from_filename(filename)) + return tuple(package_list) def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): @@ -97,7 +116,6 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) - if __name__ == "__main__": a=run_rsync(rsync_list_command) packages=pkginfo_from_rsync_output(a) diff --git a/test/test1.py b/test/test1.py index 9e94352..b810e9a 100644 --- a/test/test1.py +++ b/test/test1.py @@ -62,5 +62,5 @@ class KnownValues(unittest.TestCase): k,v = self.generate_results(example_tuple=i,attr="location") self.assertEqual(k, v) -if __name__ == "__mpain__": +if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From 1db8ccbc9221be6c30884172c00caefa217f66a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 16:47:03 -0600 Subject: Added test for pkginfo_from_file --- test/test_pkginfo_from_file.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 test/test_pkginfo_from_file.py diff --git a/test/test_pkginfo_from_file.py b/test/test_pkginfo_from_file.py new file mode 100644 index 0000000..0d7ede6 --- /dev/null +++ b/test/test_pkginfo_from_file.py @@ -0,0 +1,76 @@ +#! /usr/bin/python +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class KnownValues(unittest.TestCase): + # (filename, name, version, release, arch) + # filename is location + known=( + ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), + ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), + ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), + ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), + ) + + def generate_results(self, example_tuple, attr): + location, name, version, release, arch = example_tuple + return pkginfo_from_filename(location)[attr], locals()[attr] + + def testReturnPackageObject(self): + for i in self.known: + location, name, version, release, arch = i + self.assertIsInstance(pkginfo_from_filename(location),Package) + + def testNames(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +class BadInput(unittest.TestCase): + bad=("community-testing/os/i686/community-testing.db", + "community-testing/os/i686/community-testing.db.tar.gz", + "community-testing/os/i686/community-testing.db.tar.gz.old", + "community-testing/os/i686/community-testing.files", + "community-testing/os/i686/community-testing.files.tar.gz", + "community-testing/os/x86_64") + + def testBadInput(self): + for i in self.bad: + self.assertRaises(NonValidFile,pkginfo_from_filename,i) + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3 From b525eeca86d3c5a25f971a12ceeae44fadd6fa94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 16:59:28 -0600 Subject: renamed test1 to be informative --- test/test_pkginfo_from_rsync_output.py | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 test/test_pkginfo_from_rsync_output.py diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py new file mode 100644 index 0000000..b810e9a --- /dev/null +++ b/test/test_pkginfo_from_rsync_output.py @@ -0,0 +1,66 @@ +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class KnownValues(unittest.TestCase): + directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", + "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", + 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') + # (rsync_out, name, version, arch, release, location) + examples=( + ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), + ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), + ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", + "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), + ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", + "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), + ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", + "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") + ) + + def generate_results(self, example_tuple, attr): + rsync_out, name, version, arch, release, location = example_tuple + return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] + + def testDirectoryOutput(self): + """pkginfo_from_rsync_output should ignore directories""" + rsync_out="\n".join(self.directory_list) + result=pkginfo_from_rsync_output(rsync_out) + self.assertEqual(tuple(), result) + + def testNames(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3 From a20a7fc689fd0e8680cbc13adfa6fdc2849e4397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 17:00:13 -0600 Subject: idem --- test/test1.py | 66 ----------------------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 test/test1.py diff --git a/test/test1.py b/test/test1.py deleted file mode 100644 index b810e9a..0000000 --- a/test/test1.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -class KnownValues(unittest.TestCase): - directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", - "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", - 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') - # (rsync_out, name, version, arch, release, location) - examples=( - ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), - ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", - "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), - ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", - "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", - "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") - ) - - def generate_results(self, example_tuple, attr): - rsync_out, name, version, arch, release, location = example_tuple - return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] - - def testDirectoryOutput(self): - """pkginfo_from_rsync_output should ignore directories""" - rsync_out="\n".join(self.directory_list) - result=pkginfo_from_rsync_output(rsync_out) - self.assertEqual(tuple(), result) - - def testNames(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) - -if __name__ == "__main__": - unittest.main() -- cgit v1.2.3 From 9fecb003deaae2364e0d9d41055ca33a7d1a2131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 17:09:16 -0600 Subject: * Changed format of run_rsync debug output. --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 55e63e7..13242f7 100644 --- a/pato2.py +++ b/pato2.py @@ -203,7 +203,7 @@ def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), """ Runs rsync and gets returns it's output """ cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: - printf("rsync_command" + cmd) + printf("rsync_command: " + cmd) return commands.getoutput(cmd) if __name__ == "__main__": -- cgit v1.2.3 From e4ffbeff4bbdeb3f57c39c6f6dac3ce902db9b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 17:14:37 -0600 Subject: * Fixed generate_rsync_command --- pato2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pato2.py b/pato2.py index 13242f7..74d18be 100644 --- a/pato2.py +++ b/pato2.py @@ -195,8 +195,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - source + dir_list, destdir)) - return " ".join((base_command, source + dir_list, destdir)) + os.join(source, dir_list), destdir)) + return " ".join((base_command, os.join(source, dir_list), destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): -- cgit v1.2.3 From e5b869eade3fbf47cf9ac1b7c6be34dfee14b2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 17:17:31 -0600 Subject: * Fixed generate_rsync_command --- pato2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pato2.py b/pato2.py index 74d18be..e32a4dd 100644 --- a/pato2.py +++ b/pato2.py @@ -195,8 +195,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - os.join(source, dir_list), destdir)) - return " ".join((base_command, os.join(source, dir_list), destdir)) + os.path.join(source, dir_list), destdir)) + return " ".join((base_command, os.path.join(source, dir_list), destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): -- cgit v1.2.3 From f923ab304017a71136642ed7b9780441edcaf441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 19:36:26 -0600 Subject: * TestCase for pkginfo_from_rsync_output * Corrected __eq__ method in Package class * Corrected pkginfo_from_rsync_output --- config.py | 2 +- filter.py | 18 ++++---- test/rsync_output_sample | 14 +++++++ test/test_pkginfo_from_rsync_output.py | 76 ++++++++++++---------------------- 4 files changed, 52 insertions(+), 58 deletions(-) create mode 100644 test/rsync_output_sample diff --git a/config.py b/config.py index bd46688..465a8a8 100644 --- a/config.py +++ b/config.py @@ -82,7 +82,7 @@ class Package: if not isinstance(x, Package): return False for key in self.package_info.keys(): - if x[key] != self[key]: + if x[key] != self.package_info[key]: return False else: return True diff --git a/filter.py b/filter.py index ad3b33d..55ab94a 100644 --- a/filter.py +++ b/filter.py @@ -38,26 +38,28 @@ def pkginfo_from_rsync_output(rsync_output): Returns: ---------- package_list -> tuple Contains Package objects. """ - package_list=list() def package_or_link(line): """ Take info out of filename """ location_field = 4 - pkginfo_from_filename(line.rsplit()[location_field]) + return pkginfo_from_filename(line.rsplit()[location_field]) def do_nothing(): - pass + """""" options = { "d": do_nothing, "l": package_or_link, "-": package_or_link, " ": do_nothing} + + package_list=list() - for line in rsync_output.split("\n"): - if ".pkg.tar" in line: - pkginfo=options[line[0]](line) - if pkginfo_: - package_list.append(pkginfo) + lines=[x for x in rsync_output.split("\n") if ".pkg.tar" in x] + + for line in lines: + pkginfo=options[line[0]](line) + if pkginfo: + package_list.append(pkginfo) return tuple(package_list) diff --git a/test/rsync_output_sample b/test/rsync_output_sample new file mode 100644 index 0000000..72d9cd0 --- /dev/null +++ b/test/rsync_output_sample @@ -0,0 +1,14 @@ +dr-xr-sr-x 4096 2010/09/11 11:37:10 . +-rw-r--r-- 11 2011/02/08 00:00:01 lastsync +drwxrwxr-x 15 2010/09/11 11:28:50 community-staging +drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os +drwxrwxr-x 8192 2011/02/07 17:00:01 community-staging/os/i686 +lrwxrwxrwx 52 2010/12/23 16:51:01 community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz -> ../../../pool/community/alex-2.3.4-1-i686.pkg.tar.xz +lrwxrwxrwx 27 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db -> community-staging.db.tar.gz +-rw-rw-r-- 2237 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db.tar.gz +-rw-rw-r-- 3209 2011/02/07 14:00:13 community-staging/os/i686/community-staging.db.tar.gz.old +drwxrwxr-x 15 2009/07/22 15:07:56 community +drwxrwxr-x 40 2009/08/04 15:57:42 community/os +drwxrwsr-x 36864 2011/02/03 05:00:01 community/os/any +-rw-rw-r-- 303336 2010/07/16 10:06:28 community/os/any/any2dvd-0.34-4-any.pkg.tar.xz +-rw-rw-r-- 221664 2010/03/28 15:55:48 community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index b810e9a..81f14d9 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -12,55 +12,33 @@ from repm.filter import * import unittest class KnownValues(unittest.TestCase): - directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", - "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", - 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') - # (rsync_out, name, version, arch, release, location) - examples=( - ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), - ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", - "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), - ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", - "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", - "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") - ) - - def generate_results(self, example_tuple, attr): - rsync_out, name, version, arch, release, location = example_tuple - return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] - - def testDirectoryOutput(self): - """pkginfo_from_rsync_output should ignore directories""" - rsync_out="\n".join(self.directory_list) - result=pkginfo_from_rsync_output(rsync_out) - self.assertEqual(tuple(), result) - - def testNames(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) + try: + output_file = open("rsync_output_sample") + rsync_out= output_file.read() + output_file.close() + except IOError: print("There is no rsync_output_sample file") + + pkglist = pkginfo_from_rsync_output(rsync_out) + + def testOutputArePackages(self): + if not self.pkglist: + self.fail("not pkglist:" + str(self.pkglist)) + for pkg in self.pkglist: + self.assertIsInstance(pkg,Package) + + def testFirstPkg(self): + first_package_info=Package() + first_package_info.package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} + if self.pkglist: + first_package=self.pkglist[0] + else: + self.fail(self.pkglist) + self.assertEqual(first_package,first_package_info) if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From 3866f148dae1f6c90fc6d903784b65e452c048c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 22:45:11 -0600 Subject: * Added debug action to generate_exclude_list_from_blacklist * Added test data to test_pkginfo_from_rsync_output.py --- filter.py | 2 +- test/test_pkginfo_from_rsync_output.py | 38 ++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/filter.py b/filter.py index 55ab94a..f8182e3 100644 --- a/filter.py +++ b/filter.py @@ -108,7 +108,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, if debug: printf(a) - + return a try: fsock = open(exclude_file,"w") try: diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index 81f14d9..5ed5dd8 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -11,7 +11,27 @@ from repm.config import * from repm.filter import * import unittest -class KnownValues(unittest.TestCase): +example_package_list=(Package(),Package(),Package()) +example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} +example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz"} +example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz"} + +class pkginfoFromRsyncOutput(unittest.TestCase): try: output_file = open("rsync_output_sample") rsync_out= output_file.read() @@ -26,19 +46,15 @@ class KnownValues(unittest.TestCase): for pkg in self.pkglist: self.assertIsInstance(pkg,Package) - def testFirstPkg(self): - first_package_info=Package() - first_package_info.package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} + def testPackageInfo(self): if self.pkglist: first_package=self.pkglist[0] else: self.fail(self.pkglist) - self.assertEqual(first_package,first_package_info) - + self.assertEqual(first_package,example_package_list[0]) + +class generateRsyncBlacklist(unittest.TestCase): + """ Test Blacklist generation """ + if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From 9de298a7b88f7f36aec4c9344356a935c67cfeb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 23:27:46 -0600 Subject: * Added test for generate_exclude_list_from_blacklist * Fixed listado to strip spaces --- filter.py | 1 - pato2.py | 2 +- test/blacklist_sample | 2 ++ test/test_pkginfo_from_rsync_output.py | 17 +++++++++++------ 4 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 test/blacklist_sample diff --git a/filter.py b/filter.py index f8182e3..f91ec68 100644 --- a/filter.py +++ b/filter.py @@ -107,7 +107,6 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, a.append(package["location"]) if debug: - printf(a) return a try: fsock = open(exclude_file,"w") diff --git a/pato2.py b/pato2.py index e32a4dd..5bc97a9 100644 --- a/pato2.py +++ b/pato2.py @@ -42,7 +42,7 @@ def listado(filename_): archivo = open(filename_,"r") lista = archivo.read().split("\n") archivo.close() - return [pkg.split(":")[0] for pkg in lista if pkg] + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] def db(repo_,arch_): """Construye un nombre para sincronizar una base de datos.""" diff --git a/test/blacklist_sample b/test/blacklist_sample new file mode 100644 index 0000000..2a02af6 --- /dev/null +++ b/test/blacklist_sample @@ -0,0 +1,2 @@ +alex:alex-libre: Aquí va un comentario +gmime22 ::Non free dependencies \ No newline at end of file diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index 5ed5dd8..aca49e8 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -47,14 +47,19 @@ class pkginfoFromRsyncOutput(unittest.TestCase): self.assertIsInstance(pkg,Package) def testPackageInfo(self): - if self.pkglist: - first_package=self.pkglist[0] - else: - self.fail(self.pkglist) - self.assertEqual(first_package,example_package_list[0]) + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,example_package_list) class generateRsyncBlacklist(unittest.TestCase): """ Test Blacklist generation """ - + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=generate_exclude_list_from_blacklist(example_package_list,listado("blacklist_sample"),debug=True) + b=[example_package_list[0]["location"],example_package_list[2]["location"]] + self.assertEqual(a,b) + if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From cf3253774c64e6430d4e3afb826cc6c8b0e4b91b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 15 Feb 2011 00:13:37 -0600 Subject: * Upgraded sync_all_repo --- config.py | 2 +- pato2.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config.py b/config.py index 465a8a8..36b54df 100644 --- a/config.py +++ b/config.py @@ -44,7 +44,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " - +rsync_update_command="rsync -av --delete-after --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass diff --git a/pato2.py b/pato2.py index 5bc97a9..9b902f4 100644 --- a/pato2.py +++ b/pato2.py @@ -25,7 +25,7 @@ """ from repm.config import * - +from repm.filter import * import tarfile from glob import glob from os.path import isdir, isfile @@ -52,12 +52,14 @@ def packages(repo_, arch_, expr="*"): """ Get packages on a repo, arch folder """ return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) -def sync_all_repo(verbose_=verbose): - folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) +def sync_all_repo(debug=verbose): + cmd=generate_rsync_command(rsync_list_command) + rsout=run_rsync(cmd) + pkgs=pkginfo_from_rsync_output(rsout) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + a=run_rsync(cmd) + if debug: printf(a) def get_from_desc(desc, var,db_tar_file=False): """ Get a var from desc file """ @@ -166,7 +168,7 @@ def get_licenses(verbose_=verbose): a=commands.getoutput(cmd_) if verbose_: printf(a) -def generate_rsync_command(base_command, dir_list, destdir=repodir, +def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, source=mirror+mirrorpath, blacklist_file=False): """ Generates an rsync command for executing it by combining all parameters. @@ -198,10 +200,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, os.path.join(source, dir_list), destdir)) return " ".join((base_command, os.path.join(source, dir_list), destdir)) -def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), - debug=verbose): +def run_rsync(command,debug=verbose): """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: printf("rsync_command: " + cmd) return commands.getoutput(cmd) -- cgit v1.2.3 From 8975a9021c8b04fb81d9b9358a55b89ecdb373b7 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Feb 2011 22:18:36 -0800 Subject: changed config --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 36b54df..c60e8f3 100644 --- a/config.py +++ b/config.py @@ -33,7 +33,7 @@ other = ("any",) # Output output = True -verbose = False +verbose = True # Files blacklist = docs + "/blacklist.txt" -- cgit v1.2.3 From fb247388925beabe01162f28bfcba7fcf8809254 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Feb 2011 22:41:29 -0800 Subject: * Fixed for rsync to run --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 9b902f4..c598c2f 100644 --- a/pato2.py +++ b/pato2.py @@ -196,15 +196,15 @@ def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdi dir_list="{" + ",".join(dir_list) + "}" if blacklist_file: - return " ".join((base_command, "--exclude-from-file="+blacklist_file, + return " ".join((base_command, "--exclude-from="+blacklist_file, os.path.join(source, dir_list), destdir)) return " ".join((base_command, os.path.join(source, dir_list), destdir)) def run_rsync(command,debug=verbose): """ Runs rsync and gets returns it's output """ if debug: - printf("rsync_command: " + cmd) - return commands.getoutput(cmd) + printf("rsync_command: " + command) + return commands.getoutput(command) if __name__ == "__main__": from time import time -- cgit v1.2.3 From 7bb59afd65871ed2949b86ec6cef805ea2fd3dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 15 Feb 2011 19:25:51 -0800 Subject: Fixed add_free_repo --- config.py | 1 + pato2.py | 295 ++++++++++++++++++++++++++++++++------------------------------ 2 files changed, 152 insertions(+), 144 deletions(-) diff --git a/config.py b/config.py index c60e8f3..6420355 100644 --- a/config.py +++ b/config.py @@ -19,6 +19,7 @@ logdir = path + "/log" ## Must be defined logname= logdir + "/" + time__ + "-repo-maintainer.log" +freedir= path + "/free/" repodir= path + "/repo" tmp = home + "/tmp" archdb = tmp + "/db" diff --git a/pato2.py b/pato2.py index c598c2f..f5deb48 100644 --- a/pato2.py +++ b/pato2.py @@ -31,142 +31,149 @@ from glob import glob from os.path import isdir, isfile def printf(text,output_=output): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(logname, 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output_: print (str(text) + "\n") + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(logname, 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output_: print (str(text) + "\n") def listado(filename_): - """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename_,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename_,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] def db(repo_,arch_): - """Construye un nombre para sincronizar una base de datos.""" - return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) + """Construye un nombre para sincronizar una base de datos.""" + return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) def packages(repo_, arch_, expr="*"): - """ Get packages on a repo, arch folder """ - return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) + """ Get packages on a repo, arch folder """ + return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) def sync_all_repo(debug=verbose): - cmd=generate_rsync_command(rsync_list_command) - rsout=run_rsync(cmd) - pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) - a=run_rsync(cmd) - if debug: printf(a) + cmd=generate_rsync_command(rsync_list_command) + rsout=run_rsync(cmd) + pkgs=pkginfo_from_rsync_output(rsout) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + a=run_rsync(cmd) + if debug: printf(a) def get_from_desc(desc, var,db_tar_file=False): - """ Get a var from desc file """ - desc = desc.split("\n") - return desc[desc.index(var)+1] + """ Get a var from desc file """ + desc = desc.split("\n") + return desc[desc.index(var)+1] def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): - """ Makes a list of package name, file and license """ - info=list() - # Extract DB tar.gz - commands.getoutput("mkdir -p " + archdb) - if not db_tar_file: - db_tar_file = repodir + db(repo_,arch_) - if isfile(db_tar_file): - try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') - except tarfile.ReadError: - printf("No valid db_file %s" % db_tar_file) - return(tuple()) - else: - printf("No db_file %s" % db_tar_file) - return(tuple()) - for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) - db_open_tar.close() - # Get info from file - for dir_ in glob(archdb + "/*"): - if isdir(dir_) and isfile(dir_ + "/desc"): - pkg_desc_file = open(dir_ + "/desc", "r") - desc = pkg_desc_file.read() - pkg_desc_file.close() - info.append(( get_from_desc(desc,"%NAME%"), - dir_.split("/")[-1], - get_from_desc(desc,"%LICENSE%") )) - if verbose_: printf(info) - commands.getoutput("rm -r %s/*" % archdb) - return tuple(info) + """ Makes a list of package name, file and license """ + info=list() + # Extract DB tar.gz + commands.getoutput("mkdir -p " + archdb) + if not db_tar_file: + db_tar_file = repodir + db(repo_,arch_) + if isfile(db_tar_file): + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s" % db_tar_file) + return(tuple()) + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + pkg_desc_file = open(dir_ + "/desc", "r") + desc = pkg_desc_file.read() + pkg_desc_file.close() + info.append(( get_from_desc(desc,"%NAME%"), + dir_.split("/")[-1], + get_from_desc(desc,"%LICENSE%") )) + if verbose_: printf(info) + commands.getoutput("rm -r %s/*" % archdb) + return tuple(info) def make_pending(repo_,arch_,info_): - """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" - search = tuple( listado(blacklist) + listado (whitelist) ) - if verbose: printf("blaclist + whitelist= " + str(search) ) - lista_=list() - for (name,pkg_,license_) in info_: - if "custom" in license_: - if name not in search: - lista_.append( (name, license_ ) ) - elif not name: - printf( pkg_ + " package has no %NAME% attibute " ) - if verbose: printf( lista_ ) - a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" + search = tuple( listado(blacklist) + listado (whitelist) ) + if verbose: printf("blaclist + whitelist= " + str(search) ) + lista_=list() + for (name,pkg_,license_) in info_: + if "custom" in license_: + if name not in search: + lista_.append( (name, license_ ) ) + elif not name: + printf( pkg_ + " package has no %NAME% attibute " ) + if verbose: printf( lista_ ) + a=open( pending + "-" + repo_ + ".txt", "w" ).write( + "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) def remove_from_blacklist(repo_,arch_,info_,blacklist_): - """ Check the blacklist and remove packages on the db""" - lista_=list() - pack_=list() - for (name_, pkg_, license_) in info_: - if name_ in blacklist_: - lista_.append(name_) - for p in packages(repo_,arch_,pkg_ + "*"): - pack_.append(p) - if lista_: - lista_=" ".join(lista_) - com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ - printf(com_) - a = commands.getoutput(com_) - if verbose: printf(a) - if pack_: - pack_=" ".join(pack_) - com_="chmod a-r " + pack_ - printf(com_) - a=commands.getoutput(com_) - if verbose: printf(a) + """ Check the blacklist and remove packages on the db""" + lista_=list() + pack_=list() + for (name_, pkg_, license_) in info_: + if name_ in blacklist_: + lista_.append(name_) + for p in packages(repo_,arch_,pkg_ + "*"): + pack_.append(p) + if lista_: + lista_=" ".join(lista_) + com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ + printf(com_) + a = commands.getoutput(com_) + if verbose: printf(a) + if pack_: + pack_=" ".join(pack_) + com_="chmod a-r " + pack_ + printf(com_) + a=commands.getoutput(com_) + if verbose: printf(a) def link(repo_,arch_,file_): - """ Makes a link in the repo for the package """ - cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ - a=commands.getoutput(cmd_) - if verbose: - printf(cmd_ + a) + """ Makes a link in the repo for the package """ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + a=commands.getoutput(cmd_) + if verbose: + printf(cmd_ + a) def add_free_repo(verbose_=verbose): - for repo_ in repo_list: - for arch_ in arch_list: - lista_=list() - for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): - lista_.append(file_) - link(repo_,arch_,file_) - for dir_ in other: - for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): - lista_.append(file_) - link(repo_,arch_,file_) - if lista_: - lista_=" ".join(lista_) - if verbose: printf(lista_) - cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose: printf(a) + cmd_=home + "/usr/bin/sync-free" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + for repo_ in repo_list: + for arch_ in arch_list: + lista_=list() + for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): + lista_.append(file_) + # link(repo_,arch_,file_) + for dir_ in other: + for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): + lista_.append(file_) + # link(repo_,arch_,file_) + + printf(lista_) + + if lista_: + lista_=" ".join(lista_) + if verbose: printf(lista_) + cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose: printf(a) def get_licenses(verbose_=verbose): - """ Extract the license from packages in repo_,arch_ and in pending_ file""" - cmd_=home + "/usr/bin/get_license.sh" - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) + """ Extract the license from packages in repo_,arch_ and in pending_ file""" + cmd_=home + "/usr/bin/get_license.sh" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, source=mirror+mirrorpath, blacklist_file=False): @@ -207,33 +214,33 @@ def run_rsync(command,debug=verbose): return commands.getoutput(command) if __name__ == "__main__": - from time import time - start_time = time() - def minute(): - return str(round((time() - start_time)/60, 1)) - - printf(" Cleaning %s folder " % (tmp) ) - commands.getoutput("rm -r %s/*" % tmp) - printf(" Syncing repo") - sync_all_repo(True) - - printf(" Updating databases and pending files lists: minute %s \n" % minute() ) - for repo in repo_list: - for arch in arch_list: - printf( "\n" + repo + "-" + arch + "\n" ) - printf( "Get info: minute %s " % minute() ) - info=get_info(repo,arch) - printf( "Make pending: minute %s" % minute() ) - make_pending(repo,arch,info) - printf( "Update DB: minute %s" % minute() ) - remove_from_blacklist( - repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) - - printf("Adding Parabola Packages: minute %s\n" % minute() ) - add_free_repo(True) - - printf("Extracting licenses in pending: minute %s" % minute() ) - get_licenses() - - printf("\n\nDelay: %s minutes \n" % minute()) - + from time import time + start_time = time() + def minute(): + return str(round((time() - start_time)/60, 1)) + + printf(" Cleaning %s folder " % (tmp) ) + commands.getoutput("rm -r %s/*" % tmp) + printf(" Syncing repo") + sync_all_repo(True) + + printf(" Updating databases and pending files lists: minute %s \n" % minute() ) + for repo in repo_list: + for arch in arch_list: + printf( "\n" + repo + "-" + arch + "\n" ) + printf( "Get info: minute %s " % minute() ) + info=get_info(repo,arch) + printf( "Make pending: minute %s" % minute() ) + make_pending(repo,arch,info) + printf( "Update DB: minute %s" % minute() ) + remove_from_blacklist( + repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) + + printf("Adding Parabola Packages: minute %s\n" % minute() ) + add_free_repo(True) + + printf("Extracting licenses in pending: minute %s" % minute() ) + get_licenses() + + printf("\n\nDelay: %s minutes \n" % minute()) + -- cgit v1.2.3 From df9a29ad9051f89658874959ebbc2f538e565771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 16 Feb 2011 19:31:30 -0300 Subject: Applied changes for Parabola --- config | 22 +++++++++++----------- db-functions | 33 ++++++++++++++++++++++++--------- db-update | 9 +++------ 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/config b/config index 53191e0..e55958d 100644 --- a/config +++ b/config @@ -1,24 +1,24 @@ -FTP_BASE="/srv/ftp" -SVNREPO='' -PKGREPOS=() -PKGPOOL='' -SRCPOOL='' +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' -CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=14 +SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$HOME/staging" -TMPDIR="/srv/tmp" +STAGING="$FTP_BASE/staging" +TMPDIR="$HOME/tmp" ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" @@ -29,4 +29,4 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local -[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" diff --git a/db-functions b/db-functions index 7d431fc..0553188 100644 --- a/db-functions +++ b/db-functions @@ -74,7 +74,7 @@ in_array() { script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(stat -c %U $LOCKDIR)" error "Script $(basename $0) is already locked by $_owner." exit 1 else @@ -160,7 +160,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -193,7 +193,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -358,9 +358,24 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 + +# Decide whether to look for PKGBUILD on [libre] or [libre-testing] + case $repo in + testing) + altrepo=libre-testing + ;; + *) + altrepo=libre + ;; + esac + + cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + + [[ $? -ge 1 ]] && { + echo "Failed $_pkgbase-$_pkgver-$_pkgarch" + return 1 + } fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) @@ -443,7 +458,7 @@ set_repo_permission() { local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" if [ -w "${dbfile}" ]; then - local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" else @@ -458,7 +473,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -474,7 +489,7 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-update b/db-update index 5bdce5e..4740809 100755 --- a/db-update +++ b/db-update @@ -35,12 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" - fi - if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" - fi + #if ! check_pkgrepos "${pkg}"; then + # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + #fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then die "Missing split packages for ${repo}" -- cgit v1.2.3 From 11db46275d14c9a9c2e59019fed3de4b8803f9f4 Mon Sep 17 00:00:00 2001 From: Parabola Date: Fri, 18 Feb 2011 15:42:59 -0800 Subject: Parabola specific changes --- config | 24 ++++++++++++------------ db-functions | 33 ++++++++++++++++++++++++--------- db-update | 9 +++------ 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/config b/config index 53191e0..720d110 100644 --- a/config +++ b/config @@ -1,25 +1,25 @@ -FTP_BASE="/srv/ftp" -SVNREPO='' -PKGREPOS=() -PKGPOOL='' -SRCPOOL='' +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' -CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=14 +SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$HOME/staging" -TMPDIR="/srv/tmp" -ARCHES=(i686 x86_64) +STAGING="$FTP_BASE/staging" +TMPDIR="$HOME/tmp" +ARCHES=(i686 x86_64 mipsel) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" @@ -29,4 +29,4 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local -[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" diff --git a/db-functions b/db-functions index 7d431fc..0553188 100644 --- a/db-functions +++ b/db-functions @@ -74,7 +74,7 @@ in_array() { script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(stat -c %U $LOCKDIR)" error "Script $(basename $0) is already locked by $_owner." exit 1 else @@ -160,7 +160,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -193,7 +193,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -358,9 +358,24 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 + +# Decide whether to look for PKGBUILD on [libre] or [libre-testing] + case $repo in + testing) + altrepo=libre-testing + ;; + *) + altrepo=libre + ;; + esac + + cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + + [[ $? -ge 1 ]] && { + echo "Failed $_pkgbase-$_pkgver-$_pkgarch" + return 1 + } fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) @@ -443,7 +458,7 @@ set_repo_permission() { local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" if [ -w "${dbfile}" ]; then - local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" else @@ -458,7 +473,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -474,7 +489,7 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-update b/db-update index 5bdce5e..4740809 100755 --- a/db-update +++ b/db-update @@ -35,12 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" - fi - if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" - fi + #if ! check_pkgrepos "${pkg}"; then + # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + #fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then die "Missing split packages for ${repo}" -- cgit v1.2.3 From 439bd2112c01bc9b185d2552aa0888d7055929e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 18 Feb 2011 20:43:56 -0300 Subject: First stab at sourceballing everything --- config | 2 +- config.local.gerolde | 4 ---- config.local.sigurd | 4 ---- cron-jobs/sourceballs | 13 ++++++++----- 4 files changed, 9 insertions(+), 14 deletions(-) delete mode 100644 config.local.gerolde delete mode 100644 config.local.sigurd diff --git a/config b/config index e55958d..720d110 100644 --- a/config +++ b/config @@ -19,7 +19,7 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64) +ARCHES=(i686 x86_64 mipsel) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" diff --git a/config.local.gerolde b/config.local.gerolde deleted file mode 100644 index 4501a93..0000000 --- a/config.local.gerolde +++ /dev/null @@ -1,4 +0,0 @@ -PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' -SVNREPO='file:///srv/svn-packages' diff --git a/config.local.sigurd b/config.local.sigurd deleted file mode 100644 index d28aa37..0000000 --- a/config.local.sigurd +++ /dev/null @@ -1,4 +0,0 @@ -PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing') -PKGPOOL='pool/community' -SRCPOOL='sources/community' -SVNREPO='file:///srv/svn-packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b55de05..0df007e 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -63,10 +63,11 @@ for repo in ${PKGREPOS[@]}; do if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi + # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package - if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then - continue - fi +# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then +# continue +# fi # Store the expected file name of the source package echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" @@ -79,8 +80,10 @@ for repo in ${PKGREPOS[@]}; do # Get the sources from svn mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" continue -- cgit v1.2.3 From f8c49cc2915f66b53bd9dd248b41ab6a96c1059a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 20 Feb 2011 22:09:28 -0300 Subject: Changes to sourceball everything --- config | 1 + cron-jobs/sourceballs | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/config b/config index 720d110..12bdaaa 100644 --- a/config +++ b/config @@ -1,4 +1,5 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') PKGPOOL='pool/packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..5726484 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -59,7 +59,7 @@ for repo in ${PKGREPOS[@]}; do pkgarch=${pkginfo[2]} pkglicense=(${pkginfo[@]:3}) - # Should this package be skipped? + # Should this packages be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi @@ -82,6 +82,9 @@ for repo in ${PKGREPOS[@]}; do mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + + # If it's on official repos, nor [libre], nor [libre-testing] + cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then @@ -93,7 +96,7 @@ for repo in ${PKGREPOS[@]}; do pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done -- cgit v1.2.3 From 0162a16595059e269ba2d7b47e69212bc2ab1336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 20 Feb 2011 22:09:40 -0300 Subject: Changed looking for a package on abslibre logic --- db-functions | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/db-functions b/db-functions index 0553188..e2731cb 100644 --- a/db-functions +++ b/db-functions @@ -1,5 +1,7 @@ #!/bin/bash +. config + # Some PKGBUILDs need CARCH to be set CARCH="x86_64" @@ -16,7 +18,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d ${TMPDIR}/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color @@ -359,18 +361,9 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" -# Decide whether to look for PKGBUILD on [libre] or [libre-testing] - case $repo in - testing) - altrepo=libre-testing - ;; - *) - altrepo=libre - ;; - esac - cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + cp -r ${SVNREPO}/libre/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 [[ $? -ge 1 ]] && { echo "Failed $_pkgbase-$_pkgver-$_pkgarch" -- cgit v1.2.3 From c6516eca2803b780a01a6b2bf3e081c8bd570301 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 20 Feb 2011 17:10:40 -0800 Subject: sourceballs --- config | 1 + cron-jobs/sourceballs | 12 ++++++------ db-functions | 5 ++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config b/config index 720d110..12bdaaa 100644 --- a/config +++ b/config @@ -1,4 +1,5 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') PKGPOOL='pool/packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..53d43ce 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -91,9 +91,9 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +129,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/db-functions b/db-functions index 0553188..e790fb6 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /home/parabolavnx/tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color @@ -376,6 +376,9 @@ check_splitpkgs() { echo "Failed $_pkgbase-$_pkgver-$_pkgarch" return 1 } + + sleep 20s + fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) -- cgit v1.2.3 From 9793aaf4e9a716f126e71a1db412cbd32f161593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Thu, 3 Mar 2011 20:11:33 -0600 Subject: * repo-maintainer doesn't take out reading permission from pending files * Removed commented code --- pato2.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pato2.py b/pato2.py index f5deb48..21dde16 100644 --- a/pato2.py +++ b/pato2.py @@ -128,12 +128,6 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): printf(com_) a = commands.getoutput(com_) if verbose: printf(a) - if pack_: - pack_=" ".join(pack_) - com_="chmod a-r " + pack_ - printf(com_) - a=commands.getoutput(com_) - if verbose: printf(a) def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ @@ -152,11 +146,9 @@ def add_free_repo(verbose_=verbose): lista_=list() for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): lista_.append(file_) - # link(repo_,arch_,file_) for dir_ in other: for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): lista_.append(file_) - # link(repo_,arch_,file_) printf(lista_) -- cgit v1.2.3 From cf2baeb05af80a849bfc2192f3af1e741e50745d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 5 Mar 2011 01:25:40 -0300 Subject: sourceballs2 creates sourceballs from the entire ABSLibre --- config | 6 +--- cron-jobs/sourceballs2 | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 cron-jobs/sourceballs2 diff --git a/config b/config index 12bdaaa..f08ad2d 100644 --- a/config +++ b/config @@ -26,8 +26,4 @@ FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" -# Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') - -# Override default config with config.local -#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +MAKEPKGCONF="$HOME/etc/makepkg.conf" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100644 index 0000000..ba35298 --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,81 @@ +#!/bin/bash +# Steps +# Traverse ABSLibre +# Makepkg --allsource every package +# Remove the old sourceballs + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +pushd "${SVNREPO}" >/dev/null + +failedpkgs=() +for repo in ${PKGREPOS[@]}; do + pushd $repo >/dev/null + find . -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + + unset pkgbase pkgname + source PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ + continue + + makepkg --allsource --ignorearch -c + + [[ $? -ne 0 ]] && \ + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + done + popd >/dev/null +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +script_unlock + -- cgit v1.2.3 From a3694fd2824f6aab2905f52f64a138437f7e004b Mon Sep 17 00:00:00 2001 From: Parabola Date: Sat, 5 Mar 2011 10:56:49 -0800 Subject: =?UTF-8?q?Taken=20out=20=C2=ABsources=C2=BB=20from=20dir=5Flist?= =?UTF-8?q?=20for=20rsync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 6420355..b198450 100644 --- a/config.py +++ b/config.py @@ -27,9 +27,10 @@ archdb = tmp + "/db" free_path= path + "/free/" # Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") +# This are tuples, so **always keep a comma before closing parenthesis ** +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib",) +dir_list = ("pool",) +arch_list = ("i686", "x86_64",) other = ("any",) # Output -- cgit v1.2.3 From 5a4480f1f3e2e7d0aec965b21f72f4454618333e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 5 Mar 2011 13:13:41 -0600 Subject: =?UTF-8?q?*=20Added=20=C2=ABdepends=C2=BB=20field=20to=20Package?= =?UTF-8?q?=20class.=20*=20Merged=20tests=20for=20filter=20functions=20in?= =?UTF-8?q?=20test=5Ffilter.py=20*=20Added=20testfiles=20for=20test=5Ffilt?= =?UTF-8?q?er.py=20in=20test=20directory=20*=20Added=20pkginfo=5Ffrom=5Fde?= =?UTF-8?q?sc=20function=20to=20filter.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 3 +- filter.py | 35 ++++++- test/core.db.tar.gz | Bin 0 -> 1637 bytes test/depends | 4 + test/desc | 39 ++++++++ test/test_filter.py | 169 +++++++++++++++++++++++++++++++++ test/test_pkginfo_from_file.py | 76 --------------- test/test_pkginfo_from_rsync_output.py | 65 ------------- 8 files changed, 248 insertions(+), 143 deletions(-) create mode 100644 test/core.db.tar.gz create mode 100644 test/depends create mode 100644 test/desc create mode 100644 test/test_filter.py delete mode 100644 test/test_pkginfo_from_file.py delete mode 100644 test/test_pkginfo_from_rsync_output.py diff --git a/config.py b/config.py index 6420355..688d6b5 100644 --- a/config.py +++ b/config.py @@ -62,7 +62,8 @@ class Package: "release" : False, "arch" : False, "license" : False, - "location": False} + "location": False, + "depends" : False,} def __setitem__(self, key, item): if key in self.package_info.keys(): diff --git a/filter.py b/filter.py index f91ec68..2fe61f0 100644 --- a/filter.py +++ b/filter.py @@ -26,6 +26,37 @@ def pkginfo_from_filename(filename): pkg["name"] = "-".join(fileattrs) return pkg +def pkginfo_from_desc(filename): + """ Returns pkginfo from desc file. + + Parameters: + ---------- + filename -> str File must exist + + Returns: + ---------- + pkg -> Package object""" + if not os.path.isfile(filename): + raise NonValidFile + try: + f=open(filename) + info=f.read().rsplit() + finally: + f.close() + pkg = Package() + info_map={"name" :("%NAME%" , None), + "version" :("%VERSION%" , 0 ), + "release" :("%VERSION%" , 1 ), + "arch" :("%ARCH%" , None), + "license" :("%LICENSE%" , None), + "location":("%FILENAME%", None),} + + for key in info_map.keys(): + field,pos=info_map[key] + pkg[key]=info[info.index(field)+1] + if pos is not None: + pkg[key]=pkg[key].split("-")[pos] + return pkg def pkginfo_from_rsync_output(rsync_output): """ Generates a list of packages and versions from an rsync output @@ -45,7 +76,7 @@ def pkginfo_from_rsync_output(rsync_output): return pkginfo_from_filename(line.rsplit()[location_field]) def do_nothing(): - """""" + pass options = { "d": do_nothing, "l": package_or_link, @@ -84,6 +115,8 @@ def pkginfo_from_files_in_dir(directory): package_list.append(pkginfo_from_filename(filename)) return tuple(package_list) + + def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz new file mode 100644 index 0000000..a28ea64 Binary files /dev/null and b/test/core.db.tar.gz differ diff --git a/test/depends b/test/depends new file mode 100644 index 0000000..7ff3ad4 --- /dev/null +++ b/test/depends @@ -0,0 +1,4 @@ +%DEPENDS% +glibc>=2.13 +zlib + diff --git a/test/desc b/test/desc new file mode 100644 index 0000000..abba644 --- /dev/null +++ b/test/desc @@ -0,0 +1,39 @@ +%FILENAME% +binutils-2.21-4-x86_64.pkg.tar.xz + +%NAME% +binutils + +%VERSION% +2.21-4 + +%DESC% +A set of programs to assemble and manipulate binary and object files + +%GROUPS% +base + +%CSIZE% +3412892 + +%ISIZE% +17571840 + +%MD5SUM% +4e666f87c78998f4839f33dc06d2043a + +%URL% +http://www.gnu.org/software/binutils/ + +%LICENSE% +GPL + +%ARCH% +x86_64 + +%BUILDDATE% +1297240369 + +%PACKAGER% +Allan McRae + diff --git a/test/test_filter.py b/test/test_filter.py new file mode 100644 index 0000000..5127b75 --- /dev/null +++ b/test/test_filter.py @@ -0,0 +1,169 @@ +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class pkginfo_from_file_KnownValues(unittest.TestCase): + # (filename, name, version, release, arch) + # filename is location + known=( + ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), + ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), + ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), + ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), + ) + + def generate_results(self, example_tuple, attr): + location, name, version, release, arch = example_tuple + return pkginfo_from_filename(location)[attr], locals()[attr] + + def testReturnPackageObject(self): + for i in self.known: + location, name, version, release, arch = i + self.assertIsInstance(pkginfo_from_filename(location),Package) + + def testNames(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +class pkginfo_from_file_BadInput(unittest.TestCase): + bad=("community-testing/os/i686/community-testing.db", + "community-testing/os/i686/community-testing.db.tar.gz", + "community-testing/os/i686/community-testing.db.tar.gz.old", + "community-testing/os/i686/community-testing.files", + "community-testing/os/i686/community-testing.files.tar.gz", + "community-testing/os/x86_64") + + def testBadInput(self): + for i in self.bad: + self.assertRaises(NonValidFile,pkginfo_from_filename,i) + +class pkginfoFromRsyncOutput(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + try: + output_file = open("rsync_output_sample") + rsync_out= output_file.read() + output_file.close() + except IOError: print("There is no rsync_output_sample file") + + pkglist = pkginfo_from_rsync_output(rsync_out) + + def testOutputArePackages(self): + if not self.pkglist: + self.fail("not pkglist:" + str(self.pkglist)) + for pkg in self.pkglist: + self.assertIsInstance(pkg,Package) + + def testPackageInfo(self): + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,self.example_package_list) + +class generateRsyncBlacklist(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=generate_exclude_list_from_blacklist(self.example_package_list,listado("blacklist_sample"),debug=True) + b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] + self.assertEqual(a,b) + +class pkginfo_from_descKnownValues(unittest.TestCase): + pkgsample=Package() + pkgsample.package_info={"name" : "binutils", + "version" : "2.21", + "release" : "4", + "arch" : "x86_64", + "license" : "GPL", + "location": "binutils-2.21-4-x86_64.pkg.tar.xz", + "depends" : False,} + pkggen=pkginfo_from_desc("desc") + def testPkginfoFromDesc(self): + if self.pkggen is None: + self.fail("return value is None") + self.assertEqual(self.pkgsample,self.pkggen) + +class pkginfo_from_db(unittest.TestCase): + archdb = os.path.join("./workdir") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_pkginfo_from_file.py b/test/test_pkginfo_from_file.py deleted file mode 100644 index 0d7ede6..0000000 --- a/test/test_pkginfo_from_file.py +++ /dev/null @@ -1,76 +0,0 @@ -#! /usr/bin/python -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -class KnownValues(unittest.TestCase): - # (filename, name, version, release, arch) - # filename is location - known=( - ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), - ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), - ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), - ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), - ) - - def generate_results(self, example_tuple, attr): - location, name, version, release, arch = example_tuple - return pkginfo_from_filename(location)[attr], locals()[attr] - - def testReturnPackageObject(self): - for i in self.known: - location, name, version, release, arch = i - self.assertIsInstance(pkginfo_from_filename(location),Package) - - def testNames(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) - -class BadInput(unittest.TestCase): - bad=("community-testing/os/i686/community-testing.db", - "community-testing/os/i686/community-testing.db.tar.gz", - "community-testing/os/i686/community-testing.db.tar.gz.old", - "community-testing/os/i686/community-testing.files", - "community-testing/os/i686/community-testing.files.tar.gz", - "community-testing/os/x86_64") - - def testBadInput(self): - for i in self.bad: - self.assertRaises(NonValidFile,pkginfo_from_filename,i) - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py deleted file mode 100644 index aca49e8..0000000 --- a/test/test_pkginfo_from_rsync_output.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -example_package_list=(Package(),Package(),Package()) -example_package_list[0].package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} -example_package_list[1].package_info={ "name" : "any2dvd", - "version" : "0.34", - "release" : "4", - "arch" : "any", - "license" : False, - "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz"} -example_package_list[2].package_info={ "name" : "gmime22", - "version" : "2.2.26", - "release" : "1", - "arch" : "x86_64", - "license" : False, - "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz"} - -class pkginfoFromRsyncOutput(unittest.TestCase): - try: - output_file = open("rsync_output_sample") - rsync_out= output_file.read() - output_file.close() - except IOError: print("There is no rsync_output_sample file") - - pkglist = pkginfo_from_rsync_output(rsync_out) - - def testOutputArePackages(self): - if not self.pkglist: - self.fail("not pkglist:" + str(self.pkglist)) - for pkg in self.pkglist: - self.assertIsInstance(pkg,Package) - - def testPackageInfo(self): - if not self.pkglist: - self.fail("Pkglist doesn't exist: " + str(self.pkglist)) - self.assertEqual(self.pkglist,example_package_list) - -class generateRsyncBlacklist(unittest.TestCase): - """ Test Blacklist generation """ - def testListado(self): - self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) - - def testExcludeFiles(self): - a=generate_exclude_list_from_blacklist(example_package_list,listado("blacklist_sample"),debug=True) - b=[example_package_list[0]["location"],example_package_list[2]["location"]] - self.assertEqual(a,b) - -if __name__ == "__main__": - unittest.main() -- cgit v1.2.3 From 8c9ff0c97d0802fadc96fe9eacaef7eaa2db5ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 16:37:36 -0600 Subject: * Using os.path.join for joining path instead of str + str to avoid errors --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 21dde16..936e97a 100644 --- a/pato2.py +++ b/pato2.py @@ -137,7 +137,7 @@ def link(repo_,arch_,file_): printf(cmd_ + a) def add_free_repo(verbose_=verbose): - cmd_=home + "/usr/bin/sync-free" + cmd_=os.path.join(home,"/usr/bin/sync-free") printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) -- cgit v1.2.3 From 31a0d9e077ed10c5a4b9f461bbde8979f127bdf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 16:43:18 -0600 Subject: * Changed rsync-update-command to not delete (delete made by ftp-dir-cleanup) --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index ada17f0..37bebc0 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delete-after --delay-updates " +rsync_update_command="rsync -av --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3 From 229a9c504cbd733c93cf91399dc54bedf5160cc5 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 6 Mar 2011 15:22:21 -0800 Subject: Fixes --- config | 8 ++--- cron-jobs/sourceballs2 | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ db-functions | 4 +-- 3 files changed, 90 insertions(+), 9 deletions(-) create mode 100755 cron-jobs/sourceballs2 diff --git a/config b/config index 12bdaaa..217b627 100644 --- a/config +++ b/config @@ -20,14 +20,10 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64 mipsel) +ARCHES=(i686 x86_64 mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" -# Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') - -# Override default config with config.local -#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +MAKEPKGCONF="$HOME/etc/makepkg.conf" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100755 index 0000000..452208e --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,87 @@ +#!/bin/bash + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Steps +# Traverse the ABSLibre +# Makepkg --allsource every package +# Remove the old packages +pushd "${SVNREPO}" >/dev/null + +failedpkgs=() +for repo in ${PKGREPOS[@]}; do + pushd $repo >/dev/null + find . -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + + unset pkgbase pkgname + source PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ + continue + + makepkg --allsource --ignorearch -c >/dev/null 2>&1 + + [[ $? -ne 0 ]] && \ + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + done + popd >/dev/null +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +msg "Failed" +for _fail in ${failedpkgs[@]}; do + msg2 "$_fail" +done + + +script_unlock + diff --git a/db-functions b/db-functions index e2731cb..b900669 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,5 @@ #!/bin/bash -. config - # Some PKGBUILDs need CARCH to be set CARCH="x86_64" @@ -18,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d ${TMPDIR}/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color -- cgit v1.2.3 From e2385920015311376c1631a42a1ca8f1c45dae4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 18:08:25 -0600 Subject: * Fixed error that leaked non-free files to repo --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 936e97a..336c6fb 100644 --- a/pato2.py +++ b/pato2.py @@ -56,7 +56,7 @@ def sync_all_repo(debug=verbose): cmd=generate_rsync_command(rsync_list_command) rsout=run_rsync(cmd) pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) a=run_rsync(cmd) if debug: printf(a) -- cgit v1.2.3 From 58a67fb771fbbf433fe3ab2dafb9194e6f635f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 18:09:52 -0600 Subject: * Added --delete to rsync-update-command until ftp-dir-cleanup can do delete --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 37bebc0..ada17f0 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delay-updates " +rsync_update_command="rsync -av --delete-after --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3 From 125df5a815e0a327739e928cc765ffdcf4bd0aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 18:19:03 -0600 Subject: * Added function for cleanup dir --- pato2.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pato2.py b/pato2.py index 336c6fb..03afebd 100644 --- a/pato2.py +++ b/pato2.py @@ -129,6 +129,12 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): a = commands.getoutput(com_) if verbose: printf(a) +def cleanup_nonfree_in_dir(directory,blacklisted_names): + pkgs=pkginfo_from_files_in_dir(directory) + for package in pkgs: + if package["name"] in blacklisted_names: + os.remove(package["location"]) + def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ -- cgit v1.2.3 From 15ed2078e8743f87efcd63ac62cc0bfd5b39dc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 18:34:28 -0600 Subject: * Corrected import glob function --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.py b/filter.py index 2fe61f0..b6e8105 100644 --- a/filter.py +++ b/filter.py @@ -1,6 +1,6 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- -import glob +from glob import glob from repm.config import * from repm.pato2 import * -- cgit v1.2.3 From a41f5f44a8f812dfa2aacaf359bb51782d7a1633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 19:02:14 -0600 Subject: * Fixed blacklist for rsync_command at sync_all_repo --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 03afebd..18ace85 100644 --- a/pato2.py +++ b/pato2.py @@ -57,7 +57,7 @@ def sync_all_repo(debug=verbose): rsout=run_rsync(cmd) pkgs=pkginfo_from_rsync_output(rsout) generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) a=run_rsync(cmd) if debug: printf(a) -- cgit v1.2.3 From 0889827f252c9f6c21c1d4f2afc704b9b303e083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Thu, 10 Mar 2011 13:59:35 -0600 Subject: * Fixed issue 71: pending list ends with newline --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 18ace85..27a3962 100644 --- a/pato2.py +++ b/pato2.py @@ -111,7 +111,7 @@ def make_pending(repo_,arch_,info_): printf( pkg_ + " package has no %NAME% attibute " ) if verbose: printf( lista_ ) a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + "\n".join([name + ":" + license_ for (name,license_) in lista_]) + "\n") def remove_from_blacklist(repo_,arch_,info_,blacklist_): """ Check the blacklist and remove packages on the db""" -- cgit v1.2.3 From 47045b1934932b0695dd301a9c76b9dab1b03023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 21 Mar 2011 17:30:40 -0600 Subject: * Changed sync_all_repo to adress bug83 --- config.py | 3 ++- filter.py | 3 ++- pato2.py | 6 +++++- test/core.db.tar.gz | Bin 1637 -> 1345 bytes test/test_filter.py | 23 +++++++++++++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index ada17f0..c218ddd 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delete-after --delay-updates " +rsync_update_command="rsync -av --delay-updates --exclude='*.db.tar.{gz|xz}' " +rsync_post_command="rsync -av --delete " # Classes and Exceptions class NonValidFile(ValueError): pass diff --git a/filter.py b/filter.py index b6e8105..668822b 100644 --- a/filter.py +++ b/filter.py @@ -115,7 +115,8 @@ def pkginfo_from_files_in_dir(directory): package_list.append(pkginfo_from_filename(filename)) return tuple(package_list) - +def pkginfo_from_db(path_to_db): + """ """ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): diff --git a/pato2.py b/pato2.py index 27a3962..0d77d6b 100644 --- a/pato2.py +++ b/pato2.py @@ -59,7 +59,11 @@ def sync_all_repo(debug=verbose): generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) a=run_rsync(cmd) - if debug: printf(a) + cmd=generate_rsync_command(rsync_post_command,blacklist_file=rsync_blacklist) + b=run_rsync(cmd) + if debug: + printf(a) + printf(b) def get_from_desc(desc, var,db_tar_file=False): """ Get a var from desc file """ diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz index a28ea64..5eb2081 100644 Binary files a/test/core.db.tar.gz and b/test/core.db.tar.gz differ diff --git a/test/test_filter.py b/test/test_filter.py index 5127b75..1906b87 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -163,7 +163,30 @@ class pkginfo_from_descKnownValues(unittest.TestCase): class pkginfo_from_db(unittest.TestCase): archdb = os.path.join("./workdir") + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "acl", + "version" : "2.2.49", + "release" : "2", + "arch" : "x86_64", + "license" : ("LGPL",), + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz" + "depends" : ("attr>=2.4.41"),} + example_package_list[1].package_info={ "name" : "glibc", + "version" : "2.13", + "release" : "4", + "arch" : "x86_64", + "license" : ("GPL","LGPL"), + "location": "glibc-2.13-4-x86_64.pkg.tar.xz" + "depends" : ("linux-api-headers>=2.6.37","tzdata",),} + example_package_list[2].package_info={ "name" : "", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "" + "depends" : False,} if __name__ == "__main__": unittest.main() + -- cgit v1.2.3 From fa11d216d567cec3c96c964829c51bd923b493c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 21 Mar 2011 17:49:24 -0600 Subject: * Don't update *.abs.tar.* --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index c218ddd..c643bb5 100644 --- a/config.py +++ b/config.py @@ -46,8 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delay-updates --exclude='*.db.tar.{gz|xz}' " -rsync_post_command="rsync -av --delete " +rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " +rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3 From 72d3e1102acf8e90a1646d5c9afa6fdd01b2aeca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 8 Apr 2011 11:40:32 -0500 Subject: Config ported to bash --- config.sh | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100755 config.sh diff --git a/config.sh b/config.sh new file mode 100755 index 0000000..aae3287 --- /dev/null +++ b/config.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# -*- coding: utf-8 -*- + +# Mirror options +mirror="mirrors.eu.kernel.org" +mirrorpath="::mirrors/archlinux" + +# Directories and files + +## Optionals +paraboladir=~/parabolagnulinux.org +logtime=$(date -u +%Y%m%d-%H:%M) + +## Must be defined +logname=${paraboladir}/${logtime}-repo-maintainer.log +tempdir=~/tmp/ +docs_dir=${paraboladir}/docs +repodir=${paraboladir}/repo + +# Repos, arches, and dirs for repo +repolist="core:extra:community:testing:community-testing:multilib" +dir_list="pool" +arch_list="i686:x86_64" +other="any" + +# Output options +output="True" +debug="False" + +# Rsync commands +rsync_list_command="rsync -a --no-motd --list-only " +rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " +rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " + + +function run_python_cmd { + env \ + mirror=${mirror} \ + mirrorpath=${mirrorpath} \ + logname=${logname} \ + tempdir=${tempdir} \ + docs_dir=${docs_dir} \ + repodir=${repodir} \ + repolist=${repolist} \ + dir_list=${dir_list} \ + arch_list=${arch_list} \ + other=${other} \ + output=${output} \ + debug=${debug} \ + $1 +} \ No newline at end of file -- cgit v1.2.3 From 2b721124f5122faae5709e5e2dd8035f392c1361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 10 Apr 2011 15:18:17 -0500 Subject: config.py import config.sh settings --- config.py | 72 ++++++++++++++++++++++++++++++--------------------------------- config.sh | 6 +++--- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/config.py b/config.py index bd8ef1b..56a7c81 100644 --- a/config.py +++ b/config.py @@ -1,44 +1,36 @@ -#!/usr/bin/python +#!/usr/bin/pythonn # -*- coding: utf-8 -*- -from user import home -import commands +try: + from subprocess import check_output +except(ImportError): + from commands import getoutput as check_output +import os -time__ = commands.getoutput("date +%Y%m%d-%H:%M") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", + "repodir",) +listvars=("repo_list", "dir_list", "arch_list", "other",) +boolvars=("output", "debug",) -# Mirror Parameters -mirror = "mirrors.eu.kernel.org" -mirrorpath = "::mirrors/archlinux" +config=dict() -# Directories and files +def exit_if_none: + if os.environ.get(var) is None: + exit("%s is not defined" % var) -## Optionals -path = home + "/parabolagnulinux.org" -docs = path + "/docs" -logdir = path + "/log" - -## Must be defined -logname= logdir + "/" + time__ + "-repo-maintainer.log" -repodir= path + "/repo" -tmp = home + "/tmp" -archdb = tmp + "/db" - -free_path= path + "/free/" - -# Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") -other = ("any",) - -# Output -output = True -verbose = False - -# Files -blacklist = docs + "/blacklist.txt" -whitelist = docs + "/whitelist.txt" -pending = docs + "/pending" -rsync_blacklist = docs + "/rsyncBlacklist" +for var in stringvars: + exit_if_none(var) + config[var]=os.environ.get(var) +for var in listvars: + exit_if_none(var) + config[var]=tuple(os.environ.get(var).split(":")) +for var in boolvars: + exit_if_none(var) + if os.environ.get(var) == "True": + config[var]=True + elif os.environ.get(var) =="False": + config[var]=False + else: + print('%s is not True or False' % var) # Classes and Exceptions class NonValidFile(ValueError): pass @@ -55,7 +47,8 @@ class Package: "release" : False, "arch" : False, "license" : False, - "location": False} + "location": False, + "depends" : False,} def __setitem__(self, key, item): if key in self.package_info.keys(): @@ -76,8 +69,11 @@ class Package: if not isinstance(x, Package): return False for key in self.package_info.keys(): - if x[key] != self[key]: + if x[key] != self.package_info[key]: return False else: return True +if __name__=="__main__": + for key in config.keys(): + print("%s : %s" % (key,config[key])) diff --git a/config.sh b/config.sh index aae3287..66be477 100755 --- a/config.sh +++ b/config.sh @@ -1,6 +1,7 @@ #!/bin/sh # -*- coding: utf-8 -*- + # Mirror options mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" @@ -18,7 +19,7 @@ docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # Repos, arches, and dirs for repo -repolist="core:extra:community:testing:community-testing:multilib" +repo_list="core:extra:community:testing:community-testing:multilib" dir_list="pool" arch_list="i686:x86_64" other="any" @@ -28,7 +29,6 @@ output="True" debug="False" # Rsync commands -rsync_list_command="rsync -a --no-motd --list-only " rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " @@ -41,7 +41,7 @@ function run_python_cmd { tempdir=${tempdir} \ docs_dir=${docs_dir} \ repodir=${repodir} \ - repolist=${repolist} \ + repo_list=${repo_list} \ dir_list=${dir_list} \ arch_list=${arch_list} \ other=${other} \ -- cgit v1.2.3 From 69d84cc6edab249037522d00d3685939dc38495a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 10 Apr 2011 17:59:47 -0500 Subject: Firs main deliver --- config.py | 2 +- config.sh | 10 +++++--- libremessages | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.sh | 21 ++++++++++++++++ 4 files changed, 106 insertions(+), 4 deletions(-) create mode 100755 libremessages create mode 100644 main.sh diff --git a/config.py b/config.py index 39f512a..8cf07a2 100644 --- a/config.py +++ b/config.py @@ -7,7 +7,7 @@ except(ImportError): import os stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir",) + "repodir", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) diff --git a/config.sh b/config.sh index 66be477..60bd4ea 100755 --- a/config.sh +++ b/config.sh @@ -17,6 +17,7 @@ logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo +rsync_blacklist=${docs_dir}/rsyncBlacklist # Repos, arches, and dirs for repo repo_list="core:extra:community:testing:community-testing:multilib" @@ -29,8 +30,8 @@ output="True" debug="False" # Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " -rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " function run_python_cmd { @@ -39,6 +40,7 @@ function run_python_cmd { mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ + rsync_blacklist=${rsync_blacklist} \ docs_dir=${docs_dir} \ repodir=${repodir} \ repo_list=${repo_list} \ @@ -48,4 +50,6 @@ function run_python_cmd { output=${output} \ debug=${debug} \ $1 -} \ No newline at end of file +} + +source libremessages \ No newline at end of file diff --git a/libremessages b/libremessages new file mode 100755 index 0000000..9fbbc2b --- /dev/null +++ b/libremessages @@ -0,0 +1,77 @@ +# Copyright (c) 2006-2010 Pacman Development Team +# Copyright (c) 2002-2006 by Judd Vinet +# Copyright (c) 2005 by Aurelien Foret +# Copyright (c) 2006 by Miklos Vajna +# Copyright (c) 2005 by Christian Hamar +# Copyright (c) 2006 by Alex Smith +# Copyright (c) 2006 by Andras Voroskoi +# Copyright (c) 2011 by Joshua Haase +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# gettext initialization +export TEXTDOMAIN='libretools' +export TEXTDOMAINDIR='/usr/share/locale' + +# check if messages are to be printed using color +unset ALL_OFF BOLD BLUE GREEN RED YELLOW + +if tput setaf 0 &>/dev/null; then + ALL_OFF="$(tput sgr0)" + BOLD="$(tput bold)" + BLUE="${BOLD}$(tput setaf 4)" + GREEN="${BOLD}$(tput setaf 2)" + RED="${BOLD}$(tput setaf 1)" + YELLOW="${BOLD}$(tput setaf 3)" + PURPLE="${ALL_OFF}$(tput setaf 5)" +else + ALL_OFF="\033[1;0m" + BOLD="\033[1;1m" + BLUE="${BOLD}\033[1;34m" + GREEN="${BOLD}\033[1;32m" + RED="${BOLD}\033[1;31m" + YELLOW="${BOLD}\033[1;33m" + PURPLE="${BOLD}\033[1;30;40m" +fi + +stdnull() { + local action=$1; + eval "${action} >/dev/null 2>&1" +} + +plain() { + local mesg=$1; shift + printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg() { + local mesg=$1; shift + printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg2() { + local mesg=$1; shift + printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +warning() { + local mesg=$1; shift + printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + diff --git a/main.sh b/main.sh new file mode 100644 index 0000000..ac504a4 --- /dev/null +++ b/main.sh @@ -0,0 +1,21 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +source config.sh + +function mkrsexclude { + error=1 + while ${error}; do + run_python_cmd "filter.py" + error=$? + done +} + +msg "Cleaning $tempdir" +stdnull "rm -r $tempdir/* " + +msg "Generating exclude list for rsync" +mkrsexclude + +msg "Syncing repo files without delete" +${rsync_update_command} --exclude-from=${rsync_blacklist} ${mirror}${mirropath}/ \ No newline at end of file -- cgit v1.2.3 From 61ee5da8b31b44e80e008619a32ca886d8799646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 10 Apr 2011 19:27:12 -0500 Subject: Replaced pato2.py functionality in main.sh --- main.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/main.sh b/main.sh index ac504a4..2d59094 100644 --- a/main.sh +++ b/main.sh @@ -4,7 +4,7 @@ source config.sh function mkrsexclude { - error=1 + local error=1 while ${error}; do run_python_cmd "filter.py" error=$? @@ -17,5 +17,19 @@ stdnull "rm -r $tempdir/* " msg "Generating exclude list for rsync" mkrsexclude -msg "Syncing repo files without delete" -${rsync_update_command} --exclude-from=${rsync_blacklist} ${mirror}${mirropath}/ \ No newline at end of file +msg "Syncing repos without delete" +# rsync_update_command does not sync db or abs +${rsync_update_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} + +msg "Syncing each repo and cleaning" +for repo in $(echo ${repo_list} | tr ':' ' '); do + msg2 "Syncing ${repo}" + ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo} + msg2 "Cleaning ${repo}" + clean-repo.py -d ${repodir}/${repo} \ + -b ${repodir}/${repo}/${repo}.db.tar.gz + msg2 "Making pending list for ${repo}" + run_python_cmd "mkpending.py -r ${repo} -d ${repodir}/${repo}" +done -- cgit v1.2.3 From 3fb5f8efef231dd7784be880934cd106603ab6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 01:09:25 -0500 Subject: bash-port ready for testing. --- clean_repo.py | 45 +++++++++++ config.py | 10 ++- config.sh | 17 +++-- filter.py | 68 +++++++++++++---- get_license.sh | 10 +-- main.sh | 20 +++-- mkpending.py | 45 +++++++++++ pato2.py | 209 ++++------------------------------------------------ test/test_filter.py | 6 +- 9 files changed, 198 insertions(+), 232 deletions(-) create mode 100644 clean_repo.py create mode 100644 mkpending.py diff --git a/clean_repo.py b/clean_repo.py new file mode 100644 index 0000000..29d446d --- /dev/null +++ b/clean_repo.py @@ -0,0 +1,45 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +from repm.filter import * +import argparse + +def remove_from_blacklist(path_to_db, blacklisted_names, + debug=config["debug"]): + """ Check the blacklist and remove packages on the db""" + + pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if + pkg["name"] in blacklisted_names] + if pkgs: + lista=" ".join(pkgs) + cmd = "repo-remove " + path_to_db + " " + lista + printf(cmd) + a = check_output(cmd) + if debug: + printf(a) + return pkgs, cmd + +def cleanup_nonfree_in_dir(directory, blacklisted_names): + pkgs=pkginfo_from_files_in_dir(directory) + for package in pkgs: + if package["name"] in blacklisted_names: + os.remove(package["location"]) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Clean a repo db and packages") + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean") + parser.add_argument("-d", "--directory", type=str, + help="directory to clean") + args=parser.parse_args() + + if args.directory: + cleanup_nonfree_in_dir(args.database, listado(config["blacklist"])) + + if args.database: + pkgs=pkginfo_from_db(args.database) + remove_from_blacklist(args.database, pkgs, + tuple(listado(config["blacklist"]) + + listado(config["pending"]))) + if not args.directory and not args.database: + parser.print_help() diff --git a/config.py b/config.py index 8cf07a2..24ecfaf 100644 --- a/config.py +++ b/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/pythonn +#!/usr/bin/python # -*- coding: utf-8 -*- try: from subprocess import check_output @@ -7,22 +7,24 @@ except(ImportError): import os stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir", "rsync_blacklist") + "repodir", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) config=dict() -def exit_if_none: +def exit_if_none(var): if os.environ.get(var) is None: exit("%s is not defined" % var) for var in stringvars: exit_if_none(var) - config[var]=os.environ.get(var) + config[var]=os.environ.get(var) + for var in listvars: exit_if_none(var) config[var]=tuple(os.environ.get(var).split(":")) + for var in boolvars: exit_if_none(var) if os.environ.get(var) == "True": diff --git a/config.sh b/config.sh index 60bd4ea..741dee4 100755 --- a/config.sh +++ b/config.sh @@ -1,22 +1,25 @@ #!/bin/sh # -*- coding: utf-8 -*- - # Mirror options mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" -# Directories and files - +# Directories ## Optionals paraboladir=~/parabolagnulinux.org logtime=$(date -u +%Y%m%d-%H:%M) - ## Must be defined logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo +# End Directories + +# Files +blacklist=${docs_dir}/blacklist.txt +whitelist=${docs_dir}/whitelist.txt +pending=${docs_dir}/pending rsync_blacklist=${docs_dir}/rsyncBlacklist # Repos, arches, and dirs for repo @@ -33,16 +36,18 @@ debug="False" rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " - function run_python_cmd { env \ mirror=${mirror} \ mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ - rsync_blacklist=${rsync_blacklist} \ docs_dir=${docs_dir} \ repodir=${repodir} \ + blacklist=${blacklist} \ + whitelist=${whitelist} \ + pending=${pending} \ + rsync_blacklist=${rsync_blacklist} \ repo_list=${repo_list} \ dir_list=${dir_list} \ arch_list=${arch_list} \ diff --git a/filter.py b/filter.py index 668822b..1a0fa6f 100644 --- a/filter.py +++ b/filter.py @@ -4,6 +4,13 @@ from glob import glob from repm.config import * from repm.pato2 import * +def listado(filename): + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + def pkginfo_from_filename(filename): """ Generates a Package object with info from a filename, filename can be relative or absolute @@ -116,10 +123,48 @@ def pkginfo_from_files_in_dir(directory): return tuple(package_list) def pkginfo_from_db(path_to_db): - """ """ + """ Get PKGINFO from db. + + Parameters: + ---------- + path_to_db -> str Path to file -def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, - exclude_file=rsync_blacklist, debug=verbose): + Output: + ---------- + None """ + package_list=list() + + if not os.path.isfile(path_to_db): + raise NonValidFile(path_to_db + "is not a file") + + check_output("mkdir -p " + archdb) + + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s or not readable" % db_tar_file) + return(tuple()) + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + package_list.append(pkginfo_from_desc( + os.path.join(dir_,"desc"))) + check_output("rm -r %s/*" % archdb) + if verbose_: + printf(package_list) + return package_list + +def generate_exclude_list_from_blacklist(packages_iterable, + blacklisted_names, + exclude_file=config["rsync_blacklist"], + debug=config["debug"]): """ Generate an exclude list for rsync Parameters: @@ -132,16 +177,12 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, Output: ---------- None """ - a=list() - - for package in packages_iterable: - if not isinstance(package, Package): - raise ValueError(" %s is not a Package object " % package) - if package["name"] in blacklisted_names: - a.append(package["location"]) + pkgs=[pkg["location"] for pkg in packages_iterable + if isinstance(pkg, Package) + and pkg["name"] in blacklisted_names] if debug: - return a + return pkgs try: fsock = open(exclude_file,"w") try: @@ -149,9 +190,10 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, finally: fsock.close() except IOError: - printf("%s wasnt written" % blacklist_file) + printf("%s wasnt written" % exclude_file) if __name__ == "__main__": - a=run_rsync(rsync_list_command) + cmd=generate_rsync_command(rsync_list_command) + a=run_rsync(cmd) packages=pkginfo_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/get_license.sh b/get_license.sh index a7241a1..0da58cb 100755 --- a/get_license.sh +++ b/get_license.sh @@ -31,12 +31,12 @@ rm -rf $dir/* tempdir=$(mktemp -d) cd $tempdir -a=($(cut -d: -f1 $docs/pending*.txt)) -echo ${a[@]} +pending=($(cut -d: -f1 $docs/pending*.txt)) +echo ${pending[@]} -for x in ${a[@]}; do - b=( $(ls $repo/*/os/*/$x*) ) - for y in ${b[@]}; do +for pkg in ${pending[@]}; do + pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) + for y in ${pkg_in_repo[@]}; do echo "chmod +r $y" chmod +r $y echo "tar -xf $y usr/share/licenses" diff --git a/main.sh b/main.sh index 2d59094..1a2c6c4 100644 --- a/main.sh +++ b/main.sh @@ -23,13 +23,17 @@ ${rsync_update_command} --exclude-from=${rsync_blacklist} \ ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} msg "Syncing each repo and cleaning" +msg2 "Remove pending files" +stdnull "rm -rf ${pending}*" for repo in $(echo ${repo_list} | tr ':' ' '); do - msg2 "Syncing ${repo}" - ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo} - msg2 "Cleaning ${repo}" - clean-repo.py -d ${repodir}/${repo} \ - -b ${repodir}/${repo}/${repo}.db.tar.gz - msg2 "Making pending list for ${repo}" - run_python_cmd "mkpending.py -r ${repo} -d ${repodir}/${repo}" + for arch in $(echo ${arch_list} | tr ':' ' '); do + msg2 "Syncing ${repo} ${arch}" + ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo} + msg2 "Making pending list for ${repo} ${arch}" + run_python_cmd "mkpending.py -r ${repo} -b ${repodir}/${repo}/os/${arch}" + msg2 "Cleaning ${repo} ${arch}" + run_python_cmd "clean-repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" + get_license.sh + done done diff --git a/mkpending.py b/mkpending.py new file mode 100644 index 0000000..43a5fb2 --- /dev/null +++ b/mkpending.py @@ -0,0 +1,45 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +from repm.filter import * + +def make_pending(path_to_db): + """ Determine wich packages are pending for license auditing.""" + packages_iterable=pkginfo_from_db(path_to_db) + search = tuple(listado(config["blacklist"]) + + listado(config["whitelist"])) + + pkgs=[pkg for pkg in packages_iterable + if "custom" in pkg["license"] + and pkg["name"] not in search] + return pkgs + +def write_pending(packages_iterable, repo, prefix=config["pending"]): + """ Write a pending file with the info of the packages """ + filename=prefix + "-" + repo + ".txt" + try: + fsock=open(filename, "a") + except(IOError): + print("Can't read %s" % filename) + finally: + fsock.close() + if os.path.isfile(filename): + pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in + listado(filename)] + fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + for pkg in pkgs]) + "\n") + fsock.close() + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Clean a repo db and packages") + parser.add_argument("-b", "--dababase", type=str, required=True + help="database to check") + parser.add_argument("-r", "--repo", type=str, required=True + help="repo of database") + args=parser.parse_args() + + if args.database and args.repo: + pkgs=make_pending(args.database) + write_pending(pkgs, args.repo) + else: + parser.print_help() diff --git a/pato2.py b/pato2.py index 0d77d6b..4cdb536 100644 --- a/pato2.py +++ b/pato2.py @@ -27,159 +27,23 @@ from repm.config import * from repm.filter import * import tarfile -from glob import glob from os.path import isdir, isfile -def printf(text,output_=output): +def printf(text,output=config["output"]): """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(logname, 'a') + log_file = open(config["logname"], 'a') log_file.write("\n" + str(text) + "\n") log_file.close() - if output_: print (str(text) + "\n") - -def listado(filename_): - """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename_,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] - -def db(repo_,arch_): - """Construye un nombre para sincronizar una base de datos.""" - return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) - -def packages(repo_, arch_, expr="*"): - """ Get packages on a repo, arch folder """ - return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) - -def sync_all_repo(debug=verbose): - cmd=generate_rsync_command(rsync_list_command) - rsout=run_rsync(cmd) - pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) - a=run_rsync(cmd) - cmd=generate_rsync_command(rsync_post_command,blacklist_file=rsync_blacklist) - b=run_rsync(cmd) - if debug: - printf(a) - printf(b) - -def get_from_desc(desc, var,db_tar_file=False): - """ Get a var from desc file """ - desc = desc.split("\n") - return desc[desc.index(var)+1] - -def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): - """ Makes a list of package name, file and license """ - info=list() - # Extract DB tar.gz - commands.getoutput("mkdir -p " + archdb) - if not db_tar_file: - db_tar_file = repodir + db(repo_,arch_) - if isfile(db_tar_file): - try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') - except tarfile.ReadError: - printf("No valid db_file %s" % db_tar_file) - return(tuple()) - else: - printf("No db_file %s" % db_tar_file) - return(tuple()) - for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) - db_open_tar.close() - # Get info from file - for dir_ in glob(archdb + "/*"): - if isdir(dir_) and isfile(dir_ + "/desc"): - pkg_desc_file = open(dir_ + "/desc", "r") - desc = pkg_desc_file.read() - pkg_desc_file.close() - info.append(( get_from_desc(desc,"%NAME%"), - dir_.split("/")[-1], - get_from_desc(desc,"%LICENSE%") )) - if verbose_: printf(info) - commands.getoutput("rm -r %s/*" % archdb) - return tuple(info) - -def make_pending(repo_,arch_,info_): - """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" - search = tuple( listado(blacklist) + listado (whitelist) ) - if verbose: printf("blaclist + whitelist= " + str(search) ) - lista_=list() - for (name,pkg_,license_) in info_: - if "custom" in license_: - if name not in search: - lista_.append( (name, license_ ) ) - elif not name: - printf( pkg_ + " package has no %NAME% attibute " ) - if verbose: printf( lista_ ) - a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) + "\n") - -def remove_from_blacklist(repo_,arch_,info_,blacklist_): - """ Check the blacklist and remove packages on the db""" - lista_=list() - pack_=list() - for (name_, pkg_, license_) in info_: - if name_ in blacklist_: - lista_.append(name_) - for p in packages(repo_,arch_,pkg_ + "*"): - pack_.append(p) - if lista_: - lista_=" ".join(lista_) - com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ - printf(com_) - a = commands.getoutput(com_) - if verbose: printf(a) - -def cleanup_nonfree_in_dir(directory,blacklisted_names): - pkgs=pkginfo_from_files_in_dir(directory) - for package in pkgs: - if package["name"] in blacklisted_names: - os.remove(package["location"]) - -def link(repo_,arch_,file_): - """ Makes a link in the repo for the package """ - cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ - a=commands.getoutput(cmd_) - if verbose: - printf(cmd_ + a) - -def add_free_repo(verbose_=verbose): - cmd_=os.path.join(home,"/usr/bin/sync-free") - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) - for repo_ in repo_list: - for arch_ in arch_list: - lista_=list() - for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): - lista_.append(file_) - for dir_ in other: - for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): - lista_.append(file_) - - printf(lista_) - - if lista_: - lista_=" ".join(lista_) - if verbose: printf(lista_) - cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose: printf(a) - -def get_licenses(verbose_=verbose): - """ Extract the license from packages in repo_,arch_ and in pending_ file""" - cmd_=home + "/usr/bin/get_license.sh" - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) - -def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, - source=mirror+mirrorpath, blacklist_file=False): - """ Generates an rsync command for executing it by combining all parameters. + if output_: + print (str(text) + "\n") + +def generate_rsync_command(base_command, + dir_list=(config["repo_list"] + + config["dir_list"]), + destdir=config["repodir"], + source=config["mirror"] +config["mirrorpath"]): + """ Generates an rsync command for executing + it by combining all parameters. Parameters: ---------- @@ -192,57 +56,16 @@ def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdi Return: ---------- rsync_command -> str """ - from os.path import isfile, isdir - - if blacklist_file and not isfile(blacklist_file): - print(blacklist_file + " is not a file") - raise NonValidFile - if not os.path.isdir(destdir): print(destdir + " is not a directory") raise NonValidDir dir_list="{" + ",".join(dir_list) + "}" + return " ".join((base_command, os.path.join(source, dir_list), + destdir)) - if blacklist_file: - return " ".join((base_command, "--exclude-from="+blacklist_file, - os.path.join(source, dir_list), destdir)) - return " ".join((base_command, os.path.join(source, dir_list), destdir)) - -def run_rsync(command,debug=verbose): +def run_rsync(command,debug=config["debug"]): """ Runs rsync and gets returns it's output """ if debug: printf("rsync_command: " + command) - return commands.getoutput(command) - -if __name__ == "__main__": - from time import time - start_time = time() - def minute(): - return str(round((time() - start_time)/60, 1)) - - printf(" Cleaning %s folder " % (tmp) ) - commands.getoutput("rm -r %s/*" % tmp) - printf(" Syncing repo") - sync_all_repo(True) - - printf(" Updating databases and pending files lists: minute %s \n" % minute() ) - for repo in repo_list: - for arch in arch_list: - printf( "\n" + repo + "-" + arch + "\n" ) - printf( "Get info: minute %s " % minute() ) - info=get_info(repo,arch) - printf( "Make pending: minute %s" % minute() ) - make_pending(repo,arch,info) - printf( "Update DB: minute %s" % minute() ) - remove_from_blacklist( - repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) - - printf("Adding Parabola Packages: minute %s\n" % minute() ) - add_free_repo(True) - - printf("Extracting licenses in pending: minute %s" % minute() ) - get_licenses() - - printf("\n\nDelay: %s minutes \n" % minute()) - + return check_output(command) diff --git a/test/test_filter.py b/test/test_filter.py index 1906b87..5601d57 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -169,21 +169,21 @@ class pkginfo_from_db(unittest.TestCase): "release" : "2", "arch" : "x86_64", "license" : ("LGPL",), - "location": "acl-2.2.49-2-x86_64.pkg.tar.xz" + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz", "depends" : ("attr>=2.4.41"),} example_package_list[1].package_info={ "name" : "glibc", "version" : "2.13", "release" : "4", "arch" : "x86_64", "license" : ("GPL","LGPL"), - "location": "glibc-2.13-4-x86_64.pkg.tar.xz" + "location": "glibc-2.13-4-x86_64.pkg.tar.xz", "depends" : ("linux-api-headers>=2.6.37","tzdata",),} example_package_list[2].package_info={ "name" : "", "version" : "2.2.26", "release" : "1", "arch" : "x86_64", "license" : False, - "location": "" + "location": "", "depends" : False,} -- cgit v1.2.3 From e0e4837f17f22b7bfafafc8d0e6f7a351249417e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 01:19:32 -0500 Subject: FixedFixed a little main.sh --- main.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main.sh b/main.sh index 1a2c6c4..cf7bcd8 100644 --- a/main.sh +++ b/main.sh @@ -33,7 +33,9 @@ for repo in $(echo ${repo_list} | tr ':' ' '); do msg2 "Making pending list for ${repo} ${arch}" run_python_cmd "mkpending.py -r ${repo} -b ${repodir}/${repo}/os/${arch}" msg2 "Cleaning ${repo} ${arch}" - run_python_cmd "clean-repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" - get_license.sh + run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" done done + +msg "Checking licenses" +get_license.sh -- cgit v1.2.3 From f6184ee3c9422c40effb0265527ee5a31b027a1a Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Sun, 10 Apr 2011 23:44:01 -0700 Subject: chmod +x --- .gitignore | 3 ++- clean_repo.py | 0 config.py | 0 filter.py | 0 main.sh | 2 +- mkpending.py | 0 6 files changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 clean_repo.py mode change 100644 => 100755 config.py mode change 100644 => 100755 filter.py mode change 100644 => 100755 main.sh mode change 100644 => 100755 mkpending.py diff --git a/.gitignore b/.gitignore index e645833..cfefcc3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ -*.pyc \ No newline at end of file +*.pyc +cptobin.sh diff --git a/clean_repo.py b/clean_repo.py old mode 100644 new mode 100755 diff --git a/config.py b/config.py old mode 100644 new mode 100755 diff --git a/filter.py b/filter.py old mode 100644 new mode 100755 diff --git a/main.sh b/main.sh old mode 100644 new mode 100755 index cf7bcd8..886ea38 --- a/main.sh +++ b/main.sh @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/bin/bash # -*- coding: utf-8 -*- source config.sh diff --git a/mkpending.py b/mkpending.py old mode 100644 new mode 100755 -- cgit v1.2.3 From a5f24954011da556977859d8a306ddb39c1ec2e4 Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Sun, 10 Apr 2011 23:49:50 -0700 Subject: Added cptobin --- .gitignore | 2 +- cptobin.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100755 cptobin.sh diff --git a/.gitignore b/.gitignore index cfefcc3..eef29c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *~ *.pyc -cptobin.sh + diff --git a/cptobin.sh b/cptobin.sh new file mode 100755 index 0000000..25a22d4 --- /dev/null +++ b/cptobin.sh @@ -0,0 +1,2 @@ +#!/bin/bash +cp -f clean_repo.py config.py config.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file -- cgit v1.2.3 From 67038ba1840d0f57b0ce49fdabd3dfa8057e2451 Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Mon, 11 Apr 2011 00:22:06 -0700 Subject: fixed some errors --- clean_repo.py | 2 +- config.py | 4 ++-- config.sh | 2 ++ cptobin.sh | 2 +- filter.py | 8 ++++---- mkpending.py | 4 ++-- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index 29d446d..b0d306f 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -34,7 +34,7 @@ if __name__ == "__main__": args=parser.parse_args() if args.directory: - cleanup_nonfree_in_dir(args.database, listado(config["blacklist"])) + cleanup_nonfree_in_dir(args.directory, listado(config["blacklist"])) if args.database: pkgs=pkginfo_from_db(args.database) diff --git a/config.py b/config.py index 24ecfaf..f4089f9 100755 --- a/config.py +++ b/config.py @@ -6,8 +6,8 @@ except(ImportError): from commands import getoutput as check_output import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir", "rsync_blacklist") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", "docs_dir", + "repodir", "blacklist", "whitelist", "pending", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) diff --git a/config.sh b/config.sh index 741dee4..4f42e0b 100755 --- a/config.sh +++ b/config.sh @@ -12,6 +12,7 @@ logtime=$(date -u +%Y%m%d-%H:%M) ## Must be defined logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ +archdb=${tempdir}/db docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # End Directories @@ -42,6 +43,7 @@ function run_python_cmd { mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ + archdb=${archdb} \ docs_dir=${docs_dir} \ repodir=${repodir} \ blacklist=${blacklist} \ diff --git a/cptobin.sh b/cptobin.sh index 25a22d4..e803132 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index 1a0fa6f..78ad410 100755 --- a/filter.py +++ b/filter.py @@ -137,15 +137,15 @@ def pkginfo_from_db(path_to_db): if not os.path.isfile(path_to_db): raise NonValidFile(path_to_db + "is not a file") - check_output("mkdir -p " + archdb) + check_output("mkdir -p " + config["archdb"]) try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') + db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % db_tar_file) + printf("No valid db_file %s or not readable" % path_to_db) return(tuple()) else: - printf("No db_file %s" % db_tar_file) + printf("No db_file %s" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): diff --git a/mkpending.py b/mkpending.py index 43a5fb2..2b255a8 100755 --- a/mkpending.py +++ b/mkpending.py @@ -32,9 +32,9 @@ def write_pending(packages_iterable, repo, prefix=config["pending"]): if __name__ == "__main__": parser = argparse.ArgumentParser( description="Clean a repo db and packages") - parser.add_argument("-b", "--dababase", type=str, required=True + parser.add_argument("-b", "--dababase", type=str, required=True, help="database to check") - parser.add_argument("-r", "--repo", type=str, required=True + parser.add_argument("-r", "--repo", type=str, required=True, help="repo of database") args=parser.parse_args() -- cgit v1.2.3 From 748600bf8dfba34b336ad642a6b26dae674db85f Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 11 Apr 2011 00:25:53 -0700 Subject: Will rsync be better now? --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index c643bb5..3e00eb0 100644 --- a/config.py +++ b/config.py @@ -46,8 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " -rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3 From 857ecbe794c714919612b533303e7a9ef781c75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 10:26:40 -0500 Subject: Use non tracked file local_config for config. --- .gitignore | 2 +- config.sh | 33 +-------------------------------- local_config.example | 31 +++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 33 deletions(-) create mode 100644 local_config.example diff --git a/.gitignore b/.gitignore index eef29c1..69dec40 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *~ *.pyc - +local_config diff --git a/config.sh b/config.sh index 4f42e0b..75d4b73 100755 --- a/config.sh +++ b/config.sh @@ -1,37 +1,6 @@ #!/bin/sh # -*- coding: utf-8 -*- - -# Mirror options -mirror="mirrors.eu.kernel.org" -mirrorpath="::mirrors/archlinux" - -# Directories -## Optionals -paraboladir=~/parabolagnulinux.org -logtime=$(date -u +%Y%m%d-%H:%M) -## Must be defined -logname=${paraboladir}/${logtime}-repo-maintainer.log -tempdir=~/tmp/ -archdb=${tempdir}/db -docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo -# End Directories - -# Files -blacklist=${docs_dir}/blacklist.txt -whitelist=${docs_dir}/whitelist.txt -pending=${docs_dir}/pending -rsync_blacklist=${docs_dir}/rsyncBlacklist - -# Repos, arches, and dirs for repo -repo_list="core:extra:community:testing:community-testing:multilib" -dir_list="pool" -arch_list="i686:x86_64" -other="any" - -# Output options -output="True" -debug="False" +source local_config # Rsync commands rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " diff --git a/local_config.example b/local_config.example new file mode 100644 index 0000000..7edfde7 --- /dev/null +++ b/local_config.example @@ -0,0 +1,31 @@ +# Mirror options +mirror="mirrors.eu.kernel.org" +mirrorpath="::mirrors/archlinux" + +# Directories +## Optionals +paraboladir=~/parabolagnulinux.org +logtime=$(date -u +%Y%m%d-%H:%M) +## Must be defined +logname=${paraboladir}/${logtime}-repo-maintainer.log +tempdir=~/tmp/ +archdb=${tempdir}/db +docs_dir=${paraboladir}/docs +repodir=${paraboladir}/repo +# End Directories + +# Files +blacklist=${docs_dir}/blacklist.txt +whitelist=${docs_dir}/whitelist.txt +pending=${docs_dir}/pending +rsync_blacklist=${docs_dir}/rsyncBlacklist + +# Repos, arches, and dirs for repo +repo_list="core:extra:community:testing:community-testing:multilib" +dir_list="pool" +arch_list="i686:x86_64" +other="any" + +# Output options +output="True" +debug="False" -- cgit v1.2.3 From 3e27d11f68571bce92138f6cbfcaecac75fa1644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 12:39:40 -0500 Subject: Fixed some errors --- clean_repo.py | 6 +++++- config.py | 35 +++++++++++++++++++++++++++++------ config.sh | 5 ----- cptobin.sh | 2 +- filter.py | 32 +++++++++++++++++--------------- local_config.example | 10 +++++++--- main.sh | 3 ++- mkpending.py | 35 ++++++++++++++++++++--------------- pato2.py | 10 +--------- 9 files changed, 82 insertions(+), 56 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index b0d306f..d4e06fc 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -6,7 +6,9 @@ import argparse def remove_from_blacklist(path_to_db, blacklisted_names, debug=config["debug"]): """ Check the blacklist and remove packages on the db""" - + if "~" in path_to_db: + path_to_db=(os.path.expanduser(path_to_db)) + pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if pkg["name"] in blacklisted_names] if pkgs: @@ -19,6 +21,8 @@ def remove_from_blacklist(path_to_db, blacklisted_names, return pkgs, cmd def cleanup_nonfree_in_dir(directory, blacklisted_names): + if "~" in directory: + directory=(os.path.expanduser(directory)) pkgs=pkginfo_from_files_in_dir(directory) for package in pkgs: if package["name"] in blacklisted_names: diff --git a/config.py b/config.py index f4089f9..8b48c66 100755 --- a/config.py +++ b/config.py @@ -3,11 +3,15 @@ try: from subprocess import check_output except(ImportError): - from commands import getoutput as check_output + from commands import getoutput + def check_output(*popenargs,**kwargs): + cmd=" ".join(*popenargs) + return getoutput(cmd) import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", "docs_dir", - "repodir", "blacklist", "whitelist", "pending", "rsync_blacklist") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", + "repodir", "blacklist", "whitelist", "pending", + "rsync_blacklist",) listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) @@ -37,10 +41,29 @@ for var in boolvars: # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " +def printf(text,output=config["output"]): + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(config["logname"], 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output: + print (str(text) + "\n") + +del exit_if_none + # Classes and Exceptions -class NonValidFile(ValueError): pass -class NonValidDir(ValueError): pass -class NonValidCommand(ValueError): pass +class NonValidFile(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) +class NonValidDir(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) +class NonValidCommand(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) class Package: """ An object that has information about a package. """ diff --git a/config.sh b/config.sh index 75d4b73..9a44f50 100755 --- a/config.sh +++ b/config.sh @@ -2,10 +2,6 @@ # -*- coding: utf-8 -*- source local_config -# Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " - function run_python_cmd { env \ mirror=${mirror} \ @@ -13,7 +9,6 @@ function run_python_cmd { logname=${logname} \ tempdir=${tempdir} \ archdb=${archdb} \ - docs_dir=${docs_dir} \ repodir=${repodir} \ blacklist=${blacklist} \ whitelist=${whitelist} \ diff --git a/cptobin.sh b/cptobin.sh index e803132..068d765 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py local_config ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index 78ad410..48e2d93 100755 --- a/filter.py +++ b/filter.py @@ -4,12 +4,17 @@ from glob import glob from repm.config import * from repm.pato2 import * -def listado(filename): +def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + fsock = open(filename,"r") + lista = fsock.read().split("\n") + fsock.close() + if end is not None: + return [pkg.split(":")[start:end].rstrip() + for pkg in lista if pkg] + else: + return [pkg.split(":")[start].rstrip() + for pkg in lista if pkg] def pkginfo_from_filename(filename): """ Generates a Package object with info from a filename, @@ -135,29 +140,26 @@ def pkginfo_from_db(path_to_db): package_list=list() if not os.path.isfile(path_to_db): - raise NonValidFile(path_to_db + "is not a file") + raise NonValidFile(path_to_db + " is not a file") - check_output("mkdir -p " + config["archdb"]) + check_output(["mkdir", "-p", config["archdb"]]) try: db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % path_to_db) - return(tuple()) - else: - printf("No db_file %s" % path_to_db) + raise NonValidFile("No valid db_file %s or not readable" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) + db_open_tar.extract(file, config["archdb"]) db_open_tar.close() # Get info from file - for dir_ in glob(archdb + "/*"): + for dir_ in glob(config["archdb"] + "/*"): if isdir(dir_) and isfile(dir_ + "/desc"): package_list.append(pkginfo_from_desc( os.path.join(dir_,"desc"))) - check_output("rm -r %s/*" % archdb) - if verbose_: + check_output(["rm", "-r", config["archdb"]]) + if config["debug"]: printf(package_list) return package_list diff --git a/local_config.example b/local_config.example index 7edfde7..8015ee2 100644 --- a/local_config.example +++ b/local_config.example @@ -2,19 +2,19 @@ mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" -# Directories +# Directories: they should end without / ## Optionals paraboladir=~/parabolagnulinux.org logtime=$(date -u +%Y%m%d-%H:%M) ## Must be defined -logname=${paraboladir}/${logtime}-repo-maintainer.log -tempdir=~/tmp/ +tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # End Directories # Files +logname=${paraboladir}/log/${logtime}-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt pending=${docs_dir}/pending @@ -29,3 +29,7 @@ other="any" # Output options output="True" debug="False" + +# Rsync commands +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " \ No newline at end of file diff --git a/main.sh b/main.sh index 886ea38..46f2f75 100755 --- a/main.sh +++ b/main.sh @@ -20,7 +20,8 @@ mkrsexclude msg "Syncing repos without delete" # rsync_update_command does not sync db or abs ${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} + ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ + $(echo ${dir_list} | tr ':' ',')} ${repodir} msg "Syncing each repo and cleaning" msg2 "Remove pending files" diff --git a/mkpending.py b/mkpending.py index 2b255a8..6022206 100755 --- a/mkpending.py +++ b/mkpending.py @@ -1,9 +1,27 @@ #!/usr/bin/python # -*- coding: utf-8 -*- from repm.filter import * +import argparse -def make_pending(path_to_db): +def make_pending(path_to_db, repo, prefix=config["pending"]): """ Determine wich packages are pending for license auditing.""" + filename=prefix + "-" + repo + ".txt" + try: + fsock=open(filename, "rw") + if os.path.isfile(filename): + pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in + listado(filename)] + fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + for pkg in pkgs]) + "\n") + except(IOError): + print("Can't read %s" % filename) + exit(1) + finally: + fsock.close() + + if "~" in path_to_db: + path_to_db=(os.path.expanduser(path_to_db)) + packages_iterable=pkginfo_from_db(path_to_db) search = tuple(listado(config["blacklist"]) + listado(config["whitelist"])) @@ -15,24 +33,11 @@ def make_pending(path_to_db): def write_pending(packages_iterable, repo, prefix=config["pending"]): """ Write a pending file with the info of the packages """ - filename=prefix + "-" + repo + ".txt" - try: - fsock=open(filename, "a") - except(IOError): - print("Can't read %s" % filename) - finally: - fsock.close() - if os.path.isfile(filename): - pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in - listado(filename)] - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] - for pkg in pkgs]) + "\n") - fsock.close() if __name__ == "__main__": parser = argparse.ArgumentParser( description="Clean a repo db and packages") - parser.add_argument("-b", "--dababase", type=str, required=True, + parser.add_argument("-b", "--database", type=str, required=True, help="database to check") parser.add_argument("-r", "--repo", type=str, required=True, help="repo of database") diff --git a/pato2.py b/pato2.py index 4cdb536..6daa8b8 100644 --- a/pato2.py +++ b/pato2.py @@ -29,14 +29,6 @@ from repm.filter import * import tarfile from os.path import isdir, isfile -def printf(text,output=config["output"]): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(config["logname"], 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output_: - print (str(text) + "\n") - def generate_rsync_command(base_command, dir_list=(config["repo_list"] + config["dir_list"]), @@ -68,4 +60,4 @@ def run_rsync(command,debug=config["debug"]): """ Runs rsync and gets returns it's output """ if debug: printf("rsync_command: " + command) - return check_output(command) + return check_output(command.split()) -- cgit v1.2.3 From aca64420db93c1d1ab26ad27640458d3a1f09f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 13 Apr 2011 00:58:19 -0500 Subject: cptobin.sh not copy inexistent files --- cptobin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cptobin.sh b/cptobin.sh index 068d765..ce4b3bd 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py local_config ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/ \ No newline at end of file -- cgit v1.2.3 From ddacb9a98b90fc2b51551fb73ab8513634f4f185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 13 Apr 2011 01:01:57 -0500 Subject: filter.py to close correct file --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.py b/filter.py index 1d70a63..c7f3f18 100755 --- a/filter.py +++ b/filter.py @@ -147,7 +147,7 @@ def pkginfo_from_db(path_to_db): % path_to_db) return(tuple()) finally: - db_open_tar.close() + dbsock.close() return package_list def rsyncBlacklist_from_blacklist(packages_iterable, -- cgit v1.2.3 From 062b7a2d9b0347e4e867e588826f052d26a3bb30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 13 Apr 2011 01:11:20 -0500 Subject: For testing --- filter.py | 2 +- local_config.example | 4 ++-- main.sh | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/filter.py b/filter.py index c7f3f18..4b9603d 100755 --- a/filter.py +++ b/filter.py @@ -137,7 +137,7 @@ def pkginfo_from_db(path_to_db): try: dbsock = tarfile.open(path_to_db, 'r:gz') - desc_files=[desc for desc in db_open_tar.getnames() + desc_files=[desc for desc in dbsock.getnames() if "/desc" in desc] for name in desc_files: desc=dbsock.extractfile(name) diff --git a/local_config.example b/local_config.example index 8015ee2..99a1fdb 100644 --- a/local_config.example +++ b/local_config.example @@ -31,5 +31,5 @@ output="True" debug="False" # Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " \ No newline at end of file +rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' " \ No newline at end of file diff --git a/main.sh b/main.sh index 9f41a95..d15f736 100755 --- a/main.sh +++ b/main.sh @@ -29,8 +29,10 @@ stdnull "rm -rf ${pending}*" for repo in $(echo ${repo_list} | tr ':' ' '); do for arch in $(echo ${arch_list} | tr ':' ' '); do msg2 "Syncing ${repo} ${arch}" - ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo} + cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo}) + plain "${cmd}" + ${cmd} msg2 "Cleaning ${repo} ${arch}" # This also generates pending lists run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" -- cgit v1.2.3 From deab65fad4ced009fb31f7033b1db8ef0af78aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 15 Apr 2011 04:53:27 -0500 Subject: Python parts ready for bash usage in python 2 and 3 --- clean_repo.py | 78 ++++++++++++++++++++++++++++++----------------------- config.py | 65 +++++++++----------------------------------- filter.py | 48 ++++++++++++++++++++------------- pato2.py | 63 ------------------------------------------- test/test_filter.py | 3 ++- 5 files changed, 88 insertions(+), 169 deletions(-) delete mode 100644 pato2.py diff --git a/clean_repo.py b/clean_repo.py index eccfd01..5423c4d 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -3,21 +3,16 @@ from repm.filter import * import argparse -def mkpending(path_to_db, repo, prefix=config["pending"]): +def mkpending(packages_iterable, pending_file, blacklisted_names, + whitelisted_names): """ Determine wich packages are pending for license auditing.""" - if "~" in path_to_db: - path_to_db=(os.path.expanduser(path_to_db)) - - search = tuple(listado(config["blacklist"]) + - listado(config["whitelist"])) + search = tuple(blacklisted_names + + whitelisted_names) - pkgs=list(pkginfo_from_db(path_to_db)) - - filename=prefix + "-" + repo + ".txt" try: - fsock=open(filename, "rw") - pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) - if pkg["name"] not in listado(filename)] + fsock=open(pending_file, "r") + pkgs=[pkg for pkg in packages_iterable + if pkg["name"] not in listado(pending_file)] for line in fsock.readlines(): if line: pkg=Package() @@ -26,16 +21,16 @@ def mkpending(path_to_db, repo, prefix=config["pending"]): pkgs.append(pkg) pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] + fsock=open(pending_file, "w") fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): - raise NonValidFile("Can't read or write %s" % filename) + raise NonValidFile("Can't read or write %s" % pending_file) finally: fsock.close() return pkgs -def remove_from_blacklist(path_to_db, blacklisted_names, - debug=config["debug"]): +def remove_from_blacklist(path_to_db, blacklisted_names): """ Check the blacklist and remove packages on the db""" if "~" in path_to_db: path_to_db=(os.path.expanduser(path_to_db)) @@ -47,9 +42,7 @@ def remove_from_blacklist(path_to_db, blacklisted_names, cmd = "repo-remove " + path_to_db + " " + lista printf(cmd) a = check_output(cmd) - if debug: - printf(a) - return pkgs, cmd + return pkgs def cleanup_nonfree_in_dir(directory, blacklisted_names): if "~" in directory: @@ -61,25 +54,42 @@ def cleanup_nonfree_in_dir(directory, blacklisted_names): if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Clean a repo db and packages") - parser.add_argument("-b", "--database", type=str, - help="dabatase to clean") - parser.add_argument("-d", "--directory", type=str, - help="directory to clean") + prog="clean_repo", + description="Clean a repo db and packages",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + group_dir=parser.add_argument_group("Clean non-free packages in dir") + group_dir.add_argument("-d", "--directory", type=str, + help="directory to clean",) + + group_db=parser.add_argument_group("Clean non-free packages in db", + "All arguments need to be specified") + group_db.add_argument("-b", "--database", type=str, + help="dabatase to clean") + group_db.add_argument("-p", "--pending-file", type=str, + help="File in which to write pending list") + group_db.add_argument("-w", "--whitelist-file", type=str, + help="File containing whitelisted names") + args=parser.parse_args() + if not args.directory and not args.database: + parser.print_help() + elif not args.pending_file or not args.whitelist_file \ + and args.database: + parser.print_help() + else: + blacklisted=listado(args.blacklist_file) + if args.database: - repo=os.path.basename(args.database).split(".")[0] + whitelisted=listado(args.whitelist_file) pkgs=pkginfo_from_db(args.database) - remove_from_blacklist(args.database, pkgs, - tuple(listado(config["blacklist"]) + - listado(config["pending"] + - "-" + repo + ".txt"))) - mkpending(args.database, args.repo) + remove_from_blacklist(args.database, blacklisted) + mkpending(pkgs, args.pending_file, + blacklisted, whitelisted) if args.directory: - cleanup_nonfree_in_dir(args.directory, - listado(config["blacklist"])) - - if not args.directory and not args.database: - parser.print_help() + cleanup_nonfree_in_dir(args.directory, blacklisted) diff --git a/config.py b/config.py index 8b48c66..4e218a5 100755 --- a/config.py +++ b/config.py @@ -9,61 +9,26 @@ except(ImportError): return getoutput(cmd) import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", - "repodir", "blacklist", "whitelist", "pending", - "rsync_blacklist",) -listvars=("repo_list", "dir_list", "arch_list", "other",) -boolvars=("output", "debug",) - -config=dict() - -def exit_if_none(var): - if os.environ.get(var) is None: - exit("%s is not defined" % var) - -for var in stringvars: - exit_if_none(var) - config[var]=os.environ.get(var) - -for var in listvars: - exit_if_none(var) - config[var]=tuple(os.environ.get(var).split(":")) - -for var in boolvars: - exit_if_none(var) - if os.environ.get(var) == "True": - config[var]=True - elif os.environ.get(var) =="False": - config[var]=False - else: - print('%s is not True or False' % var) # Rsync commands -rsync_list_command="rsync -a --no-motd --list-only " -def printf(text,output=config["output"]): +def printf(text, logfile=False): """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(config["logname"], 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output: - print (str(text) + "\n") + print (str(text) + "\n") + if logfile: + try: + log = open(logfile, 'a') + log.write("\n" + str(text) + "\n") + except: + print("Can't open %s" % logfile) + finally: + log.close() -del exit_if_none # Classes and Exceptions -class NonValidFile(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) -class NonValidDir(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) -class NonValidCommand(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) +class NonValidFile(ValueError): pass +class NonValidDir(ValueError): pass +class NonValidCommand(ValueError): pass class Package: """ An object that has information about a package. """ @@ -101,7 +66,3 @@ class Package: return False else: return True - -if __name__=="__main__": - for key in config.keys(): - print("%s : %s" % (key,config[key])) diff --git a/filter.py b/filter.py index 4b9603d..add0235 100755 --- a/filter.py +++ b/filter.py @@ -2,7 +2,7 @@ #-*- encoding: utf-8 -*- from glob import glob from repm.config import * -from repm.pato2 import * +import tarfile def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" @@ -28,7 +28,7 @@ def pkginfo_from_filename(filename): ---------- pkg -> Package object""" if ".pkg.tar." not in filename: - raise NonValidFile + raise NonValidFile("File is not a pacman package") pkg = Package() pkg["location"] = filename fileattrs = os.path.basename(filename).split("-") @@ -140,19 +140,18 @@ def pkginfo_from_db(path_to_db): desc_files=[desc for desc in dbsock.getnames() if "/desc" in desc] for name in desc_files: - desc=dbsock.extractfile(name) - package_list.append(pkginfo_from_desc(desc.read())) + desc=dbsock.extractfile(name).read().decode("UTF-8") + package_list.append(pkginfo_from_desc(desc)) except tarfile.ReadError: raise NonValidFile("No valid db_file %s or not readable" % path_to_db) - return(tuple()) finally: dbsock.close() return package_list def rsyncBlacklist_from_blacklist(packages_iterable, blacklisted_names, - exclude_file=config["rsync_blacklist"]): + exclude_file): """ Generate an exclude list for rsync Parameters: @@ -168,20 +167,31 @@ def rsyncBlacklist_from_blacklist(packages_iterable, pkgs=[pkg["location"] for pkg in packages_iterable if isinstance(pkg, Package) and pkg["name"] in blacklisted_names] - - try: - fsock = open(exclude_file,"w") - fsock.write("\n".join(pkgs) + "\n") - except IOError: - printf("%s wasnt written" % exclude_file) - exit(1) - finally: - fsock.close() + if exclude_file: + try: + fsock = open(exclude_file,"w") + fsock.write("\n".join(pkgs) + "\n") + except IOError: + printf("%s wasnt written" % exclude_file) + exit(1) + finally: + fsock.close() return pkgs if __name__ == "__main__": - cmd=generate_rsync_command(rsync_list_command) - a=run_rsync(cmd) - packages=pkginfo_from_rsync_output(a) - rsyncBlaclist_from_blacklist(packages,listado(blacklist)) + import argparse + parser=argparse.ArgumentParser() + parser.add_argument("-r", "--rsync-exclude-file", type=str, + help="File in which to generate exclude list", + required=True,) + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + parser.add_argument("-c", "--rsync-command", type=str, + help="This command will be run to get a pkg list") + args=parser.parse_args() + rsout=check_output(args.rsync_command) + packages=pkginfo_from_rsync_output(rsout) + rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), + args.rsync_exclude_file) diff --git a/pato2.py b/pato2.py deleted file mode 100644 index 6daa8b8..0000000 --- a/pato2.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -""" - parabola.py - Copyright 2009 Rafik Mas'ad - Copyright 2010 Joshua Ismael Haase Hernández - - ---------- GNU General Public License 3 ---------- - - This file is part of Parabola. - - Parabola is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Parabola is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Parabola. If not, see . - - -""" -from repm.config import * -from repm.filter import * -import tarfile -from os.path import isdir, isfile - -def generate_rsync_command(base_command, - dir_list=(config["repo_list"] + - config["dir_list"]), - destdir=config["repodir"], - source=config["mirror"] +config["mirrorpath"]): - """ Generates an rsync command for executing - it by combining all parameters. - - Parameters: - ---------- - base_command -> str - dir_list -> list or tuple - destdir -> str Path to dir, dir must exist. - source -> str The source for rsync - blacklist_file -> False or str Path to file, file must exist. - - Return: - ---------- - rsync_command -> str """ - if not os.path.isdir(destdir): - print(destdir + " is not a directory") - raise NonValidDir - - dir_list="{" + ",".join(dir_list) + "}" - return " ".join((base_command, os.path.join(source, dir_list), - destdir)) - -def run_rsync(command,debug=config["debug"]): - """ Runs rsync and gets returns it's output """ - if debug: - printf("rsync_command: " + command) - return check_output(command.split()) diff --git a/test/test_filter.py b/test/test_filter.py index b6d5766..d8006f9 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -143,7 +143,8 @@ class generateRsyncBlacklist(unittest.TestCase): def testExcludeFiles(self): a=rsyncBlacklist_from_blacklist(self.example_package_list, - listado("blacklist_sample")) + listado("blacklist_sample"), + False) b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] self.assertEqual(a,b) -- cgit v1.2.3 From 374c8e2a5183cdbaefe9f54184603ad8d09e30c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 03:15:01 -0500 Subject: * bash-repo for testing * merged dbscripts with repm --- changelog | 2 -- clean_repo.py | 3 ++- config | 2 +- config.sh | 26 ---------------------- cptobin.sh | 2 -- filter.py | 5 +++-- get_license.sh | 50 +++++++++++++++++------------------------- local_config.example | 18 ++++++--------- main.sh | 62 +++++++++++++++++++++++----------------------------- 9 files changed, 60 insertions(+), 110 deletions(-) delete mode 100644 changelog delete mode 100755 config.sh delete mode 100755 cptobin.sh diff --git a/changelog b/changelog deleted file mode 100644 index 0e89f2f..0000000 --- a/changelog +++ /dev/null @@ -1,2 +0,0 @@ -repo-maintainer 1.0 - * Known working version diff --git a/clean_repo.py b/clean_repo.py index 5423c4d..e1a17c2 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -22,7 +22,8 @@ def mkpending(packages_iterable, pending_file, blacklisted_names, pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] fsock=open(pending_file, "w") - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): raise NonValidFile("Can't read or write %s" % pending_file) diff --git a/config b/config index 217b627..d9bad13 100644 --- a/config +++ b/config @@ -1,7 +1,7 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" -PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing' 'multilib') PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/config.sh b/config.sh deleted file mode 100755 index 9a44f50..0000000 --- a/config.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# -*- coding: utf-8 -*- -source local_config - -function run_python_cmd { - env \ - mirror=${mirror} \ - mirrorpath=${mirrorpath} \ - logname=${logname} \ - tempdir=${tempdir} \ - archdb=${archdb} \ - repodir=${repodir} \ - blacklist=${blacklist} \ - whitelist=${whitelist} \ - pending=${pending} \ - rsync_blacklist=${rsync_blacklist} \ - repo_list=${repo_list} \ - dir_list=${dir_list} \ - arch_list=${arch_list} \ - other=${other} \ - output=${output} \ - debug=${debug} \ - $1 -} - -source libremessages \ No newline at end of file diff --git a/cptobin.sh b/cptobin.sh deleted file mode 100755 index ce4b3bd..0000000 --- a/cptobin.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index add0235..4abdf38 100755 --- a/filter.py +++ b/filter.py @@ -189,9 +189,10 @@ if __name__ == "__main__": help="File containing blacklisted names", required=True,) parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list") + help="This command will be run to get a pkg list", + required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command) + rsout=check_output(args.rsync_command.split()) packages=pkginfo_from_rsync_output(rsout) rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) diff --git a/get_license.sh b/get_license.sh index 0da58cb..024876c 100755 --- a/get_license.sh +++ b/get_license.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # -*- coding: utf-8 -*- # get_license.sh @@ -20,36 +20,26 @@ # You should have received a copy of the GNU General Public License # along with Parabola. If not, see . - -docs="/home/parabolavnx/parabolagnulinux.org/docs" -repo="/home/parabolavnx/parabolagnulinux.org/repo" -dir="$docs/pending-licenses" - -echo "Cleaning $dir" -rm -rf $dir/* - -tempdir=$(mktemp -d) -cd $tempdir - -pending=($(cut -d: -f1 $docs/pending*.txt)) -echo ${pending[@]} - -for pkg in ${pending[@]}; do - pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) - for y in ${pkg_in_repo[@]}; do - echo "chmod +r $y" - chmod +r $y - echo "tar -xf $y usr/share/licenses" - bsdtar -xf $y usr/share/licenses - echo "chmod -r $y" - chmod -r $y +source ./config +source ./local_config +source ./libremessages + +msg "Creating pending licenses list" +pushd ${licenses_dir} +rm -rf ${licenses_dir}/* + +for repo in ${PKGREPOS[@]}; do + msg2 "Extracting licenses in ${repo}" + pending=($(cut -d: -f2 ${docs_dir}/pending-${repo})) + pushd ${repodir}/${repo} + for pkg in ${pending[@]}; do + plain "${pkg}" + bsdtar -xf ${pkg} usr/share/licenses || { + error "${pkg} has no licenses" + } + chmod -r ${pkg} done done -mv usr/share/licenses/* $dir - -cd - -rm -rf $tempdir - +popd exit 0 \ No newline at end of file diff --git a/local_config.example b/local_config.example index 99a1fdb..2c0ef3e 100644 --- a/local_config.example +++ b/local_config.example @@ -4,21 +4,21 @@ mirrorpath="::mirrors/archlinux" # Directories: they should end without / ## Optionals -paraboladir=~/parabolagnulinux.org -logtime=$(date -u +%Y%m%d-%H:%M) +paraboladir= ## Must be defined tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo +repodir=${paraboladir}/repo/staging +licenses_dir=${docs_dir}/pending_licenses # End Directories # Files -logname=${paraboladir}/log/${logtime}-repo-maintainer.log +logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt -pending=${docs_dir}/pending rsync_blacklist=${docs_dir}/rsyncBlacklist +rsync_not_needed=${tmp}/rsync_not_needed # Repos, arches, and dirs for repo repo_list="core:extra:community:testing:community-testing:multilib" @@ -26,10 +26,6 @@ dir_list="pool" arch_list="i686:x86_64" other="any" -# Output options -output="True" -debug="False" - # Rsync commands -rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' " \ No newline at end of file +rsync_list_command="rsync\ -ptgoL\ --list-only\ " +rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" diff --git a/main.sh b/main.sh index d15f736..b3ecb92 100755 --- a/main.sh +++ b/main.sh @@ -1,43 +1,35 @@ #!/bin/bash # -*- coding: utf-8 -*- -source config.sh +source ./config +source ./local_config +source ./libremessages -function mkrsexclude { - local error=1 - while ${error}; do - run_python_cmd "filter.py" - error=$? - done -} - -msg "Cleaning $tempdir" -stdnull "rm -r $tempdir/* " - -msg "Generating exclude list for rsync" -mkrsexclude - -msg "Syncing repos without delete" -# rsync_update_command does not sync db or abs -${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ - $(echo ${dir_list} | tr ':' ',')} ${repodir} - -msg "Syncing each repo and cleaning" -msg2 "Remove pending files" -stdnull "rm -rf ${pending}*" -for repo in $(echo ${repo_list} | tr ':' ' '); do - for arch in $(echo ${arch_list} | tr ':' ' '); do - msg2 "Syncing ${repo} ${arch}" - cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo}) - plain "${cmd}" - ${cmd} - msg2 "Cleaning ${repo} ${arch}" - # This also generates pending lists - run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]} 'any'; do + msg "Syncing ${repo} ${arch}" + filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + \"${rsync_list_command}\ \ + ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ + ${repodir}/${repo}/\" + find ${repodir}/${repo} -name *${PKGEXT} -print \ + > ${rsync_not_needed} + ${rsync_update_command} \ + ${mirror}${mirrorpath}/${repo}/os/${arch} \ + ${repodir}/${repo} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed} done + for arch in ${ARCHES[@]}; do + if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + clean_repo.py -k ${blacklist} -w ${whitelist} \ + -p ${docs_dir}/pending-${repo} \ + -b ${repodir}/${repo}/${repo}${DBEXT} + fi + clean_repo.py -k ${blacklist} -d ${repodir}/${repo} done -msg "Checking licenses" +db-update +ftpdir-cleanup + get_license.sh -- cgit v1.2.3 From bbb2155b1bb0dd442be86e05d1be6400e62be5d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 03:40:54 -0500 Subject: Updated TODO list. --- TODO | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/TODO b/TODO index 11ef51a..e50cefa 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,10 @@ -* Test Suite +* Test Suite for clean_repo.py - - Review all repo efectively - - Remove all blacklisted packages - - Get pending list right - - Extract licenses all right \ No newline at end of file + - Review all repo + - Remove all blacklisted packages + - Get pending list right + - Extract licenses all right + +* Fix db-move + + - Make it use abslibre \ No newline at end of file -- cgit v1.2.3 From bef1e554380b797a4dfff5950c5aea59d093b4be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 12:16:21 -0500 Subject: main.sh renamed to repo-update --- repo-state | 11 ----------- repo-update | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 11 deletions(-) delete mode 100755 repo-state create mode 100755 repo-update diff --git a/repo-state b/repo-state deleted file mode 100755 index 319db83..0000000 --- a/repo-state +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -a=$(date +%Y%m%d+%H%M) -mkdir -p $a -cd $a - -. /home/joshpar/programas/arch2parabola/repo-list-diff -# scp parabolavnx@parabolagnulinux.org:~/tmp/rsyncBlacklist.txt ./ - -cd .. -exit 0 \ No newline at end of file diff --git a/repo-update b/repo-update new file mode 100755 index 0000000..b3ecb92 --- /dev/null +++ b/repo-update @@ -0,0 +1,35 @@ +#!/bin/bash +# -*- coding: utf-8 -*- + +source ./config +source ./local_config +source ./libremessages + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]} 'any'; do + msg "Syncing ${repo} ${arch}" + filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + \"${rsync_list_command}\ \ + ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ + ${repodir}/${repo}/\" + find ${repodir}/${repo} -name *${PKGEXT} -print \ + > ${rsync_not_needed} + ${rsync_update_command} \ + ${mirror}${mirrorpath}/${repo}/os/${arch} \ + ${repodir}/${repo} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed} + done + for arch in ${ARCHES[@]}; do + if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + clean_repo.py -k ${blacklist} -w ${whitelist} \ + -p ${docs_dir}/pending-${repo} \ + -b ${repodir}/${repo}/${repo}${DBEXT} + fi + clean_repo.py -k ${blacklist} -d ${repodir}/${repo} +done + +db-update +ftpdir-cleanup + +get_license.sh -- cgit v1.2.3 From fbc8eec9e5cb3d598fb84579811f16ff8c2b71d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 12:26:13 -0500 Subject: Fixed error in repo-update --- repo-update | 1 + 1 file changed, 1 insertion(+) diff --git a/repo-update b/repo-update index b3ecb92..59a0658 100755 --- a/repo-update +++ b/repo-update @@ -27,6 +27,7 @@ for repo in ${PKGREPOS[@]}; do -b ${repodir}/${repo}/${repo}${DBEXT} fi clean_repo.py -k ${blacklist} -d ${repodir}/${repo} + done done db-update -- cgit v1.2.3 From e0aad034cd3b7e91137f85584f9b53cdc871f44f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 12:42:00 -0500 Subject: Modified local_config.example --- filter.py | 4 ++-- local_config.example | 10 +--------- main.sh | 35 ----------------------------------- 3 files changed, 3 insertions(+), 46 deletions(-) delete mode 100755 main.sh diff --git a/filter.py b/filter.py index 4abdf38..c71f54d 100755 --- a/filter.py +++ b/filter.py @@ -4,9 +4,9 @@ from glob import glob from repm.config import * import tarfile -def listado(filename,start=0,end=None): +def listado(filename, start=0, end=None): """Obtiene una lista de paquetes de un archivo.""" - fsock = open(filename,"r") + fsock = open(filename, "r") lista = fsock.read().split("\n") fsock.close() if end is not None: diff --git a/local_config.example b/local_config.example index 2c0ef3e..0a827f4 100644 --- a/local_config.example +++ b/local_config.example @@ -3,9 +3,7 @@ mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" # Directories: they should end without / -## Optionals -paraboladir= -## Must be defined +paraboladir=~/parabolagnulinux.org tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs @@ -20,12 +18,6 @@ whitelist=${docs_dir}/whitelist.txt rsync_blacklist=${docs_dir}/rsyncBlacklist rsync_not_needed=${tmp}/rsync_not_needed -# Repos, arches, and dirs for repo -repo_list="core:extra:community:testing:community-testing:multilib" -dir_list="pool" -arch_list="i686:x86_64" -other="any" - # Rsync commands rsync_list_command="rsync\ -ptgoL\ --list-only\ " rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" diff --git a/main.sh b/main.sh deleted file mode 100755 index b3ecb92..0000000 --- a/main.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- - -source ./config -source ./local_config -source ./libremessages - -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]} 'any'; do - msg "Syncing ${repo} ${arch}" - filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ - \"${rsync_list_command}\ \ - ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ - ${repodir}/${repo}/\" - find ${repodir}/${repo} -name *${PKGEXT} -print \ - > ${rsync_not_needed} - ${rsync_update_command} \ - ${mirror}${mirrorpath}/${repo}/os/${arch} \ - ${repodir}/${repo} \ - --exclude-from=${rsync_blacklist} \ - --exclude-from=${rsync_not_needed} - done - for arch in ${ARCHES[@]}; do - if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then - clean_repo.py -k ${blacklist} -w ${whitelist} \ - -p ${docs_dir}/pending-${repo} \ - -b ${repodir}/${repo}/${repo}${DBEXT} - fi - clean_repo.py -k ${blacklist} -d ${repodir}/${repo} -done - -db-update -ftpdir-cleanup - -get_license.sh -- cgit v1.2.3 From e341449526be63b061e6d2661e5f539b959e8c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 14:49:51 -0500 Subject: Will this solve problems? --- repo-update | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repo-update b/repo-update index 59a0658..495f515 100755 --- a/repo-update +++ b/repo-update @@ -8,7 +8,7 @@ source ./libremessages for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} 'any'; do msg "Syncing ${repo} ${arch}" - filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + python filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ \"${rsync_list_command}\ \ ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ ${repodir}/${repo}/\" @@ -22,11 +22,11 @@ for repo in ${PKGREPOS[@]}; do done for arch in ${ARCHES[@]}; do if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then - clean_repo.py -k ${blacklist} -w ${whitelist} \ + python clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo} \ -b ${repodir}/${repo}/${repo}${DBEXT} fi - clean_repo.py -k ${blacklist} -d ${repodir}/${repo} + python clean_repo.py -k ${blacklist} -d ${repodir}/${repo} done done -- cgit v1.2.3 From 6ce3ddee58415e052ec578e777994a60c7e4c3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 17 Apr 2011 03:11:36 -0300 Subject: sourceballs2 --- cron-jobs/sourceballs2 | 15 +++++++++++++-- db-functions | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index ba35298..b70f417 100644 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -21,8 +21,9 @@ find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null -failedpkgs=() for repo in ${PKGREPOS[@]}; do + failedpkgs=() + pushd $repo >/dev/null find . -maxdepth 1 -type d | while read pkg; do pushd "${SVNREPO}/$repo/$pkg" >/dev/null @@ -42,13 +43,23 @@ for repo in ${PKGREPOS[@]}; do [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ continue + msg2 "$pkgbase-$pkgver-$pkgrel..." makepkg --allsource --ignorearch -c - [[ $? -ne 0 ]] && \ + [[ $? -ne 0 ]] && { + warning "Failed." failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + } done popd >/dev/null + + if [ ${#failedpkgs[@]} -ge 1 ]; then + msg "Failed to create source packages for [${repo}]..." + for failed_pkg in ${failedpkgs[@]}; do + msg2 "${failed_pkg}" + done + fi done # Cleanup old source packages diff --git a/db-functions b/db-functions index e2731cb..5953ed1 100644 --- a/db-functions +++ b/db-functions @@ -353,6 +353,7 @@ check_splitpkgs() { for pkgfile in ${pkgfiles[@]}; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" + msg2 "Checking $_pkgbase" local _pkgname="$(getpkgname ${pkgfile})" local _pkgarch="$(getpkgarch ${pkgfile})" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" -- cgit v1.2.3 From 86d44994d4129c0cf0ffdc9cba71303b870f14d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 17 Apr 2011 03:24:42 -0300 Subject: fixed merge --- db-functions | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/db-functions b/db-functions index 6e0a6d3..fb20e3e 100644 --- a/db-functions +++ b/db-functions @@ -497,10 +497,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null -<<<<<<< HEAD - repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ -======= - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ + repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ >>>>>>> 1ce0c6368d0908e25f9bd1bb8183b5f29053fac8 || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ @@ -520,11 +517,7 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi -<<<<<<< HEAD - repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ -======= - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} \ ->>>>>>> 1ce0c6368d0908e25f9bd1bb8183b5f29053fac8 + repo-remove -q "${dbfile}" ${pkgs[@]} \ || error "repo-remove ${dbfile} ${pkgs[@]}" /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" -- cgit v1.2.3 From e891caf97ee563e52b8f5e1f2448db2b0dc49228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 17 Apr 2011 03:26:03 -0300 Subject: More merge fixes --- db-functions | 1 - 1 file changed, 1 deletion(-) diff --git a/db-functions b/db-functions index fb20e3e..ff38d62 100644 --- a/db-functions +++ b/db-functions @@ -498,7 +498,6 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ ->>>>>>> 1ce0c6368d0908e25f9bd1bb8183b5f29053fac8 || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" -- cgit v1.2.3 From c44ba7fa5154e71921d64f2065a71a4c6755c61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 19:14:41 -0500 Subject: Next fix --- .gitignore | 2 ++ clean_repo.py | 12 ++++++++---- filter.py | 16 +++++++++++----- local_config.example | 13 ++++++++----- repo-update | 19 ++++++++++--------- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index fb6241a..45688ab 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ local_config /config.local test/packages/*/*.pkg.tar.?z +#*# +.#* \ No newline at end of file diff --git a/clean_repo.py b/clean_repo.py index e1a17c2..bc401b5 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -48,10 +48,13 @@ def remove_from_blacklist(path_to_db, blacklisted_names): def cleanup_nonfree_in_dir(directory, blacklisted_names): if "~" in directory: directory=(os.path.expanduser(directory)) + pkglist=list() pkgs=pkginfo_from_files_in_dir(directory) for package in pkgs: if package["name"] in blacklisted_names: os.remove(package["location"]) + pkglist.append(package) + return pkglist if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -64,10 +67,10 @@ if __name__ == "__main__": group_dir=parser.add_argument_group("Clean non-free packages in dir") group_dir.add_argument("-d", "--directory", type=str, - help="directory to clean",) + help="directory to clean") group_db=parser.add_argument_group("Clean non-free packages in db", - "All arguments need to be specified") + "All arguments need to be specified for db cleaning") group_db.add_argument("-b", "--database", type=str, help="dabatase to clean") group_db.add_argument("-p", "--pending-file", type=str, @@ -82,8 +85,8 @@ if __name__ == "__main__": elif not args.pending_file or not args.whitelist_file \ and args.database: parser.print_help() - else: - blacklisted=listado(args.blacklist_file) + + blacklisted=listado(args.blacklist_file) if args.database: whitelisted=listado(args.whitelist_file) @@ -94,3 +97,4 @@ if __name__ == "__main__": if args.directory: cleanup_nonfree_in_dir(args.directory, blacklisted) + diff --git a/filter.py b/filter.py index c71f54d..5d90bdd 100755 --- a/filter.py +++ b/filter.py @@ -1,4 +1,4 @@ - #! /usr/bin/python +#! /usr/bin/python #-*- encoding: utf-8 -*- from glob import glob from repm.config import * @@ -188,11 +188,17 @@ if __name__ == "__main__": parser.add_argument("-k", "--blacklist-file", type=str, help="File containing blacklisted names", required=True,) - parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list", + parser.add_argument("-f", "--rsout-file", type=str, + help="This file will be read to get a pkg list", required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command.split()) + try: + fsock=open(args.rsout_file, "r") + rsout=fsock.read() + except IOError: + print("%s is not readable" % args.rsout_file) + finally: + fsock.close() packages=pkginfo_from_rsync_output(rsout) - rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), + rsyncBlacklist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) diff --git a/local_config.example b/local_config.example index 0a827f4..05694e9 100644 --- a/local_config.example +++ b/local_config.example @@ -1,6 +1,6 @@ # Mirror options mirror="mirrors.eu.kernel.org" -mirrorpath="::mirrors/archlinux" +mirrorpath="mirrors/archlinux" # Directories: they should end without / paraboladir=~/parabolagnulinux.org @@ -13,11 +13,14 @@ licenses_dir=${docs_dir}/pending_licenses # Files logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log +rsout_file=${tempdir}/rsout +rsync_not_needed=${tempdir}/rsync_not_needed + +rsync_blacklist=${docs_dir}/rsyncBlacklist + blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt -rsync_blacklist=${docs_dir}/rsyncBlacklist -rsync_not_needed=${tmp}/rsync_not_needed # Rsync commands -rsync_list_command="rsync\ -ptgoL\ --list-only\ " -rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" +rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " +rsync_update_command="rsync -rptgoL --exclude='*.abs.tar.*' --no-motd " diff --git a/repo-update b/repo-update index 495f515..9b36d75 100755 --- a/repo-update +++ b/repo-update @@ -8,21 +8,22 @@ source ./libremessages for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} 'any'; do msg "Syncing ${repo} ${arch}" - python filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ - \"${rsync_list_command}\ \ - ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ - ${repodir}/${repo}/\" - find ${repodir}/${repo} -name *${PKGEXT} -print \ - > ${rsync_not_needed} + ${rsync_list_command} \ + ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/${repo}/ > ${rsout_file} + filter.py -r ${rsync_blacklist} -k ${blacklist} \ + -f ${rsout_file} + find ${repodir}/${repo} -name *${PKGEXT} \ + -fprint ${rsync_not_needed} ${rsync_update_command} \ - ${mirror}${mirrorpath}/${repo}/os/${arch} \ - ${repodir}/${repo} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} + ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/${repo}/ done for arch in ${ARCHES[@]}; do if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then - python clean_repo.py -k ${blacklist} -w ${whitelist} \ + clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo} \ -b ${repodir}/${repo}/${repo}${DBEXT} fi -- cgit v1.2.3 From 79e48771c2d64b50db5c8dadb4636e98fab94480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 19:23:37 -0500 Subject: Next fix --- repo-update | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repo-update b/repo-update index 9b36d75..2354e0a 100755 --- a/repo-update +++ b/repo-update @@ -1,9 +1,9 @@ #!/bin/bash # -*- coding: utf-8 -*- -source ./config -source ./local_config -source ./libremessages +source config +source local_config +source libremessages for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} 'any'; do -- cgit v1.2.3 From a79cd8ac55d222e182bde411b94bb19cb74fa728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 18:36:24 -0700 Subject: rsync, filter.py, find cmd working now --- repo-update | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/repo-update b/repo-update index 2354e0a..719ab0c 100755 --- a/repo-update +++ b/repo-update @@ -6,28 +6,39 @@ source local_config source libremessages for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]} 'any'; do + for arch in 'i686' 'x86_64' 'any'; do msg "Syncing ${repo} ${arch}" + # makes a file containing rsync output for filter.py ${rsync_list_command} \ - ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/${repo}/ > ${rsout_file} + rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/staging/${repo}/ > ${rsout_file} + # reads blacklist and rsout_file and makes an rsync exclude-from + # list filter.py -r ${rsync_blacklist} -k ${blacklist} \ -f ${rsout_file} - find ${repodir}/${repo} -name *${PKGEXT} \ - -fprint ${rsync_not_needed} + # list files in ${repodir}/${repo} and write their names on + # rsync_not_needed for using as an rsync exclude-from + find ${repodir}/${repo} -name "*${PKGEXT}" \ + -fprintf ${rsync_not_needed} '%f\n' + # Actual rsync command ${rsync_update_command} \ --exclude-from=${rsync_blacklist} \ - --exclude-from=${rsync_not_needed} - ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/${repo}/ + --exclude-from=${rsync_not_needed} \ + rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/staging/${repo}/ done for arch in ${ARCHES[@]}; do + msg2 "Cleaning" + # if there is a db in repo (db is created on rsync) if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + # clean_repo makes pending list with files on db and remove + # packages from db clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo} \ - -b ${repodir}/${repo}/${repo}${DBEXT} + -b ${repodir}/staging/${repo}/${repo}${DBEXT} fi - python clean_repo.py -k ${blacklist} -d ${repodir}/${repo} + # if some nonfree files got pass the filter this command delete them + python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done done -- cgit v1.2.3 From b8a883720066e4f91b162ea91719f9a0f15958ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 21:38:28 -0500 Subject: Another fix --- clean_repo.py | 6 ++---- config | 3 ++- repo-update | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index bc401b5..6f3d632 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -80,11 +80,9 @@ if __name__ == "__main__": args=parser.parse_args() - if not args.directory and not args.database: - parser.print_help() - elif not args.pending_file or not args.whitelist_file \ - and args.database: + if args.database and not (args.pending_file and args.whitelist_file): parser.print_help() + exit(1) blacklisted=listado(args.blacklist_file) diff --git a/config b/config index 300893a..7180f9e 100644 --- a/config +++ b/config @@ -1,7 +1,8 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" -PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing' 'multilib') +ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') +PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/repo-update b/repo-update index 719ab0c..7241507 100755 --- a/repo-update +++ b/repo-update @@ -5,7 +5,7 @@ source config source local_config source libremessages -for repo in ${PKGREPOS[@]}; do +for repo in ${ARCHREPOS[@]}; do for arch in 'i686' 'x86_64' 'any'; do msg "Syncing ${repo} ${arch}" # makes a file containing rsync output for filter.py -- cgit v1.2.3 From ea23d5dff7b4e99e62ab582429f4a6f8b0263363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 21:02:56 -0700 Subject: Ready for testing in regular usage --- clean_repo.py | 9 ++++----- get_license.sh | 34 ++++++++++++++++++++++------------ local_config.example | 2 +- repo-update | 13 +++++++------ 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index 6f3d632..51e62d4 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -25,10 +25,9 @@ def mkpending(packages_iterable, pending_file, blacklisted_names, fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + ":" + pkg["license"] for pkg in pkgs]) + "\n") - except(IOError): - raise NonValidFile("Can't read or write %s" % pending_file) - finally: fsock.close() + except(IOError): + printf("Can't read or write %s" % pending_file) return pkgs def remove_from_blacklist(path_to_db, blacklisted_names): @@ -70,7 +69,7 @@ if __name__ == "__main__": help="directory to clean") group_db=parser.add_argument_group("Clean non-free packages in db", - "All arguments need to be specified for db cleaning") + "All these arguments need to be specified for db cleaning:") group_db.add_argument("-b", "--database", type=str, help="dabatase to clean") group_db.add_argument("-p", "--pending-file", type=str, @@ -89,7 +88,7 @@ if __name__ == "__main__": if args.database: whitelisted=listado(args.whitelist_file) pkgs=pkginfo_from_db(args.database) - remove_from_blacklist(args.database, blacklisted) + # remove_from_blacklist(args.database, blacklisted) mkpending(pkgs, args.pending_file, blacklisted, whitelisted) diff --git a/get_license.sh b/get_license.sh index 024876c..b768cd5 100755 --- a/get_license.sh +++ b/get_license.sh @@ -19,27 +19,37 @@ # GNU General Public License for more details. # You should have received a copy of the GNU General Public License - # along with Parabola. If not, see . + # along with Parabola. If not, see . + source ./config source ./local_config source ./libremessages msg "Creating pending licenses list" -pushd ${licenses_dir} +pushd ${licenses_dir} >/dev/null rm -rf ${licenses_dir}/* +popd >/dev/null + +dir=$(mktemp -d ${tempdir}/licenses.XXXX) +pushd $dir > /dev/null -for repo in ${PKGREPOS[@]}; do +for repo in ${ARCHREPOS[@]}; do msg2 "Extracting licenses in ${repo}" - pending=($(cut -d: -f2 ${docs_dir}/pending-${repo})) - pushd ${repodir}/${repo} - for pkg in ${pending[@]}; do + pending=($(cut -d: -f1 ${docs_dir}/pending-${repo}.txt)) + for name in ${pending[@]}; do plain "${pkg}" - bsdtar -xf ${pkg} usr/share/licenses || { - error "${pkg} has no licenses" - } - chmod -r ${pkg} + for pkg in $(find ${repodir}/staging/${repo} -name "${name}-*${PKGEXT}" -printf '%f '); do + chmod +r ${pkg} + bsdtar -xf ${pkg} usr/share/licenses || { + error "${pkg} has no licenses" + } + chmod -r ${pkg} + done done done -popd -exit 0 \ No newline at end of file +mv ${dir}/* ${licenses_dir}/ +rm -rf ${dir} + + +exit 0 diff --git a/local_config.example b/local_config.example index 05694e9..2280cc2 100644 --- a/local_config.example +++ b/local_config.example @@ -7,7 +7,7 @@ paraboladir=~/parabolagnulinux.org tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo/staging +repodir=${paraboladir}/repo licenses_dir=${docs_dir}/pending_licenses # End Directories diff --git a/repo-update b/repo-update index 7241507..2133f46 100755 --- a/repo-update +++ b/repo-update @@ -27,19 +27,20 @@ for repo in ${ARCHREPOS[@]}; do rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ ${repodir}/staging/${repo}/ done - for arch in ${ARCHES[@]}; do - msg2 "Cleaning" + for arch in 'i686' 'x86_64'; do + msg "Cleaning $repo $arch" # if there is a db in repo (db is created on rsync) - if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then # clean_repo makes pending list with files on db and remove # packages from db clean_repo.py -k ${blacklist} -w ${whitelist} \ - -p ${docs_dir}/pending-${repo} \ + -p ${docs_dir}/pending-${repo}.txt \ -b ${repodir}/staging/${repo}/${repo}${DBEXT} fi - # if some nonfree files got pass the filter this command delete them - python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done + # if some nonfree files got pass the filter this command delete them + msg "Fallback cleaning repo" + python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done db-update -- cgit v1.2.3 From f405f3abb1ae471e36d04e688e6583c2079ecc38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 21:09:35 -0700 Subject: using full path for ftpdir-cleanup --- repo-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-update b/repo-update index 2133f46..697e188 100755 --- a/repo-update +++ b/repo-update @@ -44,6 +44,6 @@ for repo in ${ARCHREPOS[@]}; do done db-update -ftpdir-cleanup +~/repm/cron-jobs/ftpdir-cleanup get_license.sh -- cgit v1.2.3 From e3325948742bc89c57836de0db330a64dcfe8afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 22 Apr 2011 00:51:22 -0500 Subject: fix ? --- get_license.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/get_license.sh b/get_license.sh index b768cd5..1b755c6 100755 --- a/get_license.sh +++ b/get_license.sh @@ -51,5 +51,4 @@ done mv ${dir}/* ${licenses_dir}/ rm -rf ${dir} - exit 0 -- cgit v1.2.3 From bba8c33a039ec4d1e1d13f88304ba0aeb754b03e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 22 Apr 2011 00:30:43 -0700 Subject: Midnight fix --- config | 7 +++++-- db-functions | 6 +++--- db-update | 7 ++++--- get_license.sh | 54 ------------------------------------------------------ repo-update | 17 +++++++++-------- 5 files changed, 21 insertions(+), 70 deletions(-) delete mode 100755 get_license.sh diff --git a/config b/config index 7180f9e..6936a9b 100644 --- a/config +++ b/config @@ -1,6 +1,8 @@ -FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +#!/bin/bash +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" + ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') PKGPOOL='pool/packages' @@ -23,7 +25,8 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64 mips64el) +ARCHARCHES=(i686 x86_64) +ARCHES=(${ARCHARCHES[@]} mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" diff --git a/db-functions b/db-functions index 20bfa0e..4dddcb5 100644 --- a/db-functions +++ b/db-functions @@ -211,7 +211,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -499,7 +499,7 @@ arch_repo_add() { pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ + repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -518,7 +518,7 @@ arch_repo_remove() { fi repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" - /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ + repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-update b/db-update index 6bd0209..7604547 100755 --- a/db-update +++ b/db-update @@ -39,9 +39,10 @@ for repo in ${repos[@]}; do # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" #fi done - if ! check_splitpkgs ${repo} ${pkgs[@]}; then - die "Missing split packages for ${repo}" - fi + # This is fucking obnoxious +# if ! check_splitpkgs ${repo} ${pkgs[@]}; then +# die "Missing split packages for ${repo}" +# fi else die "Could not read ${STAGING}" fi diff --git a/get_license.sh b/get_license.sh deleted file mode 100755 index 1b755c6..0000000 --- a/get_license.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- - - # get_license.sh - # Copyright 2010 Joshua Ismael Haase Hernández - - # ---------- GNU General Public License 3 ---------- - - # This file is part of Parabola. - - # Parabola is free software: you can redistribute it and/or modify - # it under the terms of the GNU General Public License as published by - # the Free Software Foundation, either version 3 of the License, or - # (at your option) any later version. - - # Parabola is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of - # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - # GNU General Public License for more details. - - # You should have received a copy of the GNU General Public License - # along with Parabola. If not, see . - -source ./config -source ./local_config -source ./libremessages - -msg "Creating pending licenses list" -pushd ${licenses_dir} >/dev/null -rm -rf ${licenses_dir}/* -popd >/dev/null - -dir=$(mktemp -d ${tempdir}/licenses.XXXX) -pushd $dir > /dev/null - -for repo in ${ARCHREPOS[@]}; do - msg2 "Extracting licenses in ${repo}" - pending=($(cut -d: -f1 ${docs_dir}/pending-${repo}.txt)) - for name in ${pending[@]}; do - plain "${pkg}" - for pkg in $(find ${repodir}/staging/${repo} -name "${name}-*${PKGEXT}" -printf '%f '); do - chmod +r ${pkg} - bsdtar -xf ${pkg} usr/share/licenses || { - error "${pkg} has no licenses" - } - chmod -r ${pkg} - done - done -done - -mv ${dir}/* ${licenses_dir}/ -rm -rf ${dir} - -exit 0 diff --git a/repo-update b/repo-update index 697e188..88ddda3 100755 --- a/repo-update +++ b/repo-update @@ -1,12 +1,12 @@ #!/bin/bash # -*- coding: utf-8 -*- -source config -source local_config -source libremessages +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages for repo in ${ARCHREPOS[@]}; do - for arch in 'i686' 'x86_64' 'any'; do + for arch in ${ARCHARCHES[@]} 'any'; do msg "Syncing ${repo} ${arch}" # makes a file containing rsync output for filter.py ${rsync_list_command} \ @@ -27,7 +27,7 @@ for repo in ${ARCHREPOS[@]}; do rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ ${repodir}/staging/${repo}/ done - for arch in 'i686' 'x86_64'; do + for arch in ${ARCHARCHES[@]}; do msg "Cleaning $repo $arch" # if there is a db in repo (db is created on rsync) if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then @@ -43,7 +43,8 @@ for repo in ${ARCHREPOS[@]}; do python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done -db-update -~/repm/cron-jobs/ftpdir-cleanup +msg "Removing leftover files..." +find ${repodir}/staging/ \! -name "*${PKGEXT}*" -delete -get_license.sh +db-update +$(dirname $0)/cron-jobs/ftpdir-cleanup -- cgit v1.2.3 From 65cb50c483621d6043685513531001a24228062b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 22 Apr 2011 01:04:55 -0700 Subject: Clean_repo deletes pending-files --- clean_repo.py | 10 ++++++---- repo-update | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index 51e62d4..cc8e811 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -88,10 +88,12 @@ if __name__ == "__main__": if args.database: whitelisted=listado(args.whitelist_file) pkgs=pkginfo_from_db(args.database) - # remove_from_blacklist(args.database, blacklisted) - mkpending(pkgs, args.pending_file, - blacklisted, whitelisted) + pending_names=[pkg["name"] for pkg in + mkpending(pkgs, args.pending_file, + blacklisted, whitelisted)] - if args.directory: + if args.directory and args.database: + cleanup_nonfree_in_dir(args.directory, (blacklisted + pending_names)) + elif args.directory: cleanup_nonfree_in_dir(args.directory, blacklisted) diff --git a/repo-update b/repo-update index 88ddda3..8ad5aef 100755 --- a/repo-update +++ b/repo-update @@ -6,8 +6,9 @@ source $(dirname $0)/local_config source $(dirname $0)/libremessages for repo in ${ARCHREPOS[@]}; do + msg "Syncing ${repo}" for arch in ${ARCHARCHES[@]} 'any'; do - msg "Syncing ${repo} ${arch}" + msg2 "${repo} ${arch}" # makes a file containing rsync output for filter.py ${rsync_list_command} \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ @@ -28,18 +29,19 @@ for repo in ${ARCHREPOS[@]}; do ${repodir}/staging/${repo}/ done for arch in ${ARCHARCHES[@]}; do - msg "Cleaning $repo $arch" + msg2 "Making pending list for $repo $arch" # if there is a db in repo (db is created on rsync) if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then # clean_repo makes pending list with files on db and remove # packages from db clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo}.txt \ - -b ${repodir}/staging/${repo}/${repo}${DBEXT} + -b ${repodir}/staging/${repo}/${repo}${DBEXT} \ + -d ${repodir}/stagging/${repo} fi done # if some nonfree files got pass the filter this command delete them - msg "Fallback cleaning repo" + msg2 "Fallback cleaning repo" python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done -- cgit v1.2.3 From 63bf094a4e472b2bc15ed03fb956401b9246660e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 22 Apr 2011 18:22:15 -0700 Subject: repo-update source ~/.bashrc --- repo-update | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo-update b/repo-update index 8ad5aef..d06b927 100755 --- a/repo-update +++ b/repo-update @@ -1,6 +1,6 @@ #!/bin/bash # -*- coding: utf-8 -*- - +source ~/.bashrc source $(dirname $0)/config source $(dirname $0)/local_config source $(dirname $0)/libremessages @@ -46,7 +46,7 @@ for repo in ${ARCHREPOS[@]}; do done msg "Removing leftover files..." -find ${repodir}/staging/ \! -name "*${PKGEXT}*" -delete +find ${repodir}/staging/ -type f \! -name "*${PKGEXT}" -delete db-update $(dirname $0)/cron-jobs/ftpdir-cleanup -- cgit v1.2.3 From 4b2bd9a51f8f59106234f8889ae0ad98598f33bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 25 Apr 2011 15:54:01 -0700 Subject: * Added a cleaning cmd for symbolic links in staging * Added repo-add, repo-remove --- repo-add | 558 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repo-remove | 505 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ repo-update | 10 +- 3 files changed, 1069 insertions(+), 4 deletions(-) create mode 100755 repo-add create mode 100755 repo-remove diff --git a/repo-add b/repo-add new file mode 100755 index 0000000..531d347 --- /dev/null +++ b/repo-add @@ -0,0 +1,558 @@ +#!/bin/bash +# +# repo-add - add a package to a given repo database file +# repo-remove - remove a package entry from a given repo database file +# Generated from repo-add.in; do not edit by hand. +# +# Copyright (c) 2006-2008 Aaron Griffin +# Copyright (c) 2007-2008 Dan McGee +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +LICENSESDIR=/home/parabolavnx/licenses + +# gettext initialization +export TEXTDOMAIN='pacman' +export TEXTDOMAINDIR='/usr/share/locale' + +myver='3.5.0' +confdir='/home/parabolavnx/etc' + +QUIET=0 +DELTA=0 +WITHFILES=0 +REPO_DB_FILE= +LOCKFILE= +CLEAN_LOCK=0 + +# ensure we have a sane umask set +umask 0022 + +msg() { + (( QUIET )) && return + local mesg=$1; shift + printf "==> ${mesg}\n" "$@" >&1 +} + +msg2() { + (( QUIET )) && return + local mesg=$1; shift + printf " -> ${mesg}\n" "$@" >&1 +} + +warning() { + local mesg=$1; shift + printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 +} + +# print usage instructions +usage() { + printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" + printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" + printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" + printf "$(gettext "\ +repo-add will update a package database by reading a package file.\n\ +Multiple packages to add can be specified on the command line.\n\n")" + printf "$(gettext "\ +repo-remove will update a package database by removing the package name\n\ +specified on the command line from the given repo database. Multiple\n\ +packages to remove can be specified on the command line.\n\n")" + printf "$(gettext "\ +Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ +and errors.\n\n")" + printf "$(gettext "\ +Use the -d/--delta flag to automatically generate and add a delta file\n\ +between the old entry and the new one, if the old package file is found\n\ +next to the new one.\n\n")" + printf "$(gettext "\ +Use the -f/--files flag to update a database including file entries.\n\n")" + echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" + echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" +} + +version() { + printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" + printf "$(gettext "\ +Copyright (C) 2006-2008 Aaron Griffin .\n\ +Copyright (c) 2007-2008 Dan McGee .\n\n\ +This is free software; see the source for copying conditions.\n\ +There is NO WARRANTY, to the extent permitted by law.\n")" +} + +# write a list entry +# arg1 - Entry name +# arg2 - List +# arg3 - File to write to +write_list_entry() { + if [[ -n $2 ]]; then + echo "%$1%" >>$3 + echo -e $2 >>$3 + fi +} + +find_pkgentry() +{ + local pkgname=$1 + local pkgentry + for pkgentry in $tmpdir/$pkgname*; do + name=${pkgentry##*/} + if [[ ${name%-*-*} = $pkgname ]]; then + echo $pkgentry + return 0 + fi + done + return 1 +} + +# Get the package name from the delta filename +get_delta_pkgname() { + local tmp + + tmp=${1##*/} + echo ${tmp%-*-*_to*} +} + +# write a delta entry +# arg1 - path to delta file +db_write_delta() +{ + deltafile="$1" + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [[ -z $pkgentry ]]; then + error "$(gettext "No database entry for package '%s'.")" "$pkgname" + return 1 + fi + deltas="$pkgentry/deltas" + if [[ ! -f $deltas ]]; then + echo -e "%DELTAS%" >$deltas + fi + # get md5sum and compressed size of package + md5sum="$(openssl dgst -md5 "$deltafile")" + md5sum="${md5sum##* }" + csize=$(stat -L -c %s "$deltafile") + + oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') + newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') + + if grep -q "$oldfile.*$newfile" $deltas; then + sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup + fi + msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" + echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas + + return 0 +} # end db_write_delta + +# remove a delta entry +# arg1 - path to delta file +db_remove_delta() +{ + deltafile="$1" + filename=${deltafile##*/} + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [[ -z $pkgentry ]]; then + return 1 + fi + deltas="$pkgentry/deltas" + if [[ ! -f $deltas ]]; then + return 1 + fi + if grep -q "$filename" $deltas; then + sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup + msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" + return 0 + fi + + return 1 +} # end db_remove_delta + +# write an entry to the pacman database +# arg1 - path to package +db_write_entry() +{ + # blank out all variables + local pkgfile="$1" + local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ + _groups _licenses _replaces _depends _conflicts _provides _optdepends + + local OLDIFS="$IFS" + # IFS (field separator) is only the newline character + IFS=" +" + + # read info from the zipped package + local line var val + for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | + grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do + # bash awesomeness here- var is always one word, val is everything else + var=${line%% *} + val=${line#* } + declare $var="$val" + case "$var" in + group) _groups="$_groups$group\n" ;; + license) _licenses="$_licenses$license\n" ;; + replaces) _replaces="$_replaces$replaces\n" ;; + depend) _depends="$_depends$depend\n" ;; + conflict) _conflicts="$_conflicts$conflict\n" ;; + provides) _provides="$_provides$provides\n" ;; + optdepend) _optdepends="$_optdepends$optdepend\n" ;; + esac + done + + IFS=$OLDIFS + + # get md5sum and compressed size of package + md5sum="$(openssl dgst -md5 "$pkgfile")" + md5sum="${md5sum##* }" + csize=$(stat -L -c %s "$pkgfile") + + # ensure $pkgname and $pkgver variables were found + if [[ -z $pkgname || -z $pkgver ]]; then + error "$(gettext "Invalid package file '%s'.")" "$pkgfile" + return 1 + fi + + pushd "$tmpdir" >/dev/null + if [[ -d $pkgname-$pkgver ]]; then + warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" + else + if (( DELTA )); then + pkgentry=$(find_pkgentry $pkgname) + if [[ -n $pkgentry ]]; then + local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) + local oldfile="$(dirname $1)/$oldfilename" + fi + fi + fi + + # remove an existing entry if it exists, ignore failures + db_remove_entry "$pkgname" + + # create package directory + mkdir "$pkgname-$pkgver" + pushd "$pkgname-$pkgver" >/dev/null + + # restore an eventual deltas file + [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas + + # create desc entry + msg2 "$(gettext "Creating '%s' db entry...")" 'desc' + echo -e "%FILENAME%\n$(basename "$1")\n" >>desc + echo -e "%NAME%\n$pkgname\n" >>desc + [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc + echo -e "%VERSION%\n$pkgver\n" >>desc + [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc + write_list_entry "GROUPS" "$_groups" "desc" + [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc + [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc + + # compute checksums + msg2 "$(gettext "Computing md5 checksums...")" + echo -e "%MD5SUM%\n$md5sum\n" >>desc + + [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc + write_list_entry "LICENSE" "$_licenses" "desc" + [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc + [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc + [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc + write_list_entry "REPLACES" "$_replaces" "desc" + + # create depends entry + msg2 "$(gettext "Creating '%s' db entry...")" 'depends' + # create the file even if it will remain empty + touch "depends" + write_list_entry "DEPENDS" "$_depends" "depends" + write_list_entry "CONFLICTS" "$_conflicts" "depends" + write_list_entry "PROVIDES" "$_provides" "depends" + write_list_entry "OPTDEPENDS" "$_optdepends" "depends" + + popd >/dev/null + popd >/dev/null + + # create files file if wanted + if (( WITHFILES )); then + msg2 "$(gettext "Creating '%s' db entry...")" 'files' + local files_path="$tmpdir/$pkgname-$pkgver/files" + echo "%FILES%" >$files_path + bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path + fi + + # create a delta file + if (( DELTA )); then + if [[ -n $oldfilename ]]; then + if [[ -f $oldfile ]]; then + delta=$(pkgdelta -q $oldfile $1) + if [[ -f $delta ]]; then + db_write_delta $delta + fi + else + warning "$(gettext "Old package file not found: %s")" "$oldfilename" + fi + fi + fi + + # Extracts licenses to a common license dir + msg "Extracting license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + # Change dir to licenses, and extract them stripping the first part of the path + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + warning "This package doesn't contain a license dir" + fi + + return 0 +} # end db_write_entry + +# remove existing entries from the DB +# arg1 - package name +db_remove_entry() { + local pkgname=$1 + local notfound=1 + local pkgentry=$(find_pkgentry $pkgname) + while [[ -n $pkgentry ]]; do + notfound=0 + if [[ -f $pkgentry/deltas ]]; then + mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" + fi + msg2 "$(gettext "Removing existing entry '%s'...")" \ + "$(basename $pkgentry)" + rm -rf $pkgentry + pkgentry=$(find_pkgentry $pkgname) + done + + msg "Removing license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + return $notfound +} # end db_remove_entry + +check_repo_db() +{ + # check lock file + if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then + CLEAN_LOCK=1 + else + error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" + [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" + exit 1 + fi + + if [[ -f $REPO_DB_FILE ]]; then + # there are two situations we can have here- a DB with some entries, + # or a DB with no contents at all. + if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then + # check empty case + if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then + error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" + exit 1 + fi + fi + msg "$(gettext "Extracting database to a temporary location...")" + bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" + else + case "$cmd" in + repo-remove) + error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" + exit 1 + ;; + repo-add) + # check if the file can be created (write permission, directory existence, etc) + if ! touch "$REPO_DB_FILE"; then + error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" + exit 1 + fi + rm -f "$REPO_DB_FILE" + ;; + esac + fi +} + +add() +{ + if [[ ! -f $1 ]]; then + error "$(gettext "File '%s' not found.")" "$1" + return 1 + fi + + if [[ ${1##*.} == "delta" ]]; then + deltafile=$1 + msg "$(gettext "Adding delta '%s'")" "$deltafile" + if ! type xdelta3 &>/dev/null; then + error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" + exit 1 + fi + if db_write_delta "$deltafile"; then + return 0 + else + return 1 + fi + fi + + pkgfile=$1 + if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then + error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" + return 1 + fi + + msg "$(gettext "Adding package '%s'")" "$pkgfile" + + db_write_entry "$pkgfile" +} + +remove() +{ + if [[ ${1##*.} == "delta" ]]; then + deltafile=$1 + msg "$(gettext "Searching for delta '%s'...")" "$deltafile" + if db_remove_delta "$deltafile"; then + return 0 + else + error "$(gettext "Delta matching '%s' not found.")" "$deltafile" + return 1 + fi + fi + + pkgname=$1 + msg "$(gettext "Searching for package '%s'...")" "$pkgname" + + if db_remove_entry "$pkgname"; then + rm -f "$tmpdir/$pkgname.deltas" + return 0 + else + error "$(gettext "Package matching '%s' not found.")" "$pkgname" + return 1 + fi +} + +trap_exit() +{ + echo + error "$@" + exit 1 +} + +clean_up() { + local exit_code=$? + + [[ -d $tmpdir ]] && rm -rf "$tmpdir" + (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" + + exit $exit_code +} + +# PROGRAM START + +# determine whether we have gettext; make it a no-op if we do not +if ! type gettext &>/dev/null; then + gettext() { + echo "$@" + } +fi + +case "$1" in + -h|--help) usage; exit 0;; + -V|--version) version; exit 0;; +esac + +# figure out what program we are +cmd="$(basename $0)" +if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then + error "$(gettext "Invalid command name '%s' specified.")" "$cmd" + exit 1 +fi + +tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ + error "$(gettext "Cannot create temp directory for database building.")"; \ + exit 1) + +trap 'clean_up' EXIT +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR + +success=0 +# parse arguments +for arg in "$@"; do + case "$arg" in + -q|--quiet) QUIET=1;; + -d|--delta) DELTA=1;; + -f|--files) WITHFILES=1;; + *) + if [[ -z $REPO_DB_FILE ]]; then + REPO_DB_FILE="$arg" + LOCKFILE="$REPO_DB_FILE.lck" + check_repo_db + else + case "$cmd" in + repo-add) add $arg && success=1 ;; + repo-remove) remove $arg && success=1 ;; + esac + fi + ;; + esac +done + +# if at least one operation was a success, re-zip database +if (( success )); then + msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" + + case "$REPO_DB_FILE" in + *tar.gz) TAR_OPT="z" ;; + *tar.bz2) TAR_OPT="j" ;; + *tar.xz) TAR_OPT="J" ;; + *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ + "$REPO_DB_FILE" ;; + esac + + filename=$(basename "$REPO_DB_FILE") + + pushd "$tmpdir" >/dev/null + if [[ -n $(ls) ]]; then + bsdtar -c${TAR_OPT}f "$filename" * + else + # we have no packages remaining? zip up some emptyness + warning "$(gettext "No packages remain, creating empty database.")" + bsdtar -c${TAR_OPT}f "$filename" -T /dev/null + fi + popd >/dev/null + + [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" + [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" + dblink="${REPO_DB_FILE%.tar.*}" + target=${REPO_DB_FILE##*/} + ln -sf "$target" "$dblink" 2>/dev/null || \ + ln -f "$target" "$dblink" 2>/dev/null || \ + cp "$REPO_DB_FILE" "$dblink" +else + msg "$(gettext "No packages modified, nothing to do.")" + exit 1 +fi + +exit 0 +# vim: set ts=2 sw=2 noet: diff --git a/repo-remove b/repo-remove new file mode 100755 index 0000000..ef6f186 --- /dev/null +++ b/repo-remove @@ -0,0 +1,505 @@ +#!/bin/bash +# +# repo-add - add a package to a given repo database file +# repo-remove - remove a package entry from a given repo database file +# Generated from repo-add.in; do not edit by hand. +# +# Copyright (c) 2006-2008 Aaron Griffin +# Copyright (c) 2007-2008 Dan McGee +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# gettext initialization +export TEXTDOMAIN='pacman' +export TEXTDOMAINDIR='/usr/share/locale' + +myver='3.3.3' +confdir='/etc' + +QUIET=0 +REPO_DB_FILE= +LOCKFILE= +CLEAN_LOCK=0 +startdir="$PWD" + +# ensure we have a sane umask set +umask 0022 + +msg() { + local mesg=$1; shift + printf "==> ${mesg}\n" "$@" >&1 +} + +msg2() { + [ $QUIET -ne 0 ] && return + local mesg=$1; shift + printf " -> ${mesg}\n" "$@" >&1 +} + +warning() { + local mesg=$1; shift + printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 +} + +# print usage instructions +usage() { + printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" + printf "$(gettext "Usage: repo-add [-q] ...\n")" + printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" + printf "$(gettext "\ +repo-add will update a package database by reading a package file.\n\ +Multiple packages to add can be specified on the command line.\n\n")" + printf "$(gettext "\ +repo-remove will update a package database by removing the package name\n\ +specified on the command line from the given repo database. Multiple\n\ +packages to remove can be specified on the command line.\n\n")" + printf "$(gettext "\ +Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ +and errors\n\n")" + echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" + echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" +} + +version() { + printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" + printf "$(gettext "\ +Copyright (C) 2006-2008 Aaron Griffin .\n\ +Copyright (c) 2007-2008 Dan McGee .\n\n\ +This is free software; see the source for copying conditions.\n\ +There is NO WARRANTY, to the extent permitted by law.\n")" +} + +# write a list entry +# arg1 - Entry name +# arg2 - List +# arg3 - File to write to +write_list_entry() { + if [ -n "$2" ]; then + echo "%$1%" >>$3 + echo -e $2 >>$3 + fi +} + +find_pkgentry() +{ + local pkgname=$1 + local pkgentry + for pkgentry in $tmpdir/$pkgname*; do + name=${pkgentry##*/} + if [ "${name%-*-*}" = "$pkgname" ]; then + echo $pkgentry + return 0 + fi + done + return 1 +} + +# Get the package name from the delta filename +get_delta_pkgname() { + local tmp + + tmp=${1##*/} + echo ${tmp%-*-*_to*} +} + +# write a delta entry +# arg1 - path to delta file +db_write_delta() +{ + deltafile="$1" + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [ -z "$pkgentry" ]; then + return 1 + fi + deltas="$pkgentry/deltas" + # create deltas file if it does not already exist + if [ ! -f "$deltas" ]; then + msg2 "$(gettext "Creating 'deltas' db entry...")" + echo -e "%DELTAS%" >>$deltas + fi + # get md5sum and compressed size of package + md5sum="$(openssl dgst -md5 "$deltafile" | awk '{print $NF}')" + csize=$(stat -L -c %s "$deltafile") + + oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') + newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') + + if grep -q "$oldfile.*$newfile" $deltas; then + warning "$(gettext "An entry for '%s' already existed")" "$deltafile" + sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup + msg2 "$(gettext "Removing existing entry '%s'...")" "$deltafile" + fi + echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas + + return 0 +} # end db_write_delta + +# remove a delta entry +# arg1 - path to delta file +db_remove_delta() +{ + deltafile="$1" + filename=${deltafile##*/} + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [ -z "$pkgentry" ]; then + return 1 + fi + deltas="$pkgentry/deltas" + if [ ! -f "$deltas" ]; then + return 1 + fi + if grep -q "$filename" $deltas; then + sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup + msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" + return 0 + fi + + return 1 +} # end db_remove_delta + +# write an entry to the pacman database +# arg1 - path to package +db_write_entry() +{ + # blank out all variables + local pkgfile="$1" + local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager force \ + _groups _licenses _replaces _depends _conflicts _provides _optdepends + + local OLDIFS="$IFS" + # IFS (field separator) is only the newline character + IFS=" +" + + # read info from the zipped package + local line var val + for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | + grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do + # bash awesomeness here- var is always one word, val is everything else + var=${line%% *} + val=${line#* } + declare $var="$val" + case "$var" in + group) _groups="$_groups$group\n" ;; + license) _licenses="$_licenses$license\n" ;; + replaces) _replaces="$_replaces$replaces\n" ;; + depend) _depends="$_depends$depend\n" ;; + conflict) _conflicts="$_conflicts$conflict\n" ;; + provides) _provides="$_provides$provides\n" ;; + optdepend) _optdepends="$_optdepends$optdepend\n" ;; + esac + done + + IFS=$OLDIFS + + # get md5sum and compressed size of package + md5sum="$(openssl dgst -md5 "$pkgfile" | awk '{print $NF}')" + csize=$(stat -L -c %s "$pkgfile") + + # ensure $pkgname and $pkgver variables were found + if [ -z "$pkgname" -o -z "$pkgver" ]; then + error "$(gettext "Invalid package file '%s'.")" "$pkgfile" + return 1 + fi + + cd "$tmpdir" + + if [ -d "$pkgname-$pkgver" ]; then + warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" + fi + + # remove an existing entry if it exists, ignore failures + db_remove_entry "$pkgname" + + # create package directory + mkdir "$pkgname-$pkgver" + cd "$pkgname-$pkgver" + + # restore an eventual deltas file + [ -f "../$pkgname.deltas" ] && mv "../$pkgname.deltas" deltas + + # create desc entry + msg2 "$(gettext "Creating 'desc' db entry...")" + echo -e "%FILENAME%\n$(basename "$1")\n" >>desc + echo -e "%NAME%\n$pkgname\n" >>desc + [ -n "$pkgbase" ] && echo -e "%BASE%\n$pkgbase\n" >>desc + echo -e "%VERSION%\n$pkgver\n" >>desc + [ -n "$pkgdesc" ] && echo -e "%DESC%\n$pkgdesc\n" >>desc + write_list_entry "GROUPS" "$_groups" "desc" + [ -n "$csize" ] && echo -e "%CSIZE%\n$csize\n" >>desc + [ -n "$size" ] && echo -e "%ISIZE%\n$size\n" >>desc + + # compute checksums + msg2 "$(gettext "Computing md5 checksums...")" + echo -e "%MD5SUM%\n$md5sum\n" >>desc + + [ -n "$url" ] && echo -e "%URL%\n$url\n" >>desc + write_list_entry "LICENSE" "$_licenses" "desc" + [ -n "$arch" ] && echo -e "%ARCH%\n$arch\n" >>desc + [ -n "$builddate" ] && echo -e "%BUILDDATE%\n$builddate\n" >>desc + [ -n "$packager" ] && echo -e "%PACKAGER%\n$packager\n" >>desc + write_list_entry "REPLACES" "$_replaces" "desc" + [ -n "$force" ] && echo -e "%FORCE%\n" >>desc + + # create depends entry + msg2 "$(gettext "Creating 'depends' db entry...")" + # create the file even if it will remain empty + touch "depends" + write_list_entry "DEPENDS" "$_depends" "depends" + write_list_entry "CONFLICTS" "$_conflicts" "depends" + write_list_entry "PROVIDES" "$_provides" "depends" + write_list_entry "OPTDEPENDS" "$_optdepends" "depends" + + cd "$startdir" + + return 0 +} # end db_write_entry + +# remove existing entries from the DB +# arg1 - package name +db_remove_entry() { + local pkgname=$1 + local notfound=1 + local pkgentry=$(find_pkgentry $pkgname) + while [ -n "$pkgentry" ]; do + notfound=0 + if [ -f "$pkgentry/deltas" ]; then + mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" + fi + msg2 "$(gettext "Removing existing entry '%s'...")" \ + "$(basename $pkgentry)" + rm -rf $pkgentry + pkgentry=$(find_pkgentry $pkgname) + done + return $notfound +} # end db_remove_entry + +check_repo_db() +{ + # check lock file + if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then + CLEAN_LOCK=1 + else + error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" + [ -f "$LOCKFILE" ] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" + exit 1 + fi + + if [ -f "$REPO_DB_FILE" ]; then + # there are two situations we can have here- a DB with some entries, + # or a DB with no contents at all. + if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then + # check empty case + if [ -n "$(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null)" ]; then + error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" + exit 1 + fi + fi + msg "$(gettext "Extracting database to a temporary location...")" + bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" + else + case "$cmd" in + repo-remove) + error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" + exit 1 + ;; + repo-add) + # check if the file can be created (write permission, directory existence, etc) + if ! touch "$REPO_DB_FILE"; then + error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" + exit 1 + fi + rm -f "$REPO_DB_FILE" + ;; + esac + fi +} + +add() +{ + if [ ! -f "$1" ]; then + error "$(gettext "File '%s' not found.")" "$1" + return 1 + fi + + if [ "${1##*.}" == "delta" ]; then + deltafile=$1 + msg "$(gettext "Adding delta '%s'")" "$deltafile" + if [ ! "$(type -p xdelta3)" ]; then + error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" + exit 1 + fi + if db_write_delta "$deltafile"; then + return 0 + else + return 1 + fi + fi + + pkgfile=$1 + if ! bsdtar -tqf "$pkgfile" .PKGINFO 2>&1 >/dev/null; then + error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" + return 1 + fi + + msg "$(gettext "Adding package '%s'")" "$pkgfile" + + db_write_entry "$pkgfile" +} + +remove() +{ + if [ "${1##*.}" == "delta" ]; then + deltafile=$1 + msg "$(gettext "Searching for delta '%s'...")" "$deltafile" + if db_remove_delta "$deltafile"; then + return 0 + else + error "$(gettext "Delta matching '%s' not found.")" "$deltafile" + return 1 + fi + fi + + pkgname=$1 + msg "$(gettext "Searching for package '%s'...")" "$pkgname" + + if db_remove_entry "$pkgname"; then + rm -f "$tmpdir/$pkgname.deltas" + return 0 + else + error "$(gettext "Package matching '%s' not found.")" "$pkgname" + return 1 + fi +} + +trap_exit() +{ + echo + error "$@" + exit 1 +} + +clean_up() { + local exit_code=$? + + cd "$startdir" + [ -d "$tmpdir" ] && rm -rf "$tmpdir" + [ $CLEAN_LOCK -eq 1 -a -f "$LOCKFILE" ] && rm -f "$LOCKFILE" + + exit $exit_code +} + +# PROGRAM START + +# determine whether we have gettext; make it a no-op if we do not +if [ ! $(type -t gettext) ]; then + gettext() { + echo "$@" + } +fi + +case "$1" in + -h|--help) usage; exit 0;; + -V|--version) version; exit 0;; +esac + +# check for correct number of args +if [ $# -lt 2 ]; then + usage + exit 1 +fi + +# figure out what program we are +cmd="$(basename $0)" +if [ "$cmd" != "repo-add" -a "$cmd" != "repo-remove" ]; then + error "$(gettext "Invalid command name '%s' specified.")" "$cmd" + exit 1 +fi + +tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ + error "$(gettext "Cannot create temp directory for database building.")"; \ + exit 1) + +trap 'clean_up' EXIT +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR + +success=0 +# parse arguments +for arg in "$@"; do + case "$arg" in + -q|--quiet) QUIET=1;; + + -f|--force) + warning "$(gettext "the -f and --force options are no longer recognized")" + msg2 "$(gettext "use options=(force) in the PKGBUILD instead")" + ;; + + *) + if [ -z "$REPO_DB_FILE" ]; then + REPO_DB_FILE="$arg" + LOCKFILE="$REPO_DB_FILE.lck" + check_repo_db + else + case "$cmd" in + repo-add) add $arg && success=1 ;; + repo-remove) remove $arg && success=1 ;; + esac + fi + ;; + esac +done + +# if at least one operation was a success, re-zip database +if [ $success -eq 1 ]; then + msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" + + case "$REPO_DB_FILE" in + *tar.gz) TAR_OPT="z" ;; + *tar.bz2) TAR_OPT="j" ;; + *tar.xz) TAR_OPT="J" ;; + *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ + "$REPO_DB_FILE" ;; + esac + + filename=$(basename "$REPO_DB_FILE") + + cd "$tmpdir" + if [ -n "$(ls)" ]; then + bsdtar -c${TAR_OPT}f "$filename" * + else + # we have no packages remaining? zip up some emptyness + warning "$(gettext "No packages remain, creating empty database.")" + bsdtar -c${TAR_OPT}f "$filename" -T /dev/null + fi + cd "$startdir" + + [ -f "$REPO_DB_FILE" ] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" + [ -f "$tmpdir/$filename" ] && mv "$tmpdir/$filename" "$REPO_DB_FILE" +else + msg "$(gettext "No packages modified, nothing to do.")" +fi + +exit 0 +# vim: set ts=2 sw=2 noet: diff --git a/repo-update b/repo-update index d06b927..60960f2 100755 --- a/repo-update +++ b/repo-update @@ -34,7 +34,7 @@ for repo in ${ARCHREPOS[@]}; do if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then # clean_repo makes pending list with files on db and remove # packages from db - clean_repo.py -k ${blacklist} -w ${whitelist} \ + $(dirname $0)/clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo}.txt \ -b ${repodir}/staging/${repo}/${repo}${DBEXT} \ -d ${repodir}/stagging/${repo} @@ -42,11 +42,13 @@ for repo in ${ARCHREPOS[@]}; do done # if some nonfree files got pass the filter this command delete them msg2 "Fallback cleaning repo" - python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} + $(dirname $0)/clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done msg "Removing leftover files..." find ${repodir}/staging/ -type f \! -name "*${PKGEXT}" -delete +# Staging should not have symbolic links +find ${repodir}/staging/ -type l -delete -db-update -$(dirname $0)/cron-jobs/ftpdir-cleanup +$(dirname $0)/db-update +#$(dirname $0)/cron-jobs/ftpdir-cleanup -- cgit v1.2.3 From 74093d95cba935badaf7572e1dcd38ffd76bdc53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 8 May 2011 18:45:02 -0500 Subject: * Remove non free packages on db-update --- config | 1 + db-update | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/config b/config index 6936a9b..effdde4 100644 --- a/config +++ b/config @@ -33,3 +33,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="$HOME/etc/makepkg.conf" +BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" \ No newline at end of file diff --git a/db-update b/db-update index 7604547..6c220d9 100755 --- a/db-update +++ b/db-update @@ -69,7 +69,7 @@ for repo in ${repos[@]}; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi - add_pkgs[${#add_pkgs[*]}]=${pkgfile} + add_pkgs[${#add_pkgs[*]}]=${pkgfile} done if [ ${#add_pkgs[@]} -ge 1 ]; then arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} @@ -77,6 +77,21 @@ for repo in ${repos[@]}; do done done +# Repo check nonfree +nonfree=($(cut -d: -f1 ${BLACKLIST_FILE})) +for repo in ${ARCHREPOS[@]}; do + for pkgarch in ${ARCHARCHES[@]}; do + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort )) + for pkgname in ${dbpkgs[@]}; do + if in_array ${pkgname} ${nonfree[@]}; then + cleanpkgs[${#cleanpkgs[*]}]=${pkgname} + fi + done + arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + done +done + for repo in ${repos[@]}; do for pkgarch in ${ARCHES[@]}; do repo_unlock ${repo} ${pkgarch} -- cgit v1.2.3 From 3e7ae87a5a2ac70811a016d684a8a24ae18f49f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 8 May 2011 19:04:39 -0700 Subject: * Unique pkgname list for nonfree and dbpkgs --- db-update | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index 6c220d9..f077aff 100755 --- a/db-update +++ b/db-update @@ -78,17 +78,24 @@ for repo in ${repos[@]}; do done # Repo check nonfree -nonfree=($(cut -d: -f1 ${BLACKLIST_FILE})) +nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) for repo in ${ARCHREPOS[@]}; do - for pkgarch in ${ARCHARCHES[@]}; do + for pkgarch in ${ARCHES[@]}; do + if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then + continue + fi + unset dbpkgs + unset cleanpkgs cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort )) + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) for pkgname in ${dbpkgs[@]}; do if in_array ${pkgname} ${nonfree[@]}; then cleanpkgs[${#cleanpkgs[*]}]=${pkgname} fi done - arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + if [ ${#cleanpkgs[@]} -ge 1 ]; then + arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + fi done done @@ -97,3 +104,5 @@ for repo in ${repos[@]}; do repo_unlock ${repo} ${pkgarch} done done + + -- cgit v1.2.3 From e6e9e1a468c15a6ca609d8c6072a7b8c16fb3833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 15 May 2011 22:01:02 -0500 Subject: db-update msg when cleaning nonfree --- db-update | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db-update b/db-update index f077aff..1606f1d 100755 --- a/db-update +++ b/db-update @@ -78,9 +78,11 @@ for repo in ${repos[@]}; do done # Repo check nonfree +msg "Check nonfree in repo:" nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) for repo in ${ARCHREPOS[@]}; do for pkgarch in ${ARCHES[@]}; do + msg2 "$repo $pkgarch" if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue fi -- cgit v1.2.3 From b88a0166be595545c2b2d68451291f8ba6bb08ab Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Wed, 18 May 2011 21:21:59 -0500 Subject: * Separated nonfree check from db-update * Added db-check-nonfree to repo-update --- db-check-nonfree | 45 +++++++++++++++++++++++++++++++++++++++++++++ db-update | 24 ------------------------ repo-update | 3 ++- 3 files changed, 47 insertions(+), 25 deletions(-) create mode 100644 db-check-nonfree diff --git a/db-check-nonfree b/db-check-nonfree new file mode 100644 index 0000000..83efb14 --- /dev/null +++ b/db-check-nonfree @@ -0,0 +1,45 @@ +#!/bin/bash + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -ge 1 ]; then + warning "Calling $(basename $0) with a specific repository is not supported" + exit 1 +fi + +# TODO: this might lock too much (architectures) +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_lock ${repo} ${pkgarch} || exit 1 + done +done + +msg "Check nonfree in repo:" +nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) +for repo in ${ARCHREPOS[@]}; do + for pkgarch in ${ARCHES[@]}; do + msg2 "$repo $pkgarch" + if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then + continue + fi + unset dbpkgs + unset cleanpkgs + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in ${dbpkgs[@]}; do + if in_array ${pkgname} ${nonfree[@]}; then + cleanpkgs[${#cleanpkgs[*]}]=${pkgname} + fi + done + if [ ${#cleanpkgs[@]} -ge 1 ]; then + arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + fi + done +done + +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_unlock ${repo} ${pkgarch} + done +done diff --git a/db-update b/db-update index 1606f1d..86eaa2e 100755 --- a/db-update +++ b/db-update @@ -77,30 +77,6 @@ for repo in ${repos[@]}; do done done -# Repo check nonfree -msg "Check nonfree in repo:" -nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) -for repo in ${ARCHREPOS[@]}; do - for pkgarch in ${ARCHES[@]}; do - msg2 "$repo $pkgarch" - if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then - continue - fi - unset dbpkgs - unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in ${dbpkgs[@]}; do - if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs[${#cleanpkgs[*]}]=${pkgname} - fi - done - if [ ${#cleanpkgs[@]} -ge 1 ]; then - arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} - fi - done -done - for repo in ${repos[@]}; do for pkgarch in ${ARCHES[@]}; do repo_unlock ${repo} ${pkgarch} diff --git a/repo-update b/repo-update index 60960f2..a44ae87 100755 --- a/repo-update +++ b/repo-update @@ -51,4 +51,5 @@ find ${repodir}/staging/ -type f \! -name "*${PKGEXT}" -delete find ${repodir}/staging/ -type l -delete $(dirname $0)/db-update -#$(dirname $0)/cron-jobs/ftpdir-cleanup +$(dirname $0)/db-check-nonfree + -- cgit v1.2.3 From 8766129748c8ad165b4245f53df1a2466be30848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 18 May 2011 19:28:28 -0700 Subject: db-check-nonfree is executable --- db-check-nonfree | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 db-check-nonfree diff --git a/db-check-nonfree b/db-check-nonfree old mode 100644 new mode 100755 -- cgit v1.2.3 From 303dba7c0ab15cd4e726fe3f72ccf8347df84d10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 10 Jun 2011 16:06:11 -0500 Subject: * repo-add only extract license if package has "custom" license. * repo-remove hardlink to repo-add. * repo-update has some more info output and do not display rsync errors. --- repo-add | 23 ++++--- repo-remove | 204 ++++++++++++++++++++++++++++++++++++++---------------------- repo-update | 8 ++- 3 files changed, 149 insertions(+), 86 deletions(-) diff --git a/repo-add b/repo-add index 531d347..c4bf96f 100755 --- a/repo-add +++ b/repo-add @@ -313,18 +313,21 @@ db_write_entry() # Extracts licenses to a common license dir msg "Extracting license" - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi + if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi # Change dir to licenses, and extract them stripping the first part of the path - bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ + --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if [ $? -ne 0 ]; then warning "This package doesn't contain a license dir" - fi - - return 0 + fi + fi + + return 0 } # end db_write_entry # remove existing entries from the DB @@ -346,8 +349,8 @@ db_remove_entry() { msg "Removing license" if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi + rm -r ${LICENSESDIR}/${pkgname} + fi return $notfound } # end db_remove_entry diff --git a/repo-remove b/repo-remove index ef6f186..c4bf96f 100755 --- a/repo-remove +++ b/repo-remove @@ -20,29 +20,33 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +LICENSESDIR=/home/parabolavnx/licenses + # gettext initialization export TEXTDOMAIN='pacman' export TEXTDOMAINDIR='/usr/share/locale' -myver='3.3.3' -confdir='/etc' +myver='3.5.0' +confdir='/home/parabolavnx/etc' QUIET=0 +DELTA=0 +WITHFILES=0 REPO_DB_FILE= LOCKFILE= CLEAN_LOCK=0 -startdir="$PWD" # ensure we have a sane umask set umask 0022 msg() { + (( QUIET )) && return local mesg=$1; shift printf "==> ${mesg}\n" "$@" >&1 } msg2() { - [ $QUIET -ne 0 ] && return + (( QUIET )) && return local mesg=$1; shift printf " -> ${mesg}\n" "$@" >&1 } @@ -60,7 +64,7 @@ error() { # print usage instructions usage() { printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "Usage: repo-add [-q] ...\n")" + printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" printf "$(gettext "\ repo-add will update a package database by reading a package file.\n\ @@ -71,7 +75,13 @@ specified on the command line from the given repo database. Multiple\n\ packages to remove can be specified on the command line.\n\n")" printf "$(gettext "\ Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ -and errors\n\n")" +and errors.\n\n")" + printf "$(gettext "\ +Use the -d/--delta flag to automatically generate and add a delta file\n\ +between the old entry and the new one, if the old package file is found\n\ +next to the new one.\n\n")" + printf "$(gettext "\ +Use the -f/--files flag to update a database including file entries.\n\n")" echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" } @@ -90,7 +100,7 @@ There is NO WARRANTY, to the extent permitted by law.\n")" # arg2 - List # arg3 - File to write to write_list_entry() { - if [ -n "$2" ]; then + if [[ -n $2 ]]; then echo "%$1%" >>$3 echo -e $2 >>$3 fi @@ -102,7 +112,7 @@ find_pkgentry() local pkgentry for pkgentry in $tmpdir/$pkgname*; do name=${pkgentry##*/} - if [ "${name%-*-*}" = "$pkgname" ]; then + if [[ ${name%-*-*} = $pkgname ]]; then echo $pkgentry return 0 fi @@ -126,27 +136,26 @@ db_write_delta() pkgname="$(get_delta_pkgname $deltafile)" pkgentry=$(find_pkgentry $pkgname) - if [ -z "$pkgentry" ]; then + if [[ -z $pkgentry ]]; then + error "$(gettext "No database entry for package '%s'.")" "$pkgname" return 1 fi deltas="$pkgentry/deltas" - # create deltas file if it does not already exist - if [ ! -f "$deltas" ]; then - msg2 "$(gettext "Creating 'deltas' db entry...")" - echo -e "%DELTAS%" >>$deltas + if [[ ! -f $deltas ]]; then + echo -e "%DELTAS%" >$deltas fi # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$deltafile" | awk '{print $NF}')" + md5sum="$(openssl dgst -md5 "$deltafile")" + md5sum="${md5sum##* }" csize=$(stat -L -c %s "$deltafile") oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') if grep -q "$oldfile.*$newfile" $deltas; then - warning "$(gettext "An entry for '%s' already existed")" "$deltafile" sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup - msg2 "$(gettext "Removing existing entry '%s'...")" "$deltafile" fi + msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas return 0 @@ -161,11 +170,11 @@ db_remove_delta() pkgname="$(get_delta_pkgname $deltafile)" pkgentry=$(find_pkgentry $pkgname) - if [ -z "$pkgentry" ]; then + if [[ -z $pkgentry ]]; then return 1 fi deltas="$pkgentry/deltas" - if [ ! -f "$deltas" ]; then + if [[ ! -f $deltas ]]; then return 1 fi if grep -q "$filename" $deltas; then @@ -183,7 +192,7 @@ db_write_entry() { # blank out all variables local pkgfile="$1" - local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager force \ + local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ _groups _licenses _replaces _depends _conflicts _provides _optdepends local OLDIFS="$IFS" @@ -213,19 +222,27 @@ db_write_entry() IFS=$OLDIFS # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$pkgfile" | awk '{print $NF}')" + md5sum="$(openssl dgst -md5 "$pkgfile")" + md5sum="${md5sum##* }" csize=$(stat -L -c %s "$pkgfile") # ensure $pkgname and $pkgver variables were found - if [ -z "$pkgname" -o -z "$pkgver" ]; then + if [[ -z $pkgname || -z $pkgver ]]; then error "$(gettext "Invalid package file '%s'.")" "$pkgfile" return 1 fi - cd "$tmpdir" - - if [ -d "$pkgname-$pkgver" ]; then + pushd "$tmpdir" >/dev/null + if [[ -d $pkgname-$pkgver ]]; then warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" + else + if (( DELTA )); then + pkgentry=$(find_pkgentry $pkgname) + if [[ -n $pkgentry ]]; then + local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) + local oldfile="$(dirname $1)/$oldfilename" + fi + fi fi # remove an existing entry if it exists, ignore failures @@ -233,36 +250,35 @@ db_write_entry() # create package directory mkdir "$pkgname-$pkgver" - cd "$pkgname-$pkgver" + pushd "$pkgname-$pkgver" >/dev/null # restore an eventual deltas file - [ -f "../$pkgname.deltas" ] && mv "../$pkgname.deltas" deltas + [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas # create desc entry - msg2 "$(gettext "Creating 'desc' db entry...")" + msg2 "$(gettext "Creating '%s' db entry...")" 'desc' echo -e "%FILENAME%\n$(basename "$1")\n" >>desc echo -e "%NAME%\n$pkgname\n" >>desc - [ -n "$pkgbase" ] && echo -e "%BASE%\n$pkgbase\n" >>desc + [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc echo -e "%VERSION%\n$pkgver\n" >>desc - [ -n "$pkgdesc" ] && echo -e "%DESC%\n$pkgdesc\n" >>desc + [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc write_list_entry "GROUPS" "$_groups" "desc" - [ -n "$csize" ] && echo -e "%CSIZE%\n$csize\n" >>desc - [ -n "$size" ] && echo -e "%ISIZE%\n$size\n" >>desc + [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc + [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc # compute checksums msg2 "$(gettext "Computing md5 checksums...")" echo -e "%MD5SUM%\n$md5sum\n" >>desc - [ -n "$url" ] && echo -e "%URL%\n$url\n" >>desc + [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc write_list_entry "LICENSE" "$_licenses" "desc" - [ -n "$arch" ] && echo -e "%ARCH%\n$arch\n" >>desc - [ -n "$builddate" ] && echo -e "%BUILDDATE%\n$builddate\n" >>desc - [ -n "$packager" ] && echo -e "%PACKAGER%\n$packager\n" >>desc + [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc + [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc + [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc write_list_entry "REPLACES" "$_replaces" "desc" - [ -n "$force" ] && echo -e "%FORCE%\n" >>desc # create depends entry - msg2 "$(gettext "Creating 'depends' db entry...")" + msg2 "$(gettext "Creating '%s' db entry...")" 'depends' # create the file even if it will remain empty touch "depends" write_list_entry "DEPENDS" "$_depends" "depends" @@ -270,9 +286,48 @@ db_write_entry() write_list_entry "PROVIDES" "$_provides" "depends" write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - cd "$startdir" + popd >/dev/null + popd >/dev/null - return 0 + # create files file if wanted + if (( WITHFILES )); then + msg2 "$(gettext "Creating '%s' db entry...")" 'files' + local files_path="$tmpdir/$pkgname-$pkgver/files" + echo "%FILES%" >$files_path + bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path + fi + + # create a delta file + if (( DELTA )); then + if [[ -n $oldfilename ]]; then + if [[ -f $oldfile ]]; then + delta=$(pkgdelta -q $oldfile $1) + if [[ -f $delta ]]; then + db_write_delta $delta + fi + else + warning "$(gettext "Old package file not found: %s")" "$oldfilename" + fi + fi + fi + + # Extracts licenses to a common license dir + msg "Extracting license" + if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + # Change dir to licenses, and extract them stripping the first part of the path + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ + --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + warning "This package doesn't contain a license dir" + fi + fi + + return 0 } # end db_write_entry # remove existing entries from the DB @@ -281,9 +336,9 @@ db_remove_entry() { local pkgname=$1 local notfound=1 local pkgentry=$(find_pkgentry $pkgname) - while [ -n "$pkgentry" ]; do + while [[ -n $pkgentry ]]; do notfound=0 - if [ -f "$pkgentry/deltas" ]; then + if [[ -f $pkgentry/deltas ]]; then mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" fi msg2 "$(gettext "Removing existing entry '%s'...")" \ @@ -291,6 +346,12 @@ db_remove_entry() { rm -rf $pkgentry pkgentry=$(find_pkgentry $pkgname) done + + msg "Removing license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + return $notfound } # end db_remove_entry @@ -301,16 +362,16 @@ check_repo_db() CLEAN_LOCK=1 else error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [ -f "$LOCKFILE" ] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" + [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" exit 1 fi - if [ -f "$REPO_DB_FILE" ]; then + if [[ -f $REPO_DB_FILE ]]; then # there are two situations we can have here- a DB with some entries, # or a DB with no contents at all. if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then # check empty case - if [ -n "$(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null)" ]; then + if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" exit 1 fi @@ -337,15 +398,15 @@ check_repo_db() add() { - if [ ! -f "$1" ]; then + if [[ ! -f $1 ]]; then error "$(gettext "File '%s' not found.")" "$1" return 1 fi - if [ "${1##*.}" == "delta" ]; then + if [[ ${1##*.} == "delta" ]]; then deltafile=$1 msg "$(gettext "Adding delta '%s'")" "$deltafile" - if [ ! "$(type -p xdelta3)" ]; then + if ! type xdelta3 &>/dev/null; then error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" exit 1 fi @@ -357,7 +418,7 @@ add() fi pkgfile=$1 - if ! bsdtar -tqf "$pkgfile" .PKGINFO 2>&1 >/dev/null; then + if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" return 1 fi @@ -369,7 +430,7 @@ add() remove() { - if [ "${1##*.}" == "delta" ]; then + if [[ ${1##*.} == "delta" ]]; then deltafile=$1 msg "$(gettext "Searching for delta '%s'...")" "$deltafile" if db_remove_delta "$deltafile"; then @@ -402,9 +463,8 @@ trap_exit() clean_up() { local exit_code=$? - cd "$startdir" - [ -d "$tmpdir" ] && rm -rf "$tmpdir" - [ $CLEAN_LOCK -eq 1 -a -f "$LOCKFILE" ] && rm -f "$LOCKFILE" + [[ -d $tmpdir ]] && rm -rf "$tmpdir" + (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" exit $exit_code } @@ -412,7 +472,7 @@ clean_up() { # PROGRAM START # determine whether we have gettext; make it a no-op if we do not -if [ ! $(type -t gettext) ]; then +if ! type gettext &>/dev/null; then gettext() { echo "$@" } @@ -423,15 +483,9 @@ case "$1" in -V|--version) version; exit 0;; esac -# check for correct number of args -if [ $# -lt 2 ]; then - usage - exit 1 -fi - # figure out what program we are cmd="$(basename $0)" -if [ "$cmd" != "repo-add" -a "$cmd" != "repo-remove" ]; then +if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then error "$(gettext "Invalid command name '%s' specified.")" "$cmd" exit 1 fi @@ -450,14 +504,10 @@ success=0 for arg in "$@"; do case "$arg" in -q|--quiet) QUIET=1;; - - -f|--force) - warning "$(gettext "the -f and --force options are no longer recognized")" - msg2 "$(gettext "use options=(force) in the PKGBUILD instead")" - ;; - + -d|--delta) DELTA=1;; + -f|--files) WITHFILES=1;; *) - if [ -z "$REPO_DB_FILE" ]; then + if [[ -z $REPO_DB_FILE ]]; then REPO_DB_FILE="$arg" LOCKFILE="$REPO_DB_FILE.lck" check_repo_db @@ -472,7 +522,7 @@ for arg in "$@"; do done # if at least one operation was a success, re-zip database -if [ $success -eq 1 ]; then +if (( success )); then msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" case "$REPO_DB_FILE" in @@ -485,20 +535,26 @@ if [ $success -eq 1 ]; then filename=$(basename "$REPO_DB_FILE") - cd "$tmpdir" - if [ -n "$(ls)" ]; then + pushd "$tmpdir" >/dev/null + if [[ -n $(ls) ]]; then bsdtar -c${TAR_OPT}f "$filename" * else # we have no packages remaining? zip up some emptyness warning "$(gettext "No packages remain, creating empty database.")" bsdtar -c${TAR_OPT}f "$filename" -T /dev/null fi - cd "$startdir" - - [ -f "$REPO_DB_FILE" ] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" - [ -f "$tmpdir/$filename" ] && mv "$tmpdir/$filename" "$REPO_DB_FILE" + popd >/dev/null + + [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" + [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" + dblink="${REPO_DB_FILE%.tar.*}" + target=${REPO_DB_FILE##*/} + ln -sf "$target" "$dblink" 2>/dev/null || \ + ln -f "$target" "$dblink" 2>/dev/null || \ + cp "$REPO_DB_FILE" "$dblink" else msg "$(gettext "No packages modified, nothing to do.")" + exit 1 fi exit 0 diff --git a/repo-update b/repo-update index a44ae87..e758cc8 100755 --- a/repo-update +++ b/repo-update @@ -10,23 +10,27 @@ for repo in ${ARCHREPOS[@]}; do for arch in ${ARCHARCHES[@]} 'any'; do msg2 "${repo} ${arch}" # makes a file containing rsync output for filter.py + plain "Checking packages..." ${rsync_list_command} \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ > ${rsout_file} + ${repodir}/staging/${repo}/ > ${rsout_file} 2&>/dev/null # reads blacklist and rsout_file and makes an rsync exclude-from # list + plain "Excluding nonfree..." filter.py -r ${rsync_blacklist} -k ${blacklist} \ -f ${rsout_file} # list files in ${repodir}/${repo} and write their names on # rsync_not_needed for using as an rsync exclude-from + plain "Excluding our packages" find ${repodir}/${repo} -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed} '%f\n' # Actual rsync command + plain "Syncing..." ${rsync_update_command} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ + ${repodir}/staging/${repo}/ 2&>/dev/null done for arch in ${ARCHARCHES[@]}; do msg2 "Making pending list for $repo $arch" -- cgit v1.2.3 From dead95b06c2a547f3cfae6c5acc21859cdd30d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 10 Jun 2011 16:33:28 -0500 Subject: * Use python for identify nonfree in db-check-nonfree --- db-check-nonfree | 11 +++-------- list_nonfree_in_db.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) create mode 100755 list_nonfree_in_db.py diff --git a/db-check-nonfree b/db-check-nonfree index 83efb14..d8f34f0 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -23,15 +23,10 @@ for repo in ${ARCHREPOS[@]}; do if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue fi - unset dbpkgs unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in ${dbpkgs[@]}; do - if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs[${#cleanpkgs[*]}]=${pkgname} - fi - done + cmd_="$(dirname $0)/list_nonfree_in_db.py -k ${BLACKLIST_FILE} \ + -b ${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" + cleanpkgs=($(${cmd_})) if [ ${#cleanpkgs[@]} -ge 1 ]; then arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} fi diff --git a/list_nonfree_in_db.py b/list_nonfree_in_db.py new file mode 100755 index 0000000..598a2e7 --- /dev/null +++ b/list_nonfree_in_db.py @@ -0,0 +1,28 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +from repm.filter import * +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="nonfree_in_db", + description="Cleans nonfree files on repo",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean", + required=True,) + + args=parser.parse_args() + + if not (args.blacklist_file and args.database): + parser.print_help() + exit(1) + + blacklist=listado(args.blacklist_file) + pkgs=get_pkginfo_from_db(args.database) + + print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) -- cgit v1.2.3 From cd93439c7eb71c380ab0355306a760b7bb5f77ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 13 Jun 2011 06:00:17 -0500 Subject: * repo-update exclude also from ${repodir}/old/packages --- repo-update | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repo-update b/repo-update index e758cc8..4631627 100755 --- a/repo-update +++ b/repo-update @@ -24,11 +24,14 @@ for repo in ${ARCHREPOS[@]}; do plain "Excluding our packages" find ${repodir}/${repo} -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed} '%f\n' + find ${repodir}/old/packages -name "*${PKGEXT}" \ + -fprintf ${rsync_not_needed}2 '%f\n' # Actual rsync command plain "Syncing..." ${rsync_update_command} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} \ + --exclude-from=${rsync_not_needed}2 \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ ${repodir}/staging/${repo}/ 2&>/dev/null done -- cgit v1.2.3 From c991efcdf97748614c99dc23bf8f1ecbab99a247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 13 Jun 2011 08:37:38 -0500 Subject: fix issue 53 --- .gitignore | 5 +++-- yf-update | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100755 yf-update diff --git a/.gitignore b/.gitignore index 45688ab..ee8bbb8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ local_config /config.local test/packages/*/*.pkg.tar.?z -#*# -.#* \ No newline at end of file +\#*# +.#* +yftime \ No newline at end of file diff --git a/yf-update b/yf-update new file mode 100755 index 0000000..c65fc09 --- /dev/null +++ b/yf-update @@ -0,0 +1,17 @@ +#!/bin/bash +source $(dirname $0)/local_config +source $(dirname $0)/config + +blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) +last_bl_mtime=$(cat $(dirname $0)/yftime) + +if [ $blacklist_mtime -gt $last_bl_mtime ]; then + pushd $(dirname $0)/yf + makepkg + rsync -L ./*.${PKGEXT} ${STAGING}/libre + popd + echo ${blacklist_mtime} > $(dirname $0)/yftime +fi + + + -- cgit v1.2.3 From 436fcd89bf7721f54a359842b4741bc67825411c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 13 Jun 2011 08:46:25 -0500 Subject: Added your-freeedom to db-scripts --- yf/PKGBUILD | 28 ++++++++++++++++++++++++++++ yf/your-freedom.install | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 yf/PKGBUILD create mode 100644 yf/your-freedom.install diff --git a/yf/PKGBUILD b/yf/PKGBUILD new file mode 100644 index 0000000..cc6f07b --- /dev/null +++ b/yf/PKGBUILD @@ -0,0 +1,28 @@ +# Maintainer: Parabola Project +pkgname=your-freedom +pkgver=$(LC_ALL=C date -u +%Y%m%d) +pkgrel=1 +pkgdesc="This package conflicts with every unfree package known to date." +arch=('any') +url="https://parabolagnulinux.org" +license=('GPL') +groups=('base') +install=${pkgname}.install +source=() +md5sums=() +noextract=() + +build() { + cd ${srcdir} + source ~/repm/local_config + install -d ${pkgdir}/usr/share/doc/${pkgname} + install -m644 $blacklist $whitelist ${pkgdir}/usr/share/doc/${pkgname}/ +} + +package() { + conflicts=($(cut -d: -f1,2 ${pkgdir}/usr/share/doc/${pkgname}/blacklist.txt | \ + sed "s/:$//" | \ + grep -v ":" | \ + sort -u + )) +} diff --git a/yf/your-freedom.install b/yf/your-freedom.install new file mode 100644 index 0000000..49ae045 --- /dev/null +++ b/yf/your-freedom.install @@ -0,0 +1,32 @@ + +pre_install() { + cat < Date: Mon, 13 Jun 2011 08:12:18 -0700 Subject: * Fixed issue131 * Fixed issue53 --- .gitignore | 4 +++- repo-update | 10 ++++++++-- yf-update | 11 ++++++----- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index ee8bbb8..dd17455 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ local_config test/packages/*/*.pkg.tar.?z \#*# .#* -yftime \ No newline at end of file +yftime +src* +pkg* \ No newline at end of file diff --git a/repo-update b/repo-update index 4631627..12c6d0c 100755 --- a/repo-update +++ b/repo-update @@ -5,6 +5,14 @@ source $(dirname $0)/config source $(dirname $0)/local_config source $(dirname $0)/libremessages +msg "Updating your-freedom" +$(dirname $0)/yf-update + +msg "Global exclude-list" +# To not downgrade repos by accident +find ${repodir}/old/packages -name "*${PKGEXT}" \ + -fprintf ${rsync_not_needed}2 '%f\n' + for repo in ${ARCHREPOS[@]}; do msg "Syncing ${repo}" for arch in ${ARCHARCHES[@]} 'any'; do @@ -24,8 +32,6 @@ for repo in ${ARCHREPOS[@]}; do plain "Excluding our packages" find ${repodir}/${repo} -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed} '%f\n' - find ${repodir}/old/packages -name "*${PKGEXT}" \ - -fprintf ${rsync_not_needed}2 '%f\n' # Actual rsync command plain "Syncing..." ${rsync_update_command} \ diff --git a/yf-update b/yf-update index c65fc09..9c2131e 100755 --- a/yf-update +++ b/yf-update @@ -1,17 +1,18 @@ #!/bin/bash source $(dirname $0)/local_config source $(dirname $0)/config +source $(dirname $0)/libremessages blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) last_bl_mtime=$(cat $(dirname $0)/yftime) if [ $blacklist_mtime -gt $last_bl_mtime ]; then pushd $(dirname $0)/yf - makepkg - rsync -L ./*.${PKGEXT} ${STAGING}/libre + makepkg -f + find . -name "*${PKGEXT}" -exec mv {} ${STAGING}/libre \; popd echo ${blacklist_mtime} > $(dirname $0)/yftime + msg2 "built and staged" +else + msg2 "nothing to do" fi - - - -- cgit v1.2.3 From 7e8840e0a76d28c27ed7efc041106d338ca0c607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 13 Jun 2011 08:17:13 -0700 Subject: repos are working fine like this --- db-check-nonfree | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/db-check-nonfree b/db-check-nonfree index d8f34f0..83efb14 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -23,10 +23,15 @@ for repo in ${ARCHREPOS[@]}; do if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue fi + unset dbpkgs unset cleanpkgs - cmd_="$(dirname $0)/list_nonfree_in_db.py -k ${BLACKLIST_FILE} \ - -b ${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" - cleanpkgs=($(${cmd_})) + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in ${dbpkgs[@]}; do + if in_array ${pkgname} ${nonfree[@]}; then + cleanpkgs[${#cleanpkgs[*]}]=${pkgname} + fi + done if [ ${#cleanpkgs[@]} -ge 1 ]; then arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} fi -- cgit v1.2.3 From 8a874d242c8d86059261c098dfec1b0b1ac9e444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 13 Jun 2011 15:33:16 -0300 Subject: mkrepo is script to create [repos] --- mkrepo | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 mkrepo diff --git a/mkrepo b/mkrepo new file mode 100755 index 0000000..5f704cc --- /dev/null +++ b/mkrepo @@ -0,0 +1,24 @@ +#!/bin/bash +# Author: Nicolás Reynolds +# License: GPLv3+ +# Description: A script to quickly create new [repos] + +source $(dirname $0)/config +source $(dirname $0)/local_config + +# TODO it would be simpler to expand arrays to {element1,element2,etc} +for repo in $@; do + + echo ":: Creating [$repo]" + mkdir -pv ${repodir}/{staging/,}${repo} + + for arch in ${ARCHES[@]}; do + mkdir -pv ${repodir}/${repo}/os/${arch} + done + +done + +echo ":: All done. Add the repo to the parabolaweb admin page" +echo " and the get_repos script on the same server." + +exit $? -- cgit v1.2.3 From 9bcf313647a3dd4ae44016894b1fe599d1f90c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 19 Jun 2011 11:01:07 -0700 Subject: Config file updated, use old repo-update --- config | 4 ++-- repo-update | 17 ++--------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/config b/config index effdde4..bc2494f 100644 --- a/config +++ b/config @@ -4,7 +4,7 @@ ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') -PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') +PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar' 'elementary' '~fauno' '~xihh') PKGPOOL='pool/packages' SRCPOOL='sources/packages' @@ -33,4 +33,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="$HOME/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" \ No newline at end of file +BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" diff --git a/repo-update b/repo-update index 12c6d0c..a44ae87 100755 --- a/repo-update +++ b/repo-update @@ -5,41 +5,28 @@ source $(dirname $0)/config source $(dirname $0)/local_config source $(dirname $0)/libremessages -msg "Updating your-freedom" -$(dirname $0)/yf-update - -msg "Global exclude-list" -# To not downgrade repos by accident -find ${repodir}/old/packages -name "*${PKGEXT}" \ - -fprintf ${rsync_not_needed}2 '%f\n' - for repo in ${ARCHREPOS[@]}; do msg "Syncing ${repo}" for arch in ${ARCHARCHES[@]} 'any'; do msg2 "${repo} ${arch}" # makes a file containing rsync output for filter.py - plain "Checking packages..." ${rsync_list_command} \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ > ${rsout_file} 2&>/dev/null + ${repodir}/staging/${repo}/ > ${rsout_file} # reads blacklist and rsout_file and makes an rsync exclude-from # list - plain "Excluding nonfree..." filter.py -r ${rsync_blacklist} -k ${blacklist} \ -f ${rsout_file} # list files in ${repodir}/${repo} and write their names on # rsync_not_needed for using as an rsync exclude-from - plain "Excluding our packages" find ${repodir}/${repo} -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed} '%f\n' # Actual rsync command - plain "Syncing..." ${rsync_update_command} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} \ - --exclude-from=${rsync_not_needed}2 \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ 2&>/dev/null + ${repodir}/staging/${repo}/ done for arch in ${ARCHARCHES[@]}; do msg2 "Making pending list for $repo $arch" -- cgit v1.2.3 From 91210f8ae18f5651ace2273cfa0c8ad7e4dbcbdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 16 Jul 2011 03:35:37 -0300 Subject: Syncs ABSLibre tarballs between ABSLibre host and repo server --- cron-jobs/update-abs-tarballs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 cron-jobs/update-abs-tarballs diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs new file mode 100644 index 0000000..9990f20 --- /dev/null +++ b/cron-jobs/update-abs-tarballs @@ -0,0 +1,7 @@ +#!/bin/bash + +. "$(dirname $0)/../config" + +rsync -av parabolagnulinux.org::abstar/ ${FTP_BASE}/ + +exit $? -- cgit v1.2.3 From d438ff5d25372ba32e42f89c808f5b33a25042f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 3 Aug 2011 16:13:56 -0300 Subject: Changed SVN to ABSLibre --- db-remove | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/db-remove b/db-remove index b44eb33..ccfeb36 100755 --- a/db-remove +++ b/db-remove @@ -12,9 +12,6 @@ pkgbase="$1" repo="$2" arch="$3" -ftppath="$FTP_BASE/$repo/os" -svnrepo="$repo-$arch" - if ! check_repo_permission $repo; then die "You don't have permission to remove packages from ${repo}" fi @@ -30,14 +27,11 @@ for tarch in ${tarches[@]}; do done msg "Removing $pkgbase from [$repo]..." -/usr/bin/svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null -if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then - pkgnames=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) - /usr/bin/svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" - /usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "$(basename $0): $pkgbase removed by $(id -un)" +if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then + pkgnames=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) else - warning "$pkgbase not found in $svnrepo" + warning "$pkgbase not found in $repo" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" pkgnames=($pkgbase) -- cgit v1.2.3 From 69d7089ab3bec9953877f20b008e7b7dd99a3459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 3 Aug 2011 12:16:56 -0700 Subject: Added personal repos --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index bc2494f..eda45eb 100644 --- a/config +++ b/config @@ -4,7 +4,7 @@ ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') -PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar' 'elementary' '~fauno' '~xihh') +PKGREPOS=(${ARCHREPOS[@]} 'libre-testing' 'libre' 'security' 'social' 'sugar' 'elementary' '~fauno' '~xihh' '~smv') PKGPOOL='pool/packages' SRCPOOL='sources/packages' -- cgit v1.2.3 From 0e4184bc0c06eccd4484c9028c0369554d482aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 5 Aug 2011 19:41:12 -0300 Subject: Added script for creating repos --- create-repo | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 create-repo diff --git a/create-repo b/create-repo new file mode 100755 index 0000000..58842c3 --- /dev/null +++ b/create-repo @@ -0,0 +1,24 @@ +#!/bin/bash +# Creates repository structure + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -eq 0 ]; then + msg "Usage: $0 repo1 [repo2 ... repoX]" + exit 1 +fi + +msg "Creating repos..." +for _repo in $@; do + msg2 "Creating [${_repo}]" + mkdir -p "${FTP_BASE}/staging/${_repo}" || \ + error "Failed creating staging dir" + + for _arch in ${ARCHES[@]}; do + mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ + error "Failed creating ${_arch} dir" + done +done + +msg "Don't forget to add them to the PKGREPOS array on $(dirname $0)/config" -- cgit v1.2.3 From cd45a4e2e000312d86b072eef9349b9494cd6d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 5 Aug 2011 19:49:23 -0300 Subject: Repo organization --- config | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config b/config index effdde4..8fd37bf 100644 --- a/config +++ b/config @@ -3,8 +3,15 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +# Repos from Arch ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') -PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') +# Official Parabola repos +OURREPOS=('libre' 'libre-testing') +# User repos +USERREPOS=('~fauno' '~smv' '~xihh') +# Community project repos +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'social') +PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' @@ -33,4 +40,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="$HOME/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" \ No newline at end of file +BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" -- cgit v1.2.3 From c1858e5d04ebe2b8c2c45ff71dc8bcfb05b49dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 9 Aug 2011 21:46:24 -0300 Subject: update-abs-tarballs +x --- cron-jobs/update-abs-tarballs | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cron-jobs/update-abs-tarballs diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs old mode 100644 new mode 100755 -- cgit v1.2.3 From 67c5e5b37efa306c92644922fad65adbb127a4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 9 Aug 2011 21:49:27 -0300 Subject: Ignore staging --- cron-jobs/update-abs-tarballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs index 9990f20..824ac34 100755 --- a/cron-jobs/update-abs-tarballs +++ b/cron-jobs/update-abs-tarballs @@ -2,6 +2,6 @@ . "$(dirname $0)/../config" -rsync -av parabolagnulinux.org::abstar/ ${FTP_BASE}/ +rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ exit $? -- cgit v1.2.3 From 986fbe51101674c290df8f5c9d0dc3f507c172e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 11 Aug 2011 20:45:29 -0300 Subject: Script to solve issue165 --- cron-jobs/repo-sanity-check | 46 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 cron-jobs/repo-sanity-check diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check new file mode 100644 index 0000000..a6249b5 --- /dev/null +++ b/cron-jobs/repo-sanity-check @@ -0,0 +1,46 @@ +#!/bin/bash +# Solves issue165 + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +# Traverse all repos +for _repo in ${REPOS[@]}; do + +# Find all pkgnames on this repo's abs + on_abs=($( + find ${SVNREPO}/${_repo} -name PKGBUILD | \ + while read pkgbuild; do + source ${pkgbuild} >/dev/null 2>&1 +# cleanup to save memory + unset build package source md5sums pkgdesc pkgver pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + +# also cleanup package functions + for _pkg in ${pkgname[@]}; do + unset package_${pkg} >/dev/null 2>&1 + done + +# this fills the on_abs array + echo ${pkgname[@]} + done + )) + +# Find all pkgnames on repos + on_repo=($( + find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ + sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + )) + +# Compares them, whatever is on repos but not on abs should be removed + remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort) \ + <(echo ${on_repo[@]} | tr ' ' "\n" | sort))) + +# Remove them from databases, ftpdir-cleanup will take care of the rest + find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -print0 | \ + xargs -0 repo-remove {} ${remove[@]} + +done + +exit $? -- cgit v1.2.3 From d1de884ad56e0756a80d5c00bb723e7f2a56491b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 11 Aug 2011 21:37:21 -0300 Subject: Changed xargs for find -exec --- cron-jobs/repo-sanity-check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index a6249b5..52ca33b 100644 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -38,8 +38,8 @@ for _repo in ${REPOS[@]}; do <(echo ${on_repo[@]} | tr ' ' "\n" | sort))) # Remove them from databases, ftpdir-cleanup will take care of the rest - find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -print0 | \ - xargs -0 repo-remove {} ${remove[@]} + find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ + repo-remove {} ${remove[@]} \; done -- cgit v1.2.3 From 1d7db341d4f27f8ebd7167a8630ea8589e40c464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 11 Aug 2011 21:39:05 -0300 Subject: Forgot +x --- cron-jobs/repo-sanity-check | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cron-jobs/repo-sanity-check diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check old mode 100644 new mode 100755 -- cgit v1.2.3 From d2f65c340f890050c99371e203288afea03d125b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 11 Aug 2011 22:42:01 -0300 Subject: Sort unique pkgnames --- cron-jobs/repo-sanity-check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 52ca33b..690a9f8 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -34,8 +34,8 @@ for _repo in ${REPOS[@]}; do )) # Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort) \ - <(echo ${on_repo[@]} | tr ' ' "\n" | sort))) + remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ + <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) # Remove them from databases, ftpdir-cleanup will take care of the rest find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ -- cgit v1.2.3 From 7d6595c750903114740c7ab726cd89cb73d6c3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 12 Aug 2011 02:37:50 -0300 Subject: Fixed unexistent REPOS array and skip removal when the ABS tree is empty. --- cron-jobs/repo-sanity-check | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 690a9f8..92b26ae 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -5,7 +5,7 @@ . "$(dirname $0)/../config" # Traverse all repos -for _repo in ${REPOS[@]}; do +for _repo in ${PKGREPOS[@]}; do # Find all pkgnames on this repo's abs on_abs=($( @@ -27,6 +27,12 @@ for _repo in ${REPOS[@]}; do done )) +# quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[${_repo}]'s ABS tree is empty, skipping" + break + fi + # Find all pkgnames on repos on_repo=($( find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ -- cgit v1.2.3 From a032c1fad3e2d57a7a1b7cef65d90ea06a766784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Aug 2011 12:42:42 -0300 Subject: Cleaner sourceballs2 --- cron-jobs/sourceballs2 | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index b29b396..a43d71e 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -33,24 +33,28 @@ for repo in ${PKGREPOS[@]}; do continue } - unset pkgbase pkgname +# Unset the previous data + unset pkgbase pkgname pkgver pkgrel source PKGBUILD + + unset build package url pkgdesc source md5sums depends makedepends \ + optdepends license arch options check mksource + + for _pkg in ${pkgname[@]}; do + unset package_${_pkg} >/dev/null 2>&1 + done + pkgbase=${pkgbase:-$pkgname} + srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" - echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs" # Skip already sourceballed - [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ - continue + [ -e "${SRCPKGDEST}/${srcfile}" ] && continue - msg2 "$pkgbase-$pkgver-$pkgrel..." - makepkg --allsource --ignorearch -c makepkg --allsource --ignorearch -c >/dev/null 2>&1 - [[ $? -ne 0 ]] && { - warning "Failed." - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" - } + [ $? -ne 0 ] && failedpkgs+=("${srcfile}") done popd >/dev/null @@ -89,10 +93,5 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -msg "Failed" -for _fail in ${failedpkgs[@]}; do - msg2 "$_fail" -done - script_unlock -- cgit v1.2.3 From f23d7103fec259ebe2ba796ae7f351dfbda5b705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Aug 2011 12:51:32 -0300 Subject: Added useful messages --- cron-jobs/repo-sanity-check | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 92b26ae..1ba90a6 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -6,6 +6,7 @@ # Traverse all repos for _repo in ${PKGREPOS[@]}; do + msg "Cleaning up [${_repo}]" # Find all pkgnames on this repo's abs on_abs=($( @@ -45,7 +46,10 @@ for _repo in ${PKGREPOS[@]}; do # Remove them from databases, ftpdir-cleanup will take care of the rest find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ - repo-remove {} ${remove[@]} \; + repo-remove {} ${remove[@]} >/dev/null 2>&1 \; + + msg2 "Removed the following packages:" + plain "$(echo ${remove[@]} | tr ' ' "\n")" done -- cgit v1.2.3 From eac9e4d43cc33bf6671fdbd7f04e5bab387259d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Aug 2011 13:15:35 -0300 Subject: Failedpkgs array wasn't working --- cron-jobs/sourceballs2 | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index a43d71e..5e228fc 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -22,10 +22,10 @@ find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null for repo in ${PKGREPOS[@]}; do - failedpkgs=() + msg "Sourceballing [${repo}]" pushd $repo >/dev/null - find . -maxdepth 1 -type d | while read pkg; do + find -maxdepth 1 -type d | while read pkg; do pushd "${SVNREPO}/$repo/$pkg" >/dev/null [[ ! -e PKGBUILD ]] && { @@ -54,18 +54,12 @@ for repo in ${PKGREPOS[@]}; do makepkg --allsource --ignorearch -c >/dev/null 2>&1 - [ $? -ne 0 ] && failedpkgs+=("${srcfile}") + [ $? -ne 0 ] && plain ${srcfile} - done + done # end find pkgs popd >/dev/null - if [ ${#failedpkgs[@]} -ge 1 ]; then - msg "Failed to create source packages for [${repo}]..." - for failed_pkg in ${failedpkgs[@]}; do - msg2 "${failed_pkg}" - done - fi -done +done # end repos # Cleanup old source packages cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -- cgit v1.2.3 From 119de3a00044c0f854c21cd3620666414b1c6a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 17 Aug 2011 18:14:44 -0300 Subject: Script to restore packages from the cleanup destdir --- repo-restore-to-normal | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 repo-restore-to-normal diff --git a/repo-restore-to-normal b/repo-restore-to-normal new file mode 100755 index 0000000..e46cfc9 --- /dev/null +++ b/repo-restore-to-normal @@ -0,0 +1,54 @@ +#!/bin/bash +# Solves issue165 + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +# Find all pkgnames on old with pkgvers + on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \ + sed "s/^\(.\+-[^-]\+\)-[^-]\+-[^-]\+$/\1/")) + +# Traverse all repos +for _repo in ${PKGREPOS[@]}; do + msg "Restoring [${_repo}]" + +# Find all pkgnames on this repo's abs + on_abs=($( + find ${SVNREPO}/${_repo} -name PKGBUILD | \ + while read pkgbuild; do + source ${pkgbuild} >/dev/null 2>&1 +# cleanup to save memory + unset build package source md5sums pkgdesc pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + +# also cleanup package functions + for _pkg in ${pkgname[@]}; do + unset package_${pkg} >/dev/null 2>&1 + done + +# this fills the on_abs array + echo ${pkgname[@]}-${pkgver} + done + )) + +# quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[${_repo}]'s ABS tree is empty, skipping" + break + fi + +# Compares them, whatever is on abs should be restored + restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ + <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) + + msg2 "Restoring the following packages:" + plain "$(echo ${restore[@]} | tr ' ' "\n")" + + for _pkg in ${restore[@]}; do + find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec mv -v '{}' ${STAGING}/${_repo} + done + +done + +exit $? -- cgit v1.2.3 From 4d760a1e9b92168a07aa8c7a3a9de481ebcac2a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 9 Sep 2011 12:33:51 -0700 Subject: Added [~mtjm], changed blacklist to git --- config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config b/config index 8fd37bf..0e8dc2e 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm') # Community project repos PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'social') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) @@ -40,4 +40,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="$HOME/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" -- cgit v1.2.3 From 178c720a4c2fd8adf20e6109784c39cc27dd6b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 9 Sep 2011 12:35:46 -0700 Subject: Inform found packages --- db-check-nonfree | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db-check-nonfree b/db-check-nonfree index 83efb14..ab6491d 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -29,10 +29,11 @@ for repo in ${ARCHREPOS[@]}; do dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) for pkgname in ${dbpkgs[@]}; do if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs[${#cleanpkgs[*]}]=${pkgname} + cleanpkgs+=(${pkgname}) fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then + msg2 "Unfree: ${cleanpkgs[@]}" arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} fi done -- cgit v1.2.3 From e9d0581b173853e647b36caa170b7c4bbee43643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 9 Sep 2011 12:39:56 -0700 Subject: Several fixes --- repo-restore-to-normal | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/repo-restore-to-normal b/repo-restore-to-normal index e46cfc9..9463731 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -4,9 +4,12 @@ . "$(dirname $0)/db-functions" . "$(dirname $0)/config" -# Find all pkgnames on old with pkgvers - on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \ - sed "s/^\(.\+-[^-]\+\)-[^-]\+-[^-]\+$/\1/")) +CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore +PKGREPOS=(community) + +# Find all pkgnames on old with pkgver-pkgrels +#on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \ +# sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/")) # Traverse all repos for _repo in ${PKGREPOS[@]}; do @@ -16,37 +19,38 @@ for _repo in ${PKGREPOS[@]}; do on_abs=($( find ${SVNREPO}/${_repo} -name PKGBUILD | \ while read pkgbuild; do + unset pkgname pkgver pkgrel source ${pkgbuild} >/dev/null 2>&1 # cleanup to save memory - unset build package source md5sums pkgdesc pkgrel epoch \ + unset build package source md5sums pkgdesc epoch \ url license arch depends makedepends optdepends options \ >/dev/null 2>&1 # also cleanup package functions for _pkg in ${pkgname[@]}; do unset package_${pkg} >/dev/null 2>&1 +# this fills the on_abs array + echo ${_pkg}-${pkgver}-${pkgrel} done -# this fills the on_abs array - echo ${pkgname[@]}-${pkgver} done )) # quit if abs is empty if [ ${#on_abs[*]} -eq 0 ]; then warning "[${_repo}]'s ABS tree is empty, skipping" - break + continue fi # Compares them, whatever is on abs should be restored - restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ - <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) +# restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ +# <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) msg2 "Restoring the following packages:" - plain "$(echo ${restore[@]} | tr ' ' "\n")" +# plain "$(echo ${restore[@]} | tr ' ' "\n")" - for _pkg in ${restore[@]}; do - find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec mv -v '{}' ${STAGING}/${_repo} + for _pkg in ${on_abs[@]}; do + find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \; done done -- cgit v1.2.3 From 756f1c3b8bb39c9932b04cea05adecb71553c982 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 14 Sep 2011 05:40:17 -0700 Subject: Move to python2 --- clean_repo.py | 4 ++-- config.py | 2 +- filter.py | 4 ++-- list_nonfree_in_db.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index cc8e811..076de60 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -1,6 +1,6 @@ -#! /usr/bin/python +#! /usr/bin/python2 #-*- encoding: utf-8 -*- -from repm.filter import * +from filter import * import argparse def mkpending(packages_iterable, pending_file, blacklisted_names, diff --git a/config.py b/config.py index 4e218a5..0ffc475 100755 --- a/config.py +++ b/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # -*- coding: utf-8 -*- try: from subprocess import check_output diff --git a/filter.py b/filter.py index 5d90bdd..70d5e29 100755 --- a/filter.py +++ b/filter.py @@ -1,7 +1,7 @@ -#! /usr/bin/python +#! /usr/bin/python2 #-*- encoding: utf-8 -*- from glob import glob -from repm.config import * +from config import * import tarfile def listado(filename, start=0, end=None): diff --git a/list_nonfree_in_db.py b/list_nonfree_in_db.py index 598a2e7..4e1b164 100755 --- a/list_nonfree_in_db.py +++ b/list_nonfree_in_db.py @@ -1,6 +1,6 @@ -#! /usr/bin/python +#! /usr/bin/python2 #-*- encoding: utf-8 -*- -from repm.filter import * +from filter import * import argparse if __name__ == "__main__": -- cgit v1.2.3 From 4e81322477747ad61524f1d18fcba58965872542 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 14 Sep 2011 05:40:47 -0700 Subject: Fixed sync paths * rsync files are splitted by repo and arch * compare available packages with pool, not repos (dead symlinks broke syncing) * receive from repo/os/arch instead of repo/ (it made packages on other arches missing) --- repo-update | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/repo-update b/repo-update index a44ae87..4f31116 100755 --- a/repo-update +++ b/repo-update @@ -6,28 +6,31 @@ source $(dirname $0)/local_config source $(dirname $0)/libremessages for repo in ${ARCHREPOS[@]}; do + msg "Syncing ${repo}" - for arch in ${ARCHARCHES[@]} 'any'; do - msg2 "${repo} ${arch}" - # makes a file containing rsync output for filter.py - ${rsync_list_command} \ - rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ > ${rsout_file} - # reads blacklist and rsout_file and makes an rsync exclude-from - # list - filter.py -r ${rsync_blacklist} -k ${blacklist} \ - -f ${rsout_file} - # list files in ${repodir}/${repo} and write their names on - # rsync_not_needed for using as an rsync exclude-from - find ${repodir}/${repo} -name "*${PKGEXT}" \ - -fprintf ${rsync_not_needed} '%f\n' - # Actual rsync command - ${rsync_update_command} \ - --exclude-from=${rsync_blacklist} \ - --exclude-from=${rsync_not_needed} \ - rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ - done + for arch in ${ARCHARCHES[@]}; do + msg2 "${repo} ${arch}" + # makes a file containing rsync output for filter.py + ${rsync_list_command} \ + rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/staging/${repo}/os/${arch}/ > ${rsout_file}-${repo}-${arch} || exit 1 + # reads blacklist and rsout_file and makes an rsync exclude-from + # list + filter.py -r ${rsync_blacklist} -k ${blacklist} \ + -f ${rsout_file}-${repo}-${arch} || exit 1 + # list files in ${repodir}/${repo} and write their names on + # rsync_not_needed for using as an rsync exclude-from + #find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ + # -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 + find ${repodir}/${PKGPOOL}/ -name "*${PKGEXT}" \ + -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 + # Actual rsync command + ${rsync_update_command} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed}-${repo}-${arch} \ + rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/staging/${repo}/ || exit 1 + done for arch in ${ARCHARCHES[@]}; do msg2 "Making pending list for $repo $arch" # if there is a db in repo (db is created on rsync) @@ -42,7 +45,7 @@ for repo in ${ARCHREPOS[@]}; do done # if some nonfree files got pass the filter this command delete them msg2 "Fallback cleaning repo" - $(dirname $0)/clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} + $(dirname $0)/clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} || exit 1 done msg "Removing leftover files..." -- cgit v1.2.3 From 9612e5d915faf63ea6d5a5ca5c3ff74cca8eb923 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Fri, 28 Oct 2011 12:15:09 -0300 Subject: Reflect server changes on config --- config | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config b/config index effdde4..4f4d81d 100644 --- a/config +++ b/config @@ -1,7 +1,7 @@ #!/bin/bash -FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +FTP_BASE="/home/repo/public" +ARCH_BASE="/home/repo/public" +SVNREPO="/home/repo/abslibre" ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') @@ -24,7 +24,7 @@ LOCK_DELAY=10 LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" -TMPDIR="$HOME/tmp" +TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) ARCHES=(${ARCHARCHES[@]} mips64el) DBEXT=".db.tar.gz" @@ -32,5 +32,5 @@ FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" -MAKEPKGCONF="$HOME/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" \ No newline at end of file +MAKEPKGCONF="/etc/makepkg.conf" +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" -- cgit v1.2.3 From 8bb0c5aa9d00dfe16101329d62691d3f60cf134b Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Fri, 28 Oct 2011 12:18:29 -0300 Subject: Sync package databases first * Get all available packages * Remove unfree from the sync list * Sync everything whitelisted --- db-sync | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 db-sync diff --git a/db-sync b/db-sync new file mode 100644 index 0000000..0baf497 --- /dev/null +++ b/db-sync @@ -0,0 +1,108 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + + +# eval this +BASEURL="ftp://ftp.archlinux.org/\$repo/os/\$arch/\$file" + +# Generates an URL from BASE_URL +# _Params_ +# * repo +# * arch +# * file +eval_url() { + repo="$1" + arch="$2" + file="$3" + + eval "${BASE_URL}" +} + +# Returns contents of a repo +get_repos() { + rsync -av --include="*.db*" --exclude="*" rsync://${mirror}/${mirror_path}/ cache/ +} + +get_repo_content() { +# Return all contents + bsdtar tf cache/$1/os/$2/$1.db.tar.* | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Get the database compression as an extension +get_repo_ext() { + file "$1" | tr A-Z a-z | sed -e "s/^[^:]\+: *\(.z\).*$/.tar.\1/" -e "s/bz/&2" +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + [ ! -f "cache/${1}/os/${2}/${1}.db.tar.*" ] && return 1 + + echo cache/${1}/os/${2}/${1}.db.tar.* +} + +# Process the databases and get the libre packages +init() { +# Fail on every error + set -E + + source $(dirname $0)/config + source $(dirname $0)/local_config + source $(dirname $0)/libremessages + +# Get the blacklisted packages + blacklist=($(get_blacklist)) + +# Sync the repos databases + get_repos + +# Traverse all repo-arch pairs + for _arch in ${ARCHARCHES[@]}; do + for _repo in ${ARCHREPOS[@]}; do + msg "Processing ${_repo}-${_arch}" + + repo_file=$(get_repo_file ${_repo} ${_arch}) + +# Remove blacklisted packages and count them + msg2 "Removing blacklisted packages: $( + LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ + grep "\-> Removing" 2>/dev/null| wc -l)" + +# Get db contents + db=($(get_repo ${_repo} ${_arch})) + + msg2 "Process clean db for syncing..." + +# Create a whitelist + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + +# Sync excluding everything but blacklist + rsync -av --include-from=/tmp/${_repo}-${_arch}.whitelist --exclude="*" + + +# Cleanup + unset db + done + done + +# Cleanup + unset blacklist _arch _repo repo_file +} + -- cgit v1.2.3 From 666f2b153c4bd06259124dcc6641db6b21495787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 30 Oct 2011 20:34:34 -0300 Subject: Seems to work, entering test status. --- createrepos | 8 +++++ db-sync | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 108 insertions(+), 15 deletions(-) create mode 100755 createrepos mode change 100644 => 100755 db-sync diff --git a/createrepos b/createrepos new file mode 100755 index 0000000..4ee057b --- /dev/null +++ b/createrepos @@ -0,0 +1,8 @@ +#!/bin/bash +# Creates the repo structure defined in config + +source $(dirname $0)/config + +mkdir -p ${FTP_BASE}/{${PKGPOOL},${SRCPOOL}} ${ARCH_BASE} ${CLEANUP_DESTDIR} ${SOURCE_CLEANUP_DESTDIR} ${STAGING} + +$(dirname $0)/create-repo ${PKGREPOS[@]} diff --git a/db-sync b/db-sync old mode 100644 new mode 100755 index 0baf497..4c5dd7a --- a/db-sync +++ b/db-sync @@ -11,8 +11,13 @@ # * Check database signatures # * Sync repo => repo +# TODO +# * verbose mode +# * make a tarball of files used for forensics +# * get files db # eval this +# *not needed* BASEURL="ftp://ftp.archlinux.org/\$repo/os/\$arch/\$file" # Generates an URL from BASE_URL @@ -30,12 +35,18 @@ eval_url() { # Returns contents of a repo get_repos() { - rsync -av --include="*.db*" --exclude="*" rsync://${mirror}/${mirror_path}/ cache/ +# Exclude everything but db files + rsync -avm --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ cache/ } get_repo_content() { # Return all contents - bsdtar tf cache/$1/os/$2/$1.db.tar.* | \ + bsdtar tf ${1} | \ cut -d "/" -f 1 | \ sort -u } @@ -53,22 +64,20 @@ get_blacklist() { # repo # arch get_repo_file() { - [ ! -f "cache/${1}/os/${2}/${1}.db.tar.*" ] && return 1 + [ ! -r "cache/${1}/os/${2}/${1}${DBEXT}" ] && return 1 - echo cache/${1}/os/${2}/${1}.db.tar.* + echo "cache/${1}/os/${2}/${1}${DBEXT}" } # Process the databases and get the libre packages init() { -# Fail on every error - set -E - - source $(dirname $0)/config - source $(dirname $0)/local_config - source $(dirname $0)/libremessages # Get the blacklisted packages blacklist=($(get_blacklist)) +# Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" # Sync the repos databases get_repos @@ -81,28 +90,104 @@ init() { repo_file=$(get_repo_file ${_repo} ${_arch}) # Remove blacklisted packages and count them +# TODO capture all removed packages for printing on debug mode msg2 "Removing blacklisted packages: $( LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ grep "\-> Removing" 2>/dev/null| wc -l)" # Get db contents - db=($(get_repo ${_repo} ${_arch})) + db=($(get_repo_content ${repo_file})) msg2 "Process clean db for syncing..." -# Create a whitelist +# Create a whitelist, add * wildcard to end +# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if +# we aren't syncing them echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist -# Sync excluding everything but blacklist - rsync -av --include-from=/tmp/${_repo}-${_arch}.whitelist --exclude="*" + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + +# Sync excluding everything but whitelist +# We delete here for cleanup + rsync -vrtlH \ + --delete-after \ + --safe-links \ + --delay-updates \ + --max-delete=1000 \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ +# Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) # Cleanup unset db done done + msg "Putting databases back in place" + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + cache/ \ + ${FTP_BASE}/ + + msg "Syncing package pool" +# Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + +# Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + + # Cleanup - unset blacklist _arch _repo repo_file + unset blacklist whitelists _arch _repo repo_file +} + +trap_exit() { + echo + error "$@" + exit 1 } + +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +init -- cgit v1.2.3 From 92a8525eb5ed349f95c080c61c821399e3917842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 30 Oct 2011 20:49:41 -0300 Subject: Local test config --- config | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config b/config index c8b3f0b..9a20f4a 100644 --- a/config +++ b/config @@ -1,10 +1,10 @@ #!/bin/bash -FTP_BASE="/home/repo/public" -ARCH_BASE="/home/repo/public" -SVNREPO="/home/repo/abslibre" +FTP_BASE="/tmp/repo" +ARCH_BASE="/tmp/repo" +SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') +ARCHREPOS=('core') #'extra' 'community' 'testing' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos @@ -32,7 +32,7 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) +ARCHARCHES=(i686) # x86_64) ARCHES=(${ARCHARCHES[@]} mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" @@ -40,4 +40,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" +BLACKLIST_FILE="$FTP_BASE/blacklist.txt" -- cgit v1.2.3 From c60d06e050e929d454b447a3cac3263a83d0a445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 3 Nov 2011 15:42:12 -0300 Subject: Removed unneeded code --- db-sync | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/db-sync b/db-sync index 4c5dd7a..0bb79b7 100755 --- a/db-sync +++ b/db-sync @@ -16,23 +16,6 @@ # * make a tarball of files used for forensics # * get files db -# eval this -# *not needed* -BASEURL="ftp://ftp.archlinux.org/\$repo/os/\$arch/\$file" - -# Generates an URL from BASE_URL -# _Params_ -# * repo -# * arch -# * file -eval_url() { - repo="$1" - arch="$2" - file="$3" - - eval "${BASE_URL}" -} - # Returns contents of a repo get_repos() { # Exclude everything but db files @@ -51,11 +34,6 @@ get_repo_content() { sort -u } -# Get the database compression as an extension -get_repo_ext() { - file "$1" | tr A-Z a-z | sed -e "s/^[^:]\+: *\(.z\).*$/.tar.\1/" -e "s/bz/&2" -} - # Prints blacklisted packages get_blacklist() { cut -d ':' -f 1 "${BLACKLIST_FILE}" -- cgit v1.2.3 From 801ea2c927ace5ee892209dd8e3c1044e1b3842e Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Nov 2011 05:14:04 -0800 Subject: Fixes while testing it --- config | 10 +++++----- db-sync | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config b/config index a0849a9..23b7bbb 100644 --- a/config +++ b/config @@ -1,10 +1,10 @@ #!/bin/bash -FTP_BASE="/tmp/repo" -ARCH_BASE="/tmp/repo" +FTP_BASE="/srv/http/repo/public/temprepo" +ARCH_BASE="/srv/http/repo/public/temprepo" SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core') #'extra' 'community' 'testing' 'multilib') +ARCHREPOS=('core' 'testing') #'extra' 'community' 'testing' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos @@ -32,7 +32,7 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="/tmp" -ARCHARCHES=(i686) # x86_64) +ARCHARCHES=(i686 x86_64) ARCHES=(${ARCHARCHES[@]} mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" @@ -40,4 +40,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="/etc/makepkg.conf" -BLACKLIST_FILE="$FTP_BASE/blacklist.txt" +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" diff --git a/db-sync b/db-sync index 0bb79b7..1a9e02f 100755 --- a/db-sync +++ b/db-sync @@ -89,7 +89,7 @@ init() { # We delete here for cleanup rsync -vrtlH \ --delete-after \ - --safe-links \ + --delete-excluded \ --delay-updates \ --max-delete=1000 \ --include-from=/tmp/${_repo}-${_arch}.whitelist \ -- cgit v1.2.3 From 93255c0baf9beb4f86815a8ba45c7cfa07ccac22 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Nov 2011 05:18:51 -0800 Subject: Local changes --- config | 21 +++++++++++---------- migrate-repo | 29 +++++++++++++++++++++++++++++ repo-add | 2 +- repo-update | 3 ++- 4 files changed, 43 insertions(+), 12 deletions(-) create mode 100755 migrate-repo diff --git a/config b/config index 0e8dc2e..0e26efb 100644 --- a/config +++ b/config @@ -1,27 +1,27 @@ #!/bin/bash -FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +FTP_BASE="/srv/http/repo/public" +ARCH_BASE="/srv/http/repo/public" +SVNREPO="/srv/http/repo/abslibre" # Repos from Arch ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') # Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'social') +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' CLEANUP_DESTDIR="$FTP_BASE/old/packages" -CLEANUP_DRYRUN=false +CLEANUP_DRYRUN=true # Time in days to keep moved packages CLEANUP_KEEP=30 SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" -SOURCE_CLEANUP_DRYRUN=false +SOURCE_CLEANUP_DRYRUN=true # Time in days to keep moved sourcepackages SOURCE_CLEANUP_KEEP=30 @@ -31,13 +31,14 @@ LOCK_DELAY=10 LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" -TMPDIR="$HOME/tmp" +TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) -ARCHES=(${ARCHARCHES[@]} mips64el) +OURARCHES=(mips64el) +ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" -MAKEPKGCONF="$HOME/etc/makepkg.conf" +MAKEPKGCONF="~/.makepkg.conf" BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" diff --git a/migrate-repo b/migrate-repo new file mode 100755 index 0000000..2e44adb --- /dev/null +++ b/migrate-repo @@ -0,0 +1,29 @@ +#!/bin/bash + +source $(dirname $0)/config + +#dryrun="--dry-run" + +# Sync our repos +#for parabola_repo in ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}; do +for parabola_repo in ${USERREPOS[@]} ${PROJREPOS[@]}; do + echo ":: Syncing ${parabola_repo}/ => ${FTP_BASE}/${parabola_repo}/" + rsync -avL ${dryrun} --progress -e ssh parabolavnx@69.163.153.218:repo/${parabola_repo}/ ${FTP_BASE}/${parabola_repo}/ +done + +# Sync our arches +#for parabola_arch in ${OURARCHES[@]}; do +#for arch_repo in ${ARCHREPOS[@]}; do + #echo ":: Syncing ${arch_repo}/os/${parabola_arch}/ => ${FTP_BASE}/${arch_repo}/os/${parabola_arch}/" +# rsync -avL ${dryrun} --progress -e ssh parabolavnx@69.163.153.218:repo/${arch_repo}/os/${parabola_arch}/ ${FTP_BASE}/${arch_repo}/os/${parabola_arch}/ +#done +#done +# +# Sync other dirs last +#for other in screenshots isos; do +for other in other; do + echo ":: Syncing ${other}/ => ${FTP_BASE}/${other}/" + rsync -avL ${dryrun} --progress -e ssh parabolavnx@69.163.153.218:repo/${other}/ ${FTP_BASE}/${other} +done + +exit $? diff --git a/repo-add b/repo-add index c4bf96f..ef9c1e1 100755 --- a/repo-add +++ b/repo-add @@ -20,7 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -LICENSESDIR=/home/parabolavnx/licenses +LICENSESDIR=/srv/http/repo/licenses # gettext initialization export TEXTDOMAIN='pacman' diff --git a/repo-update b/repo-update index 4f31116..bf8d73c 100755 --- a/repo-update +++ b/repo-update @@ -22,7 +22,8 @@ for repo in ${ARCHREPOS[@]}; do # rsync_not_needed for using as an rsync exclude-from #find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ # -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 - find ${repodir}/${PKGPOOL}/ -name "*${PKGEXT}" \ + #find ${repodir}/${PKGPOOL}/ -name "*${PKGEXT}" \ + find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 # Actual rsync command ${rsync_update_command} \ -- cgit v1.2.3 From 9ef417865ef8b102875d42d4cb8ebe277de4ec01 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 23 Nov 2011 07:54:13 -0800 Subject: Making it work --- config | 2 +- db-sync | 63 +++++++++++++++++++++++++++++++++++---------------------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/config b/config index 7122aef..e0d474c 100644 --- a/config +++ b/config @@ -4,7 +4,7 @@ ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') +ARCHREPOS=('core' 'testing' 'extra' 'community') # 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos diff --git a/db-sync b/db-sync index 1a9e02f..320a71d 100755 --- a/db-sync +++ b/db-sync @@ -18,13 +18,14 @@ # Returns contents of a repo get_repos() { + mkdir -p ${TMPDIR}/${PID}.cache # Exclude everything but db files - rsync -avm --include="*/" \ + rsync -vmrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ --exclude="*" \ --delete-after \ - rsync://${mirror}/${mirrorpath}/ cache/ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${PID}.cache } get_repo_content() { @@ -42,9 +43,9 @@ get_blacklist() { # repo # arch get_repo_file() { - [ ! -r "cache/${1}/os/${2}/${1}${DBEXT}" ] && return 1 +# shopt -s nullglob - echo "cache/${1}/os/${2}/${1}${DBEXT}" + echo "${TMPDIR}/${PID}.cache/${1}/os/${2}/${1}${DBEXT}" } # Process the databases and get the libre packages @@ -61,8 +62,8 @@ init() { get_repos # Traverse all repo-arch pairs - for _arch in ${ARCHARCHES[@]}; do - for _repo in ${ARCHREPOS[@]}; do + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do msg "Processing ${_repo}-${_arch}" repo_file=$(get_repo_file ${_repo} ${_arch}) @@ -91,7 +92,6 @@ init() { --delete-after \ --delete-excluded \ --delay-updates \ - --max-delete=1000 \ --include-from=/tmp/${_repo}-${_arch}.whitelist \ --exclude="*" \ rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ @@ -100,17 +100,18 @@ init() { # Add a new whitelist whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + msg "Putting databases back in place" + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/${PID}.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + # Cleanup unset db done done - msg "Putting databases back in place" - rsync -vrtlH \ - --delay-updates \ - --safe-links \ - cache/ \ - ${FTP_BASE}/ msg "Syncing package pool" # Concatenate all whitelists @@ -121,13 +122,15 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - rsync -vrtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ + for PKGPOOL in pool/packages pool/community extra/os/any community/os/any; do + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done # Sync sources msg "Syncing source pool" @@ -137,13 +140,15 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - rsync -vrtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ + for SRCPOOL in sources/packages sources/community ; do + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done # Cleanup @@ -153,7 +158,7 @@ init() { trap_exit() { echo error "$@" - exit 1 + exit 1 } @@ -169,3 +174,5 @@ trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init + +rm -r ${TMPDIR}/${PID}.cache -- cgit v1.2.3 From 51e733fdb525e1c1e9392aa4c3a0b882a6433ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 23 Nov 2011 13:19:35 -0300 Subject: Cleanup script + New config vars --- config | 6 ++++++ db-cleanup | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ db-sync | 4 ++-- 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100755 db-cleanup diff --git a/config b/config index e0d474c..47c9c66 100644 --- a/config +++ b/config @@ -15,6 +15,12 @@ PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' +# Directories where packages are shared between repos +# *relative to FTP_BASE* +PKGPOOLS=('pool/packages' 'pool/community' 'extra/os/any' 'community/os/any') +# Directories where sources are stored +SRCPOOLS=('sources/packages' 'sources/community') + CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=true # Time in days to keep moved packages diff --git a/db-cleanup b/db-cleanup new file mode 100755 index 0000000..0c093c2 --- /dev/null +++ b/db-cleanup @@ -0,0 +1,63 @@ +#!/bin/bash +# Syncs pools against themselves using database contents as filter to cleanup +# them up +# License: GPLv3 + +# Principles +# * Get repos dbs contents +# * Make them a include list +# * Rsync pools against themselves removing excluded files +# * Instant cleanup! + +trap_exit() { + echo + error "$@" + exit 1 +} + +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +(( CLEANUP_DRYRUN )) && EXTRAFLAGS+=" --dry-run" + +for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Getting ${_repo}-${_arch} database" + + dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" + + if [ ! -r "${dbfile}" ]; then + warning "Not found" + continue + fi + +# Echo the contents into a filter file + bsdtar tf "${dbfile}" | \ + cut -d'/' -f1 | \ + sort -u | \ + sed "s|$|*|" >> /tmp/$0.$PID.filter + + done +done + +msg "Removing old files:" + +for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do + msg2 "${POOL}" + + rsync ${EXTRAFLAGS} -va --delete-excluded \ + --include-from="${dbfile}" \ + --exclude="*" \ + ${FTP_BASE}/${POOL}/ \ + ${FTP_BASE}/${POOL}/ +done + +exit $? diff --git a/db-sync b/db-sync index 320a71d..bcdf41a 100755 --- a/db-sync +++ b/db-sync @@ -122,7 +122,7 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - for PKGPOOL in pool/packages pool/community extra/os/any community/os/any; do + for PKGPOOL in ${PKGPOOLS[@]}; do rsync -vrtlH \ --delay-updates \ --safe-links \ @@ -140,7 +140,7 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - for SRCPOOL in sources/packages sources/community ; do + for SRCPOOL in ${SRCPOOLS[@]}; do rsync -vrtlH \ --delay-updates \ --safe-links \ -- cgit v1.2.3 From ee7bba114de818df23196edb7d0d785217b7a332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 23 Nov 2011 13:24:00 -0300 Subject: DRYRUN check --- db-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-cleanup b/db-cleanup index 0c093c2..b12e9c1 100755 --- a/db-cleanup +++ b/db-cleanup @@ -26,7 +26,7 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -(( CLEANUP_DRYRUN )) && EXTRAFLAGS+=" --dry-run" +${CLEANUP_DRYRUN} && EXTRAFLAGS+=" --dry-run" for _repo in ${ARCHREPOS[@]}; do for _arch in ${ARCHARCHES[@]}; do -- cgit v1.2.3 From 4a31562deb872713acae3d17f0f638edd0c4158e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 23 Nov 2011 13:43:04 -0300 Subject: Fixed errors in logic --- db-cleanup | 8 ++++---- db-sync | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/db-cleanup b/db-cleanup index b12e9c1..72790f1 100755 --- a/db-cleanup +++ b/db-cleanup @@ -28,8 +28,8 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR ${CLEANUP_DRYRUN} && EXTRAFLAGS+=" --dry-run" -for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do +for _repo in ${PKGREPOS[@]}; do + for _arch in ${ARCHES[@]}; do msg "Getting ${_repo}-${_arch} database" dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" @@ -43,7 +43,7 @@ for _repo in ${ARCHREPOS[@]}; do bsdtar tf "${dbfile}" | \ cut -d'/' -f1 | \ sort -u | \ - sed "s|$|*|" >> /tmp/$0.$PID.filter + sed "s|$|*|" >> /tmp/$0.$$.filter done done @@ -54,7 +54,7 @@ for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do msg2 "${POOL}" rsync ${EXTRAFLAGS} -va --delete-excluded \ - --include-from="${dbfile}" \ + --include-from="/tmp/$0.$$.filter" \ --exclude="*" \ ${FTP_BASE}/${POOL}/ \ ${FTP_BASE}/${POOL}/ diff --git a/db-sync b/db-sync index bcdf41a..110dba1 100755 --- a/db-sync +++ b/db-sync @@ -18,14 +18,14 @@ # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/${PID}.cache + mkdir -p ${TMPDIR}/$0.$$.cache # Exclude everything but db files rsync -vmrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ --exclude="*" \ --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${PID}.cache + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache } get_repo_content() { @@ -45,7 +45,7 @@ get_blacklist() { get_repo_file() { # shopt -s nullglob - echo "${TMPDIR}/${PID}.cache/${1}/os/${2}/${1}${DBEXT}" + echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}" } # Process the databases and get the libre packages @@ -104,7 +104,7 @@ init() { rsync -vrtlH \ --delay-updates \ --safe-links \ - ${TMPDIR}/${PID}.cache/${_repo}/os/${_arch}/ \ + ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ ${FTP_BASE}/${_repo}/os/${_arch}/ # Cleanup @@ -175,4 +175,4 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init -rm -r ${TMPDIR}/${PID}.cache +rm -r ${TMPDIR}/$0.$$.cache -- cgit v1.2.3 From 323b2e1d83af45b7e4fe90c4822d3366cc48e588 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 7 Dec 2011 06:04:08 -0800 Subject: Added [multilib] --- config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config b/config index 47c9c66..92b7bfa 100644 --- a/config +++ b/config @@ -4,11 +4,11 @@ ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community') # 'multilib') +ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') # Community project repos PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) @@ -22,7 +22,7 @@ PKGPOOLS=('pool/packages' 'pool/community' 'extra/os/any' 'community/os/any') SRCPOOLS=('sources/packages' 'sources/community') CLEANUP_DESTDIR="$FTP_BASE/old/packages" -CLEANUP_DRYRUN=true +CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -- cgit v1.2.3 From d2d3d2785d622b9bd8539deca4d79ccc4a3d52c4 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 7 Dec 2011 06:04:39 -0800 Subject: Skip invalid repo-arch pairs (useful for [multilib]) --- db-sync | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/db-sync b/db-sync index 110dba1..d86d348 100755 --- a/db-sync +++ b/db-sync @@ -68,6 +68,11 @@ init() { repo_file=$(get_repo_file ${_repo} ${_arch}) + if [ ! -f "${repo_file}" ]; then + warning "${repo_file} doesn't exist, skipping this repo-arch" + continue + fi + # Remove blacklisted packages and count them # TODO capture all removed packages for printing on debug mode msg2 "Removing blacklisted packages: $( -- cgit v1.2.3 From 2932fb1b926827c2e12eccd7eb10c3116b6835a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 7 Dec 2011 11:55:06 -0300 Subject: Any-to-ours recycles Arch packages ("any" architecture) to Parabola specific architectures ("mips64el") --- any-to-ours | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100755 any-to-ours diff --git a/any-to-ours b/any-to-ours new file mode 100755 index 0000000..2fa8323 --- /dev/null +++ b/any-to-ours @@ -0,0 +1,67 @@ +#!/bin/bash +# Releases 'any' packages from Arch arches to ours + +trap_exit() { + echo + error "$@" + exit 1 +} + +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +# The architecture to compare with +BASEARCH='x86_64' + +# Traverse all Arch repos +for _repo in ${ARCHREPOS[@]}; do + msg "Processing ${_repo}..." + +# Find 'any' packages +# This is hardcoded but it could release other arches... + PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + -iname '*-any.pkg.tar.?z' \ + -printf "%f ")) + + if [ ${#PKGS[@]} -eq 0 ]; then + msg2 "No 'any' packages here" + continue + fi + + for _arch in ${OURARCHES[@]}; do + msg2 "Syncing ${_arch}..." + +# Sync 'any' only and extract the synced packages + SYNCED=($( + rsync -av \ + --include='*-any.pkg.tar.?z' \ + --exclude='*' \ + ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ + grep '-any.pkg.tar' | \ + cut -d ' ' -f 1 )) + + msg2 "Synced ${#SYNCED[@]} packages" + + msg2 "Adding to db..." + + pushd ${FTP_BASE}/${_repo}/os/${_arch}/ >/dev/null + +# Add the packages to the db + $(dirname $0)/repo-add ${_repo}${DBEXT} \ + ${SYNCED[@]} + + popd >/dev/null + +# Avoid mixups + unset SYNCED PKGS + done +done -- cgit v1.2.3 From 6fedfa4233c8e237d75ddec6ac7122e051f509a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 7 Dec 2011 12:10:26 -0300 Subject: Removed extra - --- any-to-ours | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/any-to-ours b/any-to-ours index 2fa8323..8c37622 100755 --- a/any-to-ours +++ b/any-to-ours @@ -46,7 +46,7 @@ for _repo in ${ARCHREPOS[@]}; do --exclude='*' \ ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ - grep '-any.pkg.tar' | \ + grep 'any.pkg.tar' | \ cut -d ' ' -f 1 )) msg2 "Synced ${#SYNCED[@]} packages" -- cgit v1.2.3 From 56b914d69d96cc410d9bf385c18bcb128af83a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 7 Dec 2011 12:11:36 -0300 Subject: Added synced check --- any-to-ours | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/any-to-ours b/any-to-ours index 8c37622..1bd7e8a 100755 --- a/any-to-ours +++ b/any-to-ours @@ -49,6 +49,11 @@ for _repo in ${ARCHREPOS[@]}; do grep 'any.pkg.tar' | \ cut -d ' ' -f 1 )) + if [ ${#SYNCED[@]} -eq 0 ]; then + msg2 "Already synced (or error happened)" + continue + fi + msg2 "Synced ${#SYNCED[@]} packages" msg2 "Adding to db..." -- cgit v1.2.3 From a749983e8f2becf7c5427b0e5c449d457afbe385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 7 Dec 2011 12:17:58 -0300 Subject: Be verbose --- any-to-ours | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/any-to-ours b/any-to-ours index 1bd7e8a..15a20be 100755 --- a/any-to-ours +++ b/any-to-ours @@ -54,7 +54,7 @@ for _repo in ${ARCHREPOS[@]}; do continue fi - msg2 "Synced ${#SYNCED[@]} packages" + msg2 "Synced ${#SYNCED[@]} packages: ${SYNCED[@]}" msg2 "Adding to db..." -- cgit v1.2.3 From 59105fccf9f5866b451cdaf90052aa8c9346e5f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 8 Jan 2012 15:17:46 -0300 Subject: remove locks --- cron-jobs/ftpdir-cleanup | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index bb1661a..83e6e17 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -22,13 +22,6 @@ clean_pkg() { fi } -script_lock - -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 - done -done ${CLEANUP_DRYRUN} && warning 'dry run mode is active' -- cgit v1.2.3 From 3b719e6ffcc9a4e30ba9ecb00de7be0e1e11bc46 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 8 Jan 2012 18:26:20 +0000 Subject: [gnu] repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 92b7bfa..949d171 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') # Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' -- cgit v1.2.3 From 6e90f39ff729714516779f96875dc9b46854b094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 8 Jan 2012 15:27:12 -0300 Subject: Get_repos scripts updates parabolaweb package database --- get_repos | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 get_repos diff --git a/get_repos b/get_repos new file mode 100755 index 0000000..824ab15 --- /dev/null +++ b/get_repos @@ -0,0 +1,33 @@ +#!/bin/bash +# TODO needs refactoring + +PDIR="/srv/http/web" +DIR="/srv/http/web/repos" + +for arch in i686 x86_64 mips64el; do + for repo in community extra core testing libre libre-testing social; do + [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch + + cd $DIR/$arch + + rm -f *.db.tar.gz + + wget http://repo.parabolagnulinux.org/$repo/os/$arch/$repo.db.tar.gz && \ + $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz + done +done + +for repo in connos connos-extra; do + arch=i586 + [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch + + cd $DIR/$arch + + rm -f *.db.tar.gz + + wget http://www.connochaetos.org/os/$arch/$repo/$repo.db.tar.gz && \ + $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz +done + + +exit 0 -- cgit v1.2.3 From a3e0b61d4d14f01442b98deaa18f66ceb3c4b3b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 8 Jan 2012 15:27:34 -0300 Subject: Use standar naming --- get-repos | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 get-repos diff --git a/get-repos b/get-repos new file mode 100755 index 0000000..824ab15 --- /dev/null +++ b/get-repos @@ -0,0 +1,33 @@ +#!/bin/bash +# TODO needs refactoring + +PDIR="/srv/http/web" +DIR="/srv/http/web/repos" + +for arch in i686 x86_64 mips64el; do + for repo in community extra core testing libre libre-testing social; do + [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch + + cd $DIR/$arch + + rm -f *.db.tar.gz + + wget http://repo.parabolagnulinux.org/$repo/os/$arch/$repo.db.tar.gz && \ + $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz + done +done + +for repo in connos connos-extra; do + arch=i586 + [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch + + cd $DIR/$arch + + rm -f *.db.tar.gz + + wget http://www.connochaetos.org/os/$arch/$repo/$repo.db.tar.gz && \ + $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz +done + + +exit 0 -- cgit v1.2.3 From 47b0e5207b933e5563324c92c8a1841322cde6e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 9 Jan 2012 15:34:13 -0300 Subject: Use config ARCHES and REPOS to get-repos --- config | 5 +++++ get-repos | 71 ++++++++++++++++++++++++++++++++++++++++++++++----------------- get_repos | 33 ----------------------------- 3 files changed, 57 insertions(+), 52 deletions(-) delete mode 100755 get_repos diff --git a/config b/config index 949d171..1304e01 100644 --- a/config +++ b/config @@ -11,6 +11,8 @@ OURREPOS=('libre' 'libre-testing') USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') # Community project repos PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu') +# Remote repos +RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' @@ -48,3 +50,6 @@ SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" + +# parabolaweb root +WEB_DIR=/srv/http/web diff --git a/get-repos b/get-repos index 824ab15..9ff701c 100755 --- a/get-repos +++ b/get-repos @@ -1,33 +1,66 @@ #!/bin/bash -# TODO needs refactoring +# Gets repo databases and updates parabolaweb +# Note: It works remotely because our parabolaweb server and repo server are +# two different hosts -PDIR="/srv/http/web" -DIR="/srv/http/web/repos" +trap_exit() { + echo + error "$@" + exit 1 +} -for arch in i686 x86_64 mips64el; do - for repo in community extra core testing libre libre-testing social; do - [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages - cd $DIR/$arch +# From makepkg +set -E - rm -f *.db.tar.gz +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - wget http://repo.parabolagnulinux.org/$repo/os/$arch/$repo.db.tar.gz && \ - $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz +TMPDIR="$(mktemp -d /tmp/$0.XXXX)" +DBLIST=() + +# Repos +for _repo in ${PKGREPOS[@]}; do + for _arch in ${ARCHES[@]}; do + DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${DBEXT}") done done -for repo in connos connos-extra; do - arch=i586 - [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch +# Remote repos +# TODO don't hardcode the remote architecture +# TODO don't hardcode the remote url +# MAYBE run scripts on get-repos.d/ which should return db urls +for _repo in ${RMTREPOS}; do + _arch=i586 + DBLIST+=("http://www.connochaetos.org/os/${_arch}/${_repo}/${_repo}${DBEXT}") +done - cd $DIR/$arch +# Get them all +msg "Retrieving ${#DBLIST[@]} databases" +wget --directory-prefix=${TMPDIR} \ + --no-verbose \ + --force-directories \ + --no-host-directories \ + ${DBLIST[@]} || true +# Always return true, some databases are expect to be missing - rm -f *.db.tar.gz +# Create the arches regexp arch1|arch2 +arch_re="$(echo "(${ARCHES[@]} i586)" | tr ' ' '|')" - wget http://www.connochaetos.org/os/$arch/$repo/$repo.db.tar.gz && \ - $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz -done +msg "Adding to parabolaweb" +find "${TMPDIR}" -iname "*${DBEXT}" | while read _db; do + _arch=$(echo "${_db}" | egrep -o "${arch_re}") + if [ -z "${_arch}" ]; then + error "Can't find database architecture: ${_db}" + continue + fi + + "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" +done -exit 0 +exit $? diff --git a/get_repos b/get_repos deleted file mode 100755 index 824ab15..0000000 --- a/get_repos +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# TODO needs refactoring - -PDIR="/srv/http/web" -DIR="/srv/http/web/repos" - -for arch in i686 x86_64 mips64el; do - for repo in community extra core testing libre libre-testing social; do - [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch - - cd $DIR/$arch - - rm -f *.db.tar.gz - - wget http://repo.parabolagnulinux.org/$repo/os/$arch/$repo.db.tar.gz && \ - $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz - done -done - -for repo in connos connos-extra; do - arch=i586 - [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch - - cd $DIR/$arch - - rm -f *.db.tar.gz - - wget http://www.connochaetos.org/os/$arch/$repo/$repo.db.tar.gz && \ - $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz -done - - -exit 0 -- cgit v1.2.3 From 88773a8de82e8064fcf1e9fb6b73d64c717b7cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 23 Jan 2012 12:06:56 -0300 Subject: Cross-toolchains repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 1304e01..4de9d00 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') # Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu') +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3 From 864dae08b0431febd00349e60c73e392f8be8ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 30 Jan 2012 19:59:18 -0300 Subject: ABSLibre creation script --- abslibre | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100755 abslibre diff --git a/abslibre b/abslibre new file mode 100755 index 0000000..9485986 --- /dev/null +++ b/abslibre @@ -0,0 +1,93 @@ +#!/bin/bash + +ABSLIBRE=/var/abslibre +ABSGIT=/srv/git/repositories/abslibre.git +# Remote +# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git +BLACKLIST='http://repo.parabolagnulinux.org/docs/blacklist.txt' +SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' +BLFILE=/tmp/blacklist.txt + +. /etc/abs.conf + +# Steps +# * Sync abs +# * Download blacklist.txt +# * Sync abslibre from abs excluding from blacklist +# * Create repo.abs.tar.gz tarballs + +function sync_abs() { + for ARCH in any i686 x86_64; do + rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? + done +} + +function get_blacklist() { + printf ":: Updating blacklist...\t" + wget -q -O - "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ + sed "s/^/**\//" > ${BLFILE} || { + printf "[FAILED]\n" + return 1 + } + + printf "[OK]\n" +} + +function sync_abs_libre() { + +# Clone ABSLibre git repo + if [ -d /tmp/abslibre/.git ]; then + pushd /tmp/abslibre >/dev/null 2>&1 + git pull + popd >/dev/null 2>&1 + else + git clone /srv/git/repositories/abslibre.git /tmp/abslibre + fi + +# Sync from ABS and then sync from ABSLibre + printf ":: Syncing ABSLibre...\t" + (rsync ${SYNCARGS} --delete-excluded \ + --exclude-from=${BLFILE} \ + ${ABSROOT} \ + ${ABSLIBRE} \ + && + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + printf "[FAILED]\n" + return 1 + } + + printf "[OK]\n" +} + +sync_pre_mips64el() { + pushd /home/parabola/abslibre-pre-mips64el >/dev/null + + rsync ${SYNCARGS} --exclude=.git* ${ABSLIBRE}/x86_64/ /home/parabola/abslibre-pre-mips64el/ && git add * && git commit -m "$(date)" +} + +# Create .abs.tar.gz tarballs +create_tarballs() { + for repo in ${ABSLIBRE}/{i686,x86_64}/*; do + baserepo=$(basename $repo) + arch=$(basename $(dirname $repo)) + +# Remove the old one + mkdir -p /srv/http/web/media/abs/$baserepo/os/$arch/ + rm /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz +# Create a new one joining arch and any +# Remove the first part of the path (it could be $repo but any isn't hit) + bsdtar -czvf /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ + $repo/* ${ABSLIBRE}/any/${baserepo}/* + + done +} + +sync_abs || exit 1 +get_blacklist || exit 1 +sync_abs_libre || exit 1 +# This is being done at repo server now +sync_pre_mips64el || exit 1 +create_tarballs || exit 1 + +exit 0 -- cgit v1.2.3 From 4c6b6514e5eb78742369bea3e79fa2c749c1741a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Wed, 1 Feb 2012 16:37:10 +0100 Subject: check-package-libraries.py, aiming to help issue224. --- COPYING | 674 +++++++++++++++++++++++++++++++++++++++++++++ check-package-libraries.py | 193 +++++++++++++ 2 files changed, 867 insertions(+) create mode 100644 COPYING create mode 100755 check-package-libraries.py diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/check-package-libraries.py b/check-package-libraries.py new file mode 100755 index 0000000..bc2349b --- /dev/null +++ b/check-package-libraries.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Check which libraries are provided or required by a package, store +this in a database, update and list broken packages. + +Dependencies: + +- Python 3.2 or later with SQLite 3 support + +- ``bsdtar`` + +- ``readelf`` +""" + + +import os.path +import re +import sqlite3 +import subprocess +import tempfile + + +#: Regexp matching an interesting dynamic entry. +_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" + "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") + + +def make_db(path): + """Make a new, empty, library database at *path*.""" + con = sqlite3.connect(path) + con.executescript(""" +create table provided( + library varchar not null, + package varchar not null +); +create table used( + library varchar not null, + package varchar not null +); +""") + con.close() + + +def begin(database): + """Connect to *database* and start a transaction.""" + con = sqlite3.connect(database) + con.execute("begin exclusive") + return con + + +def add_provided(con, package, libraries): + """Write that *package* provides *libraries*.""" + for library in libraries: + con.execute("insert into provided (package, library) values (?,?)", + (package, library)) + + +def add_used(con, package, libraries): + """Write that *package* uses *libraries*.""" + for library in libraries: + con.execute("insert into used (package, library) values (?,?)", + (package, library)) + + +def remove_package(con, package): + """Remove all entries for a package.""" + con.execute("delete from provided where package=?", (package,)) + con.execute("delete from used where package=?", (package,)) + + +def add_package(con, package): + """Add entries from a named *package*.""" + # Extract to a temporary directory. This could be done more + # efficiently, since there is no need to store more than one file + # at once. + with tempfile.TemporaryDirectory() as temp: + tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) + tar.communicate() + with open(os.path.join(temp, ".PKGINFO")) as pkginfo: + for line in pkginfo: + if line.startswith("pkgname ="): + pkgname = line[len("pkgname ="):].strip() + break + # Don't list previously removed libraries. + remove_package(con, pkgname) + provided = set() + used = set() + # Search for ELFs. + for dirname, dirnames, filenames in os.walk(temp): + assert dirnames is not None # unused, avoid pylint warning + for file_name in filenames: + path = os.path.join(dirname, file_name) + with open(path, "rb") as file_object: + if file_object.read(4) != b"\177ELF": + continue + readelf = subprocess.Popen(("readelf", "-d", path), + stdout=subprocess.PIPE) + for line in readelf.communicate()[0].split(b"\n"): + match = _DYNAMIC.match(line.decode("ascii")) + if match: + if match.group(1) == "SONAME": + provided.add(match.group(2)) + elif match.group(1) == "NEEDED": + used.add(match.group(2)) + else: + raise AssertionError("unknown entry type " + + match.group(1)) + add_provided(con, pkgname, provided) + add_used(con, pkgname, used) + + +def init(arguments): + """Initialize.""" + make_db(arguments.database) + + +def add(arguments): + """Add packages.""" + con = begin(arguments.database) + for package in arguments.packages: + add_package(con, package) + con.commit() + con.close() + + +def remove(arguments): + """Remove packages.""" + con = begin(arguments.database) + for package in arguments.packages: + remove_package(con, package) + con.commit() + con.close() + + +def check(arguments): + """List broken packages.""" + con = begin(arguments.database) + available = set(row[0] for row + in con.execute("select library from provided")) + for package, library in con.execute("select package, library from used"): + if library not in available: + print(package, "needs", library) + con.close() + + +def main(): + """Get arguments and run the command.""" + from argparse import ArgumentParser + parser = ArgumentParser(prog="check-package-libraries.py", + description="Check packages for " + "provided/needed libraries") + parser.add_argument("-d", "--database", type=str, + help="Database file to use", + default="package-libraries.sqlite") + subparsers = parser.add_subparsers() + subparser = subparsers.add_parser(name="init", + help="initialize the database") + subparser.set_defaults(command=init) + subparser = subparsers.add_parser(name="add", + help="add packages to database") + subparser.add_argument("packages", nargs="+", type=str, + help="package files to add") + subparser.set_defaults(command=add) + subparser = subparsers.add_parser(name="remove", + help="remove packages from database") + subparser.add_argument("packages", nargs="+", type=str, + help="package names to remove") + subparser.set_defaults(command=remove) + subparser = subparsers.add_parser(name="check", + help="list broken packages") + subparser.set_defaults(command=check) + arguments = parser.parse_args() + arguments.command(arguments) + + +if __name__ == "__main__": + main() -- cgit v1.2.3 From 834327b7defe8273f7978328ad1b822960cd5ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 14 Feb 2012 12:00:50 -0300 Subject: Ignore rsync temp files --- db-functions | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/db-functions b/db-functions index 4dddcb5..4c247a7 100644 --- a/db-functions +++ b/db-functions @@ -293,6 +293,9 @@ getpkgfile() { } getpkgfiles() { +# Ignore anything that doesn't glob to PKGEXT + shopt -s nullglob + local f if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then error 'Duplicate packages found!' @@ -310,6 +313,8 @@ getpkgfiles() { done echo ${@} + + shopt -u nullglob } check_pkgfile() { -- cgit v1.2.3 From c90a9245b331f43f369b4ef59e67cb2b16c72988 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 15 Feb 2012 20:30:26 +0000 Subject: Use basename --- get-repos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get-repos b/get-repos index 9ff701c..8a96125 100755 --- a/get-repos +++ b/get-repos @@ -20,7 +20,7 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -TMPDIR="$(mktemp -d /tmp/$0.XXXX)" +TMPDIR="$(mktemp -d /tmp/$(basename $0).XXXX)" DBLIST=() # Repos -- cgit v1.2.3 From 75ad12313337cfb57fed8633b667e1fa8e3d48db Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 27 Feb 2012 19:34:41 +0000 Subject: Added user repos --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 4de9d00..af31f37 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien') # Community project repos PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos -- cgit v1.2.3 From 46224d22f8f1f59144b1feab2abd7aca18901e9a Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 27 Feb 2012 19:46:45 +0000 Subject: [extra] doesn't have the any dir anymore --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index af31f37..b1a7308 100644 --- a/config +++ b/config @@ -19,7 +19,7 @@ SRCPOOL='sources/packages' # Directories where packages are shared between repos # *relative to FTP_BASE* -PKGPOOLS=('pool/packages' 'pool/community' 'extra/os/any' 'community/os/any') +PKGPOOLS=('pool/packages' 'pool/community' 'community/os/any') # Directories where sources are stored SRCPOOLS=('sources/packages' 'sources/community') -- cgit v1.2.3 From 623ccccff54b5497708debd0a6271d8143d03257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 22 Mar 2012 12:02:45 -0300 Subject: community/os/any doesn't exist anymore --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index b1a7308..cb552e4 100644 --- a/config +++ b/config @@ -19,7 +19,7 @@ SRCPOOL='sources/packages' # Directories where packages are shared between repos # *relative to FTP_BASE* -PKGPOOLS=('pool/packages' 'pool/community' 'community/os/any') +PKGPOOLS=('pool/packages' 'pool/community') # Directories where sources are stored SRCPOOLS=('sources/packages' 'sources/community') -- cgit v1.2.3 From a9680de1b0b37d8ceba6ab90864681d0bf4d7fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 11 Apr 2012 18:16:02 -0300 Subject: Git-PBS forks packages from Arch svntogit/packages.git It creates a single package git repo and tracks upstream changes. It's intended to fork upstream PKGBUILDs into their -libre versions. **STILL UNDER TEST** --- git-pbs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 git-pbs diff --git a/git-pbs b/git-pbs new file mode 100755 index 0000000..b815863 --- /dev/null +++ b/git-pbs @@ -0,0 +1,44 @@ +#!/bin/bash + +_pkg=$1 + +mkdir -p $_pkg +pushd $_pkg + + +if [ ! -d .git ]; then +# Start a git repo for the package +# Add the remote origin +# Pull the package branch onto an unmodified branch + git init + git remote add arch git://projects.archlinux.org/svntogit/packages.git + +# Export the repository + touch .git/git-daemon-export-ok + +# Pass the -b flag to checkout to create the branches + extra="-b" +fi + +git checkout ${extra} upstream +git pull arch packages/$_pkg + +# Move PKGBUILD and files to the basedir +# Remove everything else from the repo +git checkout ${extra} master + +# This produces a lot of merging conflicts +git merge upstream + +# This apparently solves them +git mv trunk/* . +git rm -rf repos + +# Remove the actual files +rm -rf trunk repos + +# Commit everything +git commit -a -m "Converted to PBS" + +# Return to the repo +popd >/dev/null -- cgit v1.2.3 From 65a4d2f541a4292efe422f829d5eac41c548efd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 18 Apr 2012 18:42:20 -0300 Subject: adding artistic repo to community project --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index cb552e4..b3a4658 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien') # Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') +PROJREPOS=('social' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3 From 6942fc113939c487a4292589225d0cc60a6a4222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Fri, 20 Apr 2012 02:07:01 -0300 Subject: adding jorginho repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index b3a4658..e6f52e1 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho') # Community project repos PROJREPOS=('social' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos -- cgit v1.2.3 From 6aa0446283a8ee995fae1e5da196a8027032d68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 26 Apr 2012 15:41:01 -0300 Subject: [gis] repo for geospatial software --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index e6f52e1..4909b6d 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho') # Community project repos -PROJREPOS=('social' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') +PROJREPOS=('social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3 From bd8ba6724d09ca83a533f5641fc84ad3ac492e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Silva?= Date: Mon, 21 May 2012 16:11:56 -0300 Subject: adding coadde repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 4909b6d..c582bea 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos PROJREPOS=('social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos -- cgit v1.2.3 From f43b4881b168b951f39dd00f075c8d47ef17df4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Tue, 22 May 2012 11:27:20 -0300 Subject: adding pcr (Parabola Community Repo) --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index c582bea..20ad307 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3 From 4d3cf060c4a7d2b671d9cdf0ab0f6c1d89a8204b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 28 May 2012 14:03:30 -0300 Subject: Remove TMPDIR --- get-repos | 2 ++ 1 file changed, 2 insertions(+) diff --git a/get-repos b/get-repos index 8a96125..86b371c 100755 --- a/get-repos +++ b/get-repos @@ -63,4 +63,6 @@ find "${TMPDIR}" -iname "*${DBEXT}" | while read _db; do "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" done +rm -r ${TMPDIR} + exit $? -- cgit v1.2.3 From 7592118ed1c8952d05f1239e3b84d85d39ea2982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 26 Jun 2012 16:06:53 -0300 Subject: Re-enabled check on other repos; skip already released packages --- db-update | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index 86eaa2e..1fe5256 100755 --- a/db-update +++ b/db-update @@ -35,9 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi - #if ! check_pkgrepos "${pkg}"; then - # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" - #fi + if ! check_pkgrepos "${pkg}"; then + die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + fi done # This is fucking obnoxious # if ! check_splitpkgs ${repo} ${pkgs[@]}; then @@ -61,7 +61,7 @@ for repo in ${repos[@]}; do if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" fi - ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" + ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" 2>/dev/null || { error "Package already in repo, skipping" ; continue } # also move signatures if [ -f "${pkg}.sig" ]; then mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" -- cgit v1.2.3 From 0e43f902b1056bfbed57cba6bb299a75fcdb19a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 21 Aug 2012 16:37:28 -0300 Subject: Sync signatures too --- any-to-ours | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/any-to-ours b/any-to-ours index 15a20be..3a58946 100755 --- a/any-to-ours +++ b/any-to-ours @@ -43,10 +43,11 @@ for _repo in ${ARCHREPOS[@]}; do SYNCED=($( rsync -av \ --include='*-any.pkg.tar.?z' \ + --include='*-any.pkg.tar.?z.sig' \ --exclude='*' \ ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ - grep 'any.pkg.tar' | \ + grep 'any\.pkg\.tar\..z$' | \ cut -d ' ' -f 1 )) if [ ${#SYNCED[@]} -eq 0 ]; then -- cgit v1.2.3 From 9a535045c075d90a962d2817651140feaeb59952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 21 Aug 2012 17:15:33 -0300 Subject: Fix --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 1fe5256..7a8d6d1 100755 --- a/db-update +++ b/db-update @@ -61,7 +61,7 @@ for repo in ${repos[@]}; do if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" fi - ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" 2>/dev/null || { error "Package already in repo, skipping" ; continue } + ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" 2>/dev/null || { error "Package already in repo, removing and skipping" ; continue 2; } # also move signatures if [ -f "${pkg}.sig" ]; then mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" -- cgit v1.2.3 From d5aa8796a3b3e7314ab2ae0ec566e38a8de8bc28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 21 Aug 2012 17:16:24 -0300 Subject: This is clearly not the way --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 7a8d6d1..875b87f 100755 --- a/db-update +++ b/db-update @@ -61,7 +61,7 @@ for repo in ${repos[@]}; do if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" fi - ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" 2>/dev/null || { error "Package already in repo, removing and skipping" ; continue 2; } + ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" # also move signatures if [ -f "${pkg}.sig" ]; then mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" -- cgit v1.2.3 From 5f5b5b10e700af5d7e6be352cde1e1e3380e1d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Thu, 13 Sep 2012 11:09:35 +0200 Subject: db-update: Ignore repos in repos. --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 875b87f..0359697 100755 --- a/db-update +++ b/db-update @@ -9,7 +9,7 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -repos=($(find "${STAGING}" -mindepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) if [ $? -ge 1 ]; then die "Could not read ${STAGING}" fi -- cgit v1.2.3 From d8ee98931359b87871bf0bef0b2ac5fca19d4d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 2 Oct 2012 13:03:11 -0300 Subject: Deprecating repo-add with license extraction --- repo-add | 561 --------------------------------------------------------------- 1 file changed, 561 deletions(-) delete mode 100755 repo-add diff --git a/repo-add b/repo-add deleted file mode 100755 index ef9c1e1..0000000 --- a/repo-add +++ /dev/null @@ -1,561 +0,0 @@ -#!/bin/bash -# -# repo-add - add a package to a given repo database file -# repo-remove - remove a package entry from a given repo database file -# Generated from repo-add.in; do not edit by hand. -# -# Copyright (c) 2006-2008 Aaron Griffin -# Copyright (c) 2007-2008 Dan McGee -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -LICENSESDIR=/srv/http/repo/licenses - -# gettext initialization -export TEXTDOMAIN='pacman' -export TEXTDOMAINDIR='/usr/share/locale' - -myver='3.5.0' -confdir='/home/parabolavnx/etc' - -QUIET=0 -DELTA=0 -WITHFILES=0 -REPO_DB_FILE= -LOCKFILE= -CLEAN_LOCK=0 - -# ensure we have a sane umask set -umask 0022 - -msg() { - (( QUIET )) && return - local mesg=$1; shift - printf "==> ${mesg}\n" "$@" >&1 -} - -msg2() { - (( QUIET )) && return - local mesg=$1; shift - printf " -> ${mesg}\n" "$@" >&1 -} - -warning() { - local mesg=$1; shift - printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 -} - -# print usage instructions -usage() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" - printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" - printf "$(gettext "\ -repo-add will update a package database by reading a package file.\n\ -Multiple packages to add can be specified on the command line.\n\n")" - printf "$(gettext "\ -repo-remove will update a package database by removing the package name\n\ -specified on the command line from the given repo database. Multiple\n\ -packages to remove can be specified on the command line.\n\n")" - printf "$(gettext "\ -Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ -and errors.\n\n")" - printf "$(gettext "\ -Use the -d/--delta flag to automatically generate and add a delta file\n\ -between the old entry and the new one, if the old package file is found\n\ -next to the new one.\n\n")" - printf "$(gettext "\ -Use the -f/--files flag to update a database including file entries.\n\n")" - echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" - echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" -} - -version() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "\ -Copyright (C) 2006-2008 Aaron Griffin .\n\ -Copyright (c) 2007-2008 Dan McGee .\n\n\ -This is free software; see the source for copying conditions.\n\ -There is NO WARRANTY, to the extent permitted by law.\n")" -} - -# write a list entry -# arg1 - Entry name -# arg2 - List -# arg3 - File to write to -write_list_entry() { - if [[ -n $2 ]]; then - echo "%$1%" >>$3 - echo -e $2 >>$3 - fi -} - -find_pkgentry() -{ - local pkgname=$1 - local pkgentry - for pkgentry in $tmpdir/$pkgname*; do - name=${pkgentry##*/} - if [[ ${name%-*-*} = $pkgname ]]; then - echo $pkgentry - return 0 - fi - done - return 1 -} - -# Get the package name from the delta filename -get_delta_pkgname() { - local tmp - - tmp=${1##*/} - echo ${tmp%-*-*_to*} -} - -# write a delta entry -# arg1 - path to delta file -db_write_delta() -{ - deltafile="$1" - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - error "$(gettext "No database entry for package '%s'.")" "$pkgname" - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - echo -e "%DELTAS%" >$deltas - fi - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$deltafile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$deltafile") - - oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') - newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') - - if grep -q "$oldfile.*$newfile" $deltas; then - sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup - fi - msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" - echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas - - return 0 -} # end db_write_delta - -# remove a delta entry -# arg1 - path to delta file -db_remove_delta() -{ - deltafile="$1" - filename=${deltafile##*/} - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - return 1 - fi - if grep -q "$filename" $deltas; then - sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup - msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" - return 0 - fi - - return 1 -} # end db_remove_delta - -# write an entry to the pacman database -# arg1 - path to package -db_write_entry() -{ - # blank out all variables - local pkgfile="$1" - local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ - _groups _licenses _replaces _depends _conflicts _provides _optdepends - - local OLDIFS="$IFS" - # IFS (field separator) is only the newline character - IFS=" -" - - # read info from the zipped package - local line var val - for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | - grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do - # bash awesomeness here- var is always one word, val is everything else - var=${line%% *} - val=${line#* } - declare $var="$val" - case "$var" in - group) _groups="$_groups$group\n" ;; - license) _licenses="$_licenses$license\n" ;; - replaces) _replaces="$_replaces$replaces\n" ;; - depend) _depends="$_depends$depend\n" ;; - conflict) _conflicts="$_conflicts$conflict\n" ;; - provides) _provides="$_provides$provides\n" ;; - optdepend) _optdepends="$_optdepends$optdepend\n" ;; - esac - done - - IFS=$OLDIFS - - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$pkgfile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$pkgfile") - - # ensure $pkgname and $pkgver variables were found - if [[ -z $pkgname || -z $pkgver ]]; then - error "$(gettext "Invalid package file '%s'.")" "$pkgfile" - return 1 - fi - - pushd "$tmpdir" >/dev/null - if [[ -d $pkgname-$pkgver ]]; then - warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" - else - if (( DELTA )); then - pkgentry=$(find_pkgentry $pkgname) - if [[ -n $pkgentry ]]; then - local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) - local oldfile="$(dirname $1)/$oldfilename" - fi - fi - fi - - # remove an existing entry if it exists, ignore failures - db_remove_entry "$pkgname" - - # create package directory - mkdir "$pkgname-$pkgver" - pushd "$pkgname-$pkgver" >/dev/null - - # restore an eventual deltas file - [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas - - # create desc entry - msg2 "$(gettext "Creating '%s' db entry...")" 'desc' - echo -e "%FILENAME%\n$(basename "$1")\n" >>desc - echo -e "%NAME%\n$pkgname\n" >>desc - [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc - echo -e "%VERSION%\n$pkgver\n" >>desc - [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc - write_list_entry "GROUPS" "$_groups" "desc" - [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc - [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc - - # compute checksums - msg2 "$(gettext "Computing md5 checksums...")" - echo -e "%MD5SUM%\n$md5sum\n" >>desc - - [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc - write_list_entry "LICENSE" "$_licenses" "desc" - [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc - [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc - [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc - write_list_entry "REPLACES" "$_replaces" "desc" - - # create depends entry - msg2 "$(gettext "Creating '%s' db entry...")" 'depends' - # create the file even if it will remain empty - touch "depends" - write_list_entry "DEPENDS" "$_depends" "depends" - write_list_entry "CONFLICTS" "$_conflicts" "depends" - write_list_entry "PROVIDES" "$_provides" "depends" - write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - - popd >/dev/null - popd >/dev/null - - # create files file if wanted - if (( WITHFILES )); then - msg2 "$(gettext "Creating '%s' db entry...")" 'files' - local files_path="$tmpdir/$pkgname-$pkgver/files" - echo "%FILES%" >$files_path - bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path - fi - - # create a delta file - if (( DELTA )); then - if [[ -n $oldfilename ]]; then - if [[ -f $oldfile ]]; then - delta=$(pkgdelta -q $oldfile $1) - if [[ -f $delta ]]; then - db_write_delta $delta - fi - else - warning "$(gettext "Old package file not found: %s")" "$oldfilename" - fi - fi - fi - - # Extracts licenses to a common license dir - msg "Extracting license" - if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - # Change dir to licenses, and extract them stripping the first part of the path - bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ - --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 - - if [ $? -ne 0 ]; then - warning "This package doesn't contain a license dir" - fi - fi - - return 0 -} # end db_write_entry - -# remove existing entries from the DB -# arg1 - package name -db_remove_entry() { - local pkgname=$1 - local notfound=1 - local pkgentry=$(find_pkgentry $pkgname) - while [[ -n $pkgentry ]]; do - notfound=0 - if [[ -f $pkgentry/deltas ]]; then - mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" - fi - msg2 "$(gettext "Removing existing entry '%s'...")" \ - "$(basename $pkgentry)" - rm -rf $pkgentry - pkgentry=$(find_pkgentry $pkgname) - done - - msg "Removing license" - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - return $notfound -} # end db_remove_entry - -check_repo_db() -{ - # check lock file - if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then - CLEAN_LOCK=1 - else - error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" - exit 1 - fi - - if [[ -f $REPO_DB_FILE ]]; then - # there are two situations we can have here- a DB with some entries, - # or a DB with no contents at all. - if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then - # check empty case - if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then - error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" - exit 1 - fi - fi - msg "$(gettext "Extracting database to a temporary location...")" - bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" - else - case "$cmd" in - repo-remove) - error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" - exit 1 - ;; - repo-add) - # check if the file can be created (write permission, directory existence, etc) - if ! touch "$REPO_DB_FILE"; then - error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" - exit 1 - fi - rm -f "$REPO_DB_FILE" - ;; - esac - fi -} - -add() -{ - if [[ ! -f $1 ]]; then - error "$(gettext "File '%s' not found.")" "$1" - return 1 - fi - - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Adding delta '%s'")" "$deltafile" - if ! type xdelta3 &>/dev/null; then - error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" - exit 1 - fi - if db_write_delta "$deltafile"; then - return 0 - else - return 1 - fi - fi - - pkgfile=$1 - if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then - error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" - return 1 - fi - - msg "$(gettext "Adding package '%s'")" "$pkgfile" - - db_write_entry "$pkgfile" -} - -remove() -{ - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Searching for delta '%s'...")" "$deltafile" - if db_remove_delta "$deltafile"; then - return 0 - else - error "$(gettext "Delta matching '%s' not found.")" "$deltafile" - return 1 - fi - fi - - pkgname=$1 - msg "$(gettext "Searching for package '%s'...")" "$pkgname" - - if db_remove_entry "$pkgname"; then - rm -f "$tmpdir/$pkgname.deltas" - return 0 - else - error "$(gettext "Package matching '%s' not found.")" "$pkgname" - return 1 - fi -} - -trap_exit() -{ - echo - error "$@" - exit 1 -} - -clean_up() { - local exit_code=$? - - [[ -d $tmpdir ]] && rm -rf "$tmpdir" - (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" - - exit $exit_code -} - -# PROGRAM START - -# determine whether we have gettext; make it a no-op if we do not -if ! type gettext &>/dev/null; then - gettext() { - echo "$@" - } -fi - -case "$1" in - -h|--help) usage; exit 0;; - -V|--version) version; exit 0;; -esac - -# figure out what program we are -cmd="$(basename $0)" -if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then - error "$(gettext "Invalid command name '%s' specified.")" "$cmd" - exit 1 -fi - -tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ - error "$(gettext "Cannot create temp directory for database building.")"; \ - exit 1) - -trap 'clean_up' EXIT -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR - -success=0 -# parse arguments -for arg in "$@"; do - case "$arg" in - -q|--quiet) QUIET=1;; - -d|--delta) DELTA=1;; - -f|--files) WITHFILES=1;; - *) - if [[ -z $REPO_DB_FILE ]]; then - REPO_DB_FILE="$arg" - LOCKFILE="$REPO_DB_FILE.lck" - check_repo_db - else - case "$cmd" in - repo-add) add $arg && success=1 ;; - repo-remove) remove $arg && success=1 ;; - esac - fi - ;; - esac -done - -# if at least one operation was a success, re-zip database -if (( success )); then - msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" - - case "$REPO_DB_FILE" in - *tar.gz) TAR_OPT="z" ;; - *tar.bz2) TAR_OPT="j" ;; - *tar.xz) TAR_OPT="J" ;; - *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ - "$REPO_DB_FILE" ;; - esac - - filename=$(basename "$REPO_DB_FILE") - - pushd "$tmpdir" >/dev/null - if [[ -n $(ls) ]]; then - bsdtar -c${TAR_OPT}f "$filename" * - else - # we have no packages remaining? zip up some emptyness - warning "$(gettext "No packages remain, creating empty database.")" - bsdtar -c${TAR_OPT}f "$filename" -T /dev/null - fi - popd >/dev/null - - [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" - [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" - dblink="${REPO_DB_FILE%.tar.*}" - target=${REPO_DB_FILE##*/} - ln -sf "$target" "$dblink" 2>/dev/null || \ - ln -f "$target" "$dblink" 2>/dev/null || \ - cp "$REPO_DB_FILE" "$dblink" -else - msg "$(gettext "No packages modified, nothing to do.")" - exit 1 -fi - -exit 0 -# vim: set ts=2 sw=2 noet: -- cgit v1.2.3 From 53eb3d26ed11098b0d1803dcee46bef40c1c67d8 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 7 Oct 2012 18:52:52 +0000 Subject: Use system repo-add --- any-to-ours | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/any-to-ours b/any-to-ours index 3a58946..a1d6686 100755 --- a/any-to-ours +++ b/any-to-ours @@ -62,8 +62,8 @@ for _repo in ${ARCHREPOS[@]}; do pushd ${FTP_BASE}/${_repo}/os/${_arch}/ >/dev/null # Add the packages to the db - $(dirname $0)/repo-add ${_repo}${DBEXT} \ - ${SYNCED[@]} + repo-add ${_repo}${DBEXT} \ + ${SYNCED[@]} popd >/dev/null -- cgit v1.2.3 From 58dcd2bd938eb64e3c0d31841f98f43a8a1ca346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Oct 2012 10:57:28 -0300 Subject: Deprecating repo-update Thanks to aurelien to make me find the misterious garbage packages under staging/! --- repo-update | 59 ----------------------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100755 repo-update diff --git a/repo-update b/repo-update deleted file mode 100755 index bf8d73c..0000000 --- a/repo-update +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- -source ~/.bashrc -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages - -for repo in ${ARCHREPOS[@]}; do - - msg "Syncing ${repo}" - for arch in ${ARCHARCHES[@]}; do - msg2 "${repo} ${arch}" - # makes a file containing rsync output for filter.py - ${rsync_list_command} \ - rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/os/${arch}/ > ${rsout_file}-${repo}-${arch} || exit 1 - # reads blacklist and rsout_file and makes an rsync exclude-from - # list - filter.py -r ${rsync_blacklist} -k ${blacklist} \ - -f ${rsout_file}-${repo}-${arch} || exit 1 - # list files in ${repodir}/${repo} and write their names on - # rsync_not_needed for using as an rsync exclude-from - #find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ - # -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 - #find ${repodir}/${PKGPOOL}/ -name "*${PKGEXT}" \ - find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ - -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 - # Actual rsync command - ${rsync_update_command} \ - --exclude-from=${rsync_blacklist} \ - --exclude-from=${rsync_not_needed}-${repo}-${arch} \ - rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ || exit 1 - done - for arch in ${ARCHARCHES[@]}; do - msg2 "Making pending list for $repo $arch" - # if there is a db in repo (db is created on rsync) - if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then - # clean_repo makes pending list with files on db and remove - # packages from db - $(dirname $0)/clean_repo.py -k ${blacklist} -w ${whitelist} \ - -p ${docs_dir}/pending-${repo}.txt \ - -b ${repodir}/staging/${repo}/${repo}${DBEXT} \ - -d ${repodir}/stagging/${repo} - fi - done - # if some nonfree files got pass the filter this command delete them - msg2 "Fallback cleaning repo" - $(dirname $0)/clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} || exit 1 -done - -msg "Removing leftover files..." -find ${repodir}/staging/ -type f \! -name "*${PKGEXT}" -delete -# Staging should not have symbolic links -find ${repodir}/staging/ -type l -delete - -$(dirname $0)/db-update -$(dirname $0)/db-check-nonfree - -- cgit v1.2.3 From f516cdb29d89027d267b3a09c98e518e7177251a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Oct 2012 11:13:25 -0300 Subject: Deprecating old python scripts --- __init__.py | 0 clean_repo.py | 99 ---------------------------- config.py | 68 -------------------- filter.py | 204 ---------------------------------------------------------- 4 files changed, 371 deletions(-) delete mode 100644 __init__.py delete mode 100755 clean_repo.py delete mode 100755 config.py delete mode 100755 filter.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/clean_repo.py b/clean_repo.py deleted file mode 100755 index 076de60..0000000 --- a/clean_repo.py +++ /dev/null @@ -1,99 +0,0 @@ -#! /usr/bin/python2 -#-*- encoding: utf-8 -*- -from filter import * -import argparse - -def mkpending(packages_iterable, pending_file, blacklisted_names, - whitelisted_names): - """ Determine wich packages are pending for license auditing.""" - search = tuple(blacklisted_names + - whitelisted_names) - - try: - fsock=open(pending_file, "r") - pkgs=[pkg for pkg in packages_iterable - if pkg["name"] not in listado(pending_file)] - for line in fsock.readlines(): - if line: - pkg=Package() - pkg["name"]=line.split(":")[0] - pkg["license"]=":".join(line.split(":")[1:]) - pkgs.append(pkg) - pkgs=[pkg for pkg in pkgs if pkg["name"] not in search - and "custom" in pkg["license"]] - fsock=open(pending_file, "w") - fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + - ":" + pkg["license"] - for pkg in pkgs]) + "\n") - fsock.close() - except(IOError): - printf("Can't read or write %s" % pending_file) - return pkgs - -def remove_from_blacklist(path_to_db, blacklisted_names): - """ Check the blacklist and remove packages on the db""" - if "~" in path_to_db: - path_to_db=(os.path.expanduser(path_to_db)) - - pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if - pkg["name"] in blacklisted_names] - if pkgs: - lista=" ".join(pkgs) - cmd = "repo-remove " + path_to_db + " " + lista - printf(cmd) - a = check_output(cmd) - return pkgs - -def cleanup_nonfree_in_dir(directory, blacklisted_names): - if "~" in directory: - directory=(os.path.expanduser(directory)) - pkglist=list() - pkgs=pkginfo_from_files_in_dir(directory) - for package in pkgs: - if package["name"] in blacklisted_names: - os.remove(package["location"]) - pkglist.append(package) - return pkglist - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="clean_repo", - description="Clean a repo db and packages",) - - parser.add_argument("-k", "--blacklist-file", type=str, - help="File containing blacklisted names", - required=True,) - - group_dir=parser.add_argument_group("Clean non-free packages in dir") - group_dir.add_argument("-d", "--directory", type=str, - help="directory to clean") - - group_db=parser.add_argument_group("Clean non-free packages in db", - "All these arguments need to be specified for db cleaning:") - group_db.add_argument("-b", "--database", type=str, - help="dabatase to clean") - group_db.add_argument("-p", "--pending-file", type=str, - help="File in which to write pending list") - group_db.add_argument("-w", "--whitelist-file", type=str, - help="File containing whitelisted names") - - args=parser.parse_args() - - if args.database and not (args.pending_file and args.whitelist_file): - parser.print_help() - exit(1) - - blacklisted=listado(args.blacklist_file) - - if args.database: - whitelisted=listado(args.whitelist_file) - pkgs=pkginfo_from_db(args.database) - pending_names=[pkg["name"] for pkg in - mkpending(pkgs, args.pending_file, - blacklisted, whitelisted)] - - if args.directory and args.database: - cleanup_nonfree_in_dir(args.directory, (blacklisted + pending_names)) - elif args.directory: - cleanup_nonfree_in_dir(args.directory, blacklisted) - diff --git a/config.py b/config.py deleted file mode 100755 index 0ffc475..0000000 --- a/config.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/python2 -# -*- coding: utf-8 -*- -try: - from subprocess import check_output -except(ImportError): - from commands import getoutput - def check_output(*popenargs,**kwargs): - cmd=" ".join(*popenargs) - return getoutput(cmd) -import os - - -# Rsync commands - -def printf(text, logfile=False): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - print (str(text) + "\n") - if logfile: - try: - log = open(logfile, 'a') - log.write("\n" + str(text) + "\n") - except: - print("Can't open %s" % logfile) - finally: - log.close() - - -# Classes and Exceptions -class NonValidFile(ValueError): pass -class NonValidDir(ValueError): pass -class NonValidCommand(ValueError): pass - -class Package: - """ An object that has information about a package. """ - package_info=dict() - - def __init__(self): - self.package_info={ "name" : False, - "version" : False, - "release" : False, - "arch" : False, - "license" : False, - "location": False, - "depends" : False,} - - def __setitem__(self, key, item): - if key in self.package_info.keys(): - return self.package_info.__setitem__(key, item) - else: - raise ValueError("Package has no %s attribute" % key) - - def __getitem__(self, key): - return self.package_info.__getitem__(key) - - def __unicode__(self): - return str(self.package_info) - - def __repr__(self): - return str(self.package_info) - - def __eq__(self,x): - if not isinstance(x, Package): - return False - for key in self.package_info.keys(): - if x[key] != self.package_info[key]: - return False - else: - return True diff --git a/filter.py b/filter.py deleted file mode 100755 index 70d5e29..0000000 --- a/filter.py +++ /dev/null @@ -1,204 +0,0 @@ -#! /usr/bin/python2 -#-*- encoding: utf-8 -*- -from glob import glob -from config import * -import tarfile - -def listado(filename, start=0, end=None): - """Obtiene una lista de paquetes de un archivo.""" - fsock = open(filename, "r") - lista = fsock.read().split("\n") - fsock.close() - if end is not None: - return [pkg.split(":")[start:end].rstrip() - for pkg in lista if pkg] - else: - return [pkg.split(":")[start].rstrip() - for pkg in lista if pkg] - -def pkginfo_from_filename(filename): - """ Generates a Package object with info from a filename, - filename can be relative or absolute - - Parameters: - ---------- - filename -> str Must contain .pkg.tar. - - Returns: - ---------- - pkg -> Package object""" - if ".pkg.tar." not in filename: - raise NonValidFile("File is not a pacman package") - pkg = Package() - pkg["location"] = filename - fileattrs = os.path.basename(filename).split("-") - pkg["arch"] = fileattrs.pop(-1).split(".")[0] - pkg["release"] = fileattrs.pop(-1) - pkg["version"] = fileattrs.pop(-1) - pkg["name"] = "-".join(fileattrs) - return pkg - -def pkginfo_from_desc(info_from_desc, pkg=Package()): - """ Returns pkginfo from desc file. - - Parameters: - ---------- - filename -> str File must exist - - Returns: - ---------- - pkg -> Package object""" - info=info_from_desc.rsplit() - info_map={"name" :("%NAME%" , None), - "version" :("%VERSION%" , 0 ), - "release" :("%VERSION%" , 1 ), - "arch" :("%ARCH%" , None), - "license" :("%LICENSE%" , None), - "location":("%FILENAME%", None),} - - for key in info_map.keys(): - field,pos=info_map[key] - pkg[key]=info[info.index(field)+1] - if pos is not None: - pkg[key]=pkg[key].split("-")[pos] - return pkg - -def pkginfo_from_rsync_output(rsync_output): - """ Generates a list of packages and versions from an rsync output - wich uses --list-only and --no-motd options. - - Parameters: - ---------- - rsync_output -> str Contains output from rsync - - Returns: - ---------- - package_list -> tuple Contains Package objects. """ - - def package_or_link(line): - """ Take info out of filename """ - location_field = 4 - return pkginfo_from_filename(line.rsplit()[location_field]) - - def do_nothing(): - pass - - options = { "d": do_nothing, - "l": package_or_link, - "-": package_or_link, - " ": do_nothing} - - package_list=list() - - lines=[x for x in rsync_output.split("\n") if ".pkg.tar" in x] - - for line in lines: - pkginfo=options[line[0]](line) - if pkginfo: - package_list.append(pkginfo) - - return tuple(package_list) - -def pkginfo_from_files_in_dir(directory): - """ Returns pkginfo from filenames of packages in dir - wich has .pkg.tar. on them - - Parameters: - ---------- - directory -> str Directory must exist - - Returns: - ---------- - package_list -> tuple Contains Package objects """ - package_list=list() - - if not os.path.isdir(directory): - raise NonValidDir - - for filename in glob(os.path.join(directory,"*")): - if ".pkg.tar." in filename: - package_list.append(pkginfo_from_filename(filename)) - return tuple(package_list) - -def pkginfo_from_db(path_to_db): - """ Get pkginfo from db. - - Parameters: - ---------- - path_to_db -> str Path to file - - Output: - ---------- - package_list -> tuple of Package objects""" - package_list=list() - - if not os.path.isfile(path_to_db): - raise NonValidFile(path_to_db + " is not a file") - - try: - dbsock = tarfile.open(path_to_db, 'r:gz') - desc_files=[desc for desc in dbsock.getnames() - if "/desc" in desc] - for name in desc_files: - desc=dbsock.extractfile(name).read().decode("UTF-8") - package_list.append(pkginfo_from_desc(desc)) - except tarfile.ReadError: - raise NonValidFile("No valid db_file %s or not readable" - % path_to_db) - finally: - dbsock.close() - return package_list - -def rsyncBlacklist_from_blacklist(packages_iterable, - blacklisted_names, - exclude_file): - """ Generate an exclude list for rsync - - Parameters: - ---------- - package_iterable -> list or tuple Contains Package objects - blacklisted_names-> list or tuple Contains blacklisted names - exclude_file -> str Path to file - debug -> bool If True, file list gets logged - - Output: - ---------- - None """ - pkgs=[pkg["location"] for pkg in packages_iterable - if isinstance(pkg, Package) - and pkg["name"] in blacklisted_names] - if exclude_file: - try: - fsock = open(exclude_file,"w") - fsock.write("\n".join(pkgs) + "\n") - except IOError: - printf("%s wasnt written" % exclude_file) - exit(1) - finally: - fsock.close() - return pkgs - - -if __name__ == "__main__": - import argparse - parser=argparse.ArgumentParser() - parser.add_argument("-r", "--rsync-exclude-file", type=str, - help="File in which to generate exclude list", - required=True,) - parser.add_argument("-k", "--blacklist-file", type=str, - help="File containing blacklisted names", - required=True,) - parser.add_argument("-f", "--rsout-file", type=str, - help="This file will be read to get a pkg list", - required=True,) - args=parser.parse_args() - try: - fsock=open(args.rsout_file, "r") - rsout=fsock.read() - except IOError: - print("%s is not readable" % args.rsout_file) - finally: - fsock.close() - packages=pkginfo_from_rsync_output(rsout) - rsyncBlacklist_from_blacklist(packages, listado(args.blacklist_file), - args.rsync_exclude_file) -- cgit v1.2.3 From f4c42f9e68131ad4fbfb1be07fb44678eb90d6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Sat, 13 Oct 2012 18:06:52 +0200 Subject: db-list-unsigned-packages: New script. --- db-list-unsigned-packages | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 db-list-unsigned-packages diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages new file mode 100755 index 0000000..35a5421 --- /dev/null +++ b/db-list-unsigned-packages @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -e + +# Output a list of repo/package-name-and-version pairs representing +# unsigned packages available for architecture $1 and specified for +# architecture $2 (usually $1 or any, default is to list all). + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -lt 1 ]; then + msg "usage: $(basename $0) " + exit 1 +fi + +arch=$1 +pkgarch=$2 + +for repo in ${PKGREPOS[@]} +do + db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" + for f in $(tar tf $db | egrep /desc$) + do + if ! tar xOf $db $f | fgrep %PGPSIG% > /dev/null + then + if [ -z $2 ] || tar xOf $db $f | fgrep -A1 %ARCH% | fgrep $pkgarch > /dev/null + then + echo $repo/${f%/desc} + fi + fi + done +done -- cgit v1.2.3 From c6542576f04e6dc159e731b50a19f1a10d9d5ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Tue, 16 Oct 2012 15:42:10 +0200 Subject: db-list-unsigned-packages: rewrite using a helper Python script. The previous implementation parsed each tarball multiple times having quadratic time complexity in the number of packages. It was too slow for a complete run. --- db-list-unsigned-packages | 13 ++-------- db-list-unsigned-packages.py | 59 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 11 deletions(-) create mode 100755 db-list-unsigned-packages.py diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 35a5421..26e22eb 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -29,19 +29,10 @@ if [ $# -lt 1 ]; then fi arch=$1 -pkgarch=$2 +shift for repo in ${PKGREPOS[@]} do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - for f in $(tar tf $db | egrep /desc$) - do - if ! tar xOf $db $f | fgrep %PGPSIG% > /dev/null - then - if [ -z $2 ] || tar xOf $db $f | fgrep -A1 %ARCH% | fgrep $pkgarch > /dev/null - then - echo $repo/${f%/desc} - fi - fi - done + "$(dirname $0)/db-list-unsigned-packages.py" "$@" < "$db" done diff --git a/db-list-unsigned-packages.py b/db-list-unsigned-packages.py new file mode 100755 index 0000000..ccbec3a --- /dev/null +++ b/db-list-unsigned-packages.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Output a list of repo/package-name-and-version pairs representing +unsigned packages in the database at standard input of repo named in +the first argument and specified for architectures listed in the +following arguments (usually the one of the database or any, default +is to list all). +""" + + +import sys +import tarfile + + +def main(): + """Do the job.""" + repo = sys.argv[1] + pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) + with tarfile.open(fileobj=sys.stdin.buffer) as archive: + for entry in archive: + if entry.name.endswith("/desc"): + content = archive.extractfile(entry) + skip = False + is_arch = False + for line in content: + if is_arch: + is_arch = False + if pkgarches and line.strip() not in pkgarches: + skip = True # different architecture + break + if line == b"%PGPSIG%\n": + skip = True # signed + break + if line == b"%ARCH%\n": + is_arch = True + if skip: + print("skip " + repo + "/" + entry.name[:-5]) + continue + print(repo + "/" + entry.name[:-5]) + + +if __name__ == "__main__": + main() -- cgit v1.2.3 From be6cff9570eb4e55cfef7a8a83ffd65c37aa2c91 Mon Sep 17 00:00:00 2001 From: Parabola Date: Tue, 16 Oct 2012 13:53:57 +0000 Subject: db-list-unsigned-packages: Ignore missing packages, pass the repo name. --- db-list-unsigned-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 26e22eb..3b5a5bd 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -34,5 +34,5 @@ shift for repo in ${PKGREPOS[@]} do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - "$(dirname $0)/db-list-unsigned-packages.py" "$@" < "$db" + [ -f "$db" ] && "$(dirname $0)/db-list-unsigned-packages.py" "$repo" "$@" < "$db" done -- cgit v1.2.3 From 43aacb891613909fdb0e2898fc9d85bfa20bcfad Mon Sep 17 00:00:00 2001 From: Parabola Date: Tue, 16 Oct 2012 13:54:48 +0000 Subject: db-list-unsigned-packages.py: Don't list signed packages. --- db-list-unsigned-packages.py | 1 - 1 file changed, 1 deletion(-) diff --git a/db-list-unsigned-packages.py b/db-list-unsigned-packages.py index ccbec3a..36be93a 100755 --- a/db-list-unsigned-packages.py +++ b/db-list-unsigned-packages.py @@ -50,7 +50,6 @@ def main(): if line == b"%ARCH%\n": is_arch = True if skip: - print("skip " + repo + "/" + entry.name[:-5]) continue print(repo + "/" + entry.name[:-5]) -- cgit v1.2.3 From 7a2f916a360b16c3649622d9f04c6424aab8f32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 20 Oct 2012 16:25:07 -0300 Subject: Remove broken symlinks after cleanup --- db-cleanup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db-cleanup b/db-cleanup index 72790f1..904c06e 100755 --- a/db-cleanup +++ b/db-cleanup @@ -60,4 +60,8 @@ for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do ${FTP_BASE}/${POOL}/ done +msg "Removing symlinks:" +find -L ${FTP_BASE}/ -type l +${CLEANUP_DRYRUN} || find -L ${FTP_BASE}/ -type l -delete + exit $? -- cgit v1.2.3 From f8f54ad1926d3f10e895fda553474db278c2ed7d Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 31 Oct 2012 04:44:23 +0000 Subject: sitting on the server --- README | 3 +++ config | 2 +- config.orig | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 README create mode 100644 config.orig diff --git a/README b/README new file mode 100644 index 0000000..9ce2d3f --- /dev/null +++ b/README @@ -0,0 +1,3 @@ +* Script to run if there is error and for update + + - db-sync diff --git a/config b/config index 20ad307..7601527 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) diff --git a/config.orig b/config.orig new file mode 100644 index 0000000..a32f82f --- /dev/null +++ b/config.orig @@ -0,0 +1,54 @@ +#!/bin/bash +<<<<<<< HEAD +FTP_BASE="/srv/http/repo/public" +ARCH_BASE="/srv/http/repo/public" +SVNREPO="/srv/http/repo/abslibre" +======= +FTP_BASE="/srv/http/repo/public/temprepo" +ARCH_BASE="/srv/http/repo/public/temprepo" +SVNREPO="/var/abs" +>>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e + +# Repos from Arch +ARCHREPOS=('core' 'testing') #'extra' 'community' 'testing' 'multilib') +# Official Parabola repos +OURREPOS=('libre' 'libre-testing') +# User repos +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') +# Community project repos +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') +PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' + +CLEANUP_DESTDIR="$FTP_BASE/old/packages" +CLEANUP_DRYRUN=true +# Time in days to keep moved packages +CLEANUP_KEEP=30 + +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" +SOURCE_CLEANUP_DRYRUN=true +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP=30 + +REQUIRE_SIGNATURE=true + +LOCK_DELAY=10 +LOCK_TIMEOUT=300 + +STAGING="$FTP_BASE/staging" +TMPDIR="/tmp" +ARCHARCHES=(i686 x86_64) +OURARCHES=(mips64el) +ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) +DBEXT=".db.tar.gz" +FILESEXT=".files.tar.gz" +PKGEXT=".pkg.tar.?z" +SRCEXT=".src.tar.gz" + +<<<<<<< HEAD +MAKEPKGCONF="~/.makepkg.conf" +======= +MAKEPKGCONF="/etc/makepkg.conf" +>>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" -- cgit v1.2.3 From 0d409373d58f1e2ab8a6df6f2123b986aed562a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Tue, 13 Nov 2012 22:35:07 +0100 Subject: get-repos: Don't stop after reporead fails. Issue #220 prevents it from working for some repos. --- get-repos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get-repos b/get-repos index 86b371c..4397d54 100755 --- a/get-repos +++ b/get-repos @@ -60,7 +60,7 @@ find "${TMPDIR}" -iname "*${DBEXT}" | while read _db; do continue fi - "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" + "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" || true done rm -r ${TMPDIR} -- cgit v1.2.3 From 9fba20c1e3a70d5cb6e2a7f59d026ce3e28185e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 30 Apr 2013 14:11:10 -0300 Subject: Removed svn, should work with regular abs --- db-move | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/db-move b/db-move index 019faa2..010d941 100755 --- a/db-move +++ b/db-move @@ -24,12 +24,10 @@ for pkgarch in ${ARCHES[@]}; do repo_lock ${repo_from} ${pkgarch} || exit 1 done -# check if packages to be moved exist in svn and ftp dir -/usr/bin/svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null +# No idea why we loop twice... -- fauno for pkgbase in ${args[@]:2}; do - /usr/bin/svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null for pkgarch in ${ARCHES[@]} 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) if [ ${#pkgnames[@]} -lt 1 ]; then @@ -64,8 +62,7 @@ declare -A add_pkgs declare -A remove_pkgs for pkgbase in ${args[@]:2}; do for pkgarch in ${ARCHES[@]} 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" + svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" if [ -f "${svnrepo_from}/PKGBUILD" ]; then if [ "${pkgarch}" == 'any' ]; then @@ -77,21 +74,6 @@ for pkgbase in ${args[@]:2}; do pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - if [ -d "${svnrepo_to}" ]; then - for file in $(/usr/bin/svn ls "${svnrepo_to}"); do - /usr/bin/svn rm -q "${svnrepo_to}/$file" - done - else - mkdir "${svnrepo_to}" - /usr/bin/svn add -q "${svnrepo_to}" - fi - - for file in $(svn ls "${svnrepo_from}"); do - /usr/bin/svn mv -q -r HEAD "${svnrepo_from}/$file" "${svnrepo_to}/" - done - /usr/bin/svn rm --force -q "${svnrepo_from}" - /usr/bin/svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "$(basename $0): moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${pkgarch})" - for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) -- cgit v1.2.3 From 86aeadb4c3d56f168e0bce34e0e2fa06c1ccb78b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 12 Jun 2013 19:30:34 -0600 Subject: test/: use libremakepkg instead of arch-*build --- test/lib/common.inc | 4 ++-- test/test.d/db-update.sh | 4 ++-- test/test.d/testing2x.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/lib/common.inc b/test/lib/common.inc index aeceece..be039fa 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -46,10 +46,10 @@ oneTimeSetUp() { if ! ${build}; then if [ "${pkgarch[0]}" == 'any' ]; then - sudo extra-x86_64-build || die 'extra-x86_64-build failed' + sudo libremakepkg || die 'libremakepkg failed' else for a in ${pkgarch[@]}; do - sudo extra-${a}-build || die "extra-${a}-build failed" + sudo setarch $a libremakepkg -n $a || die "setarch ${a} libremakepkg -n ${a} failed" done fi for a in ${pkgarch[@]}; do diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 7a5ce4c..4b3008e 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -82,7 +82,7 @@ testUpdateAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build + sudo libremakepkg mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -100,7 +100,7 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build + sudo libremakepkg mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index 31d85b4..0c2fa83 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -10,7 +10,7 @@ testTesting2xAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build + sudo libremakepkg mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null -- cgit v1.2.3 From 5ba4756a52e8f1280d4dbcff62dec4a4aeb47d1c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 12 Jun 2013 19:32:48 -0600 Subject: Don't make assumptions about the host architecture. --- db-functions | 2 +- test/lib/common.inc | 25 +++++++++++++++++-------- test/test.d/create-filelists.sh | 8 ++++---- test/test.d/db-move.sh | 8 ++++---- test/test.d/db-remove.sh | 4 ++-- test/test.d/db-repo-add.sh | 4 ++-- test/test.d/db-repo-remove.sh | 4 ++-- test/test.d/db-update.sh | 8 ++++---- test/test.d/ftpdir-cleanup.sh | 6 +++--- test/test.d/sourceballs.sh | 6 +++--- 10 files changed, 42 insertions(+), 33 deletions(-) diff --git a/db-functions b/db-functions index b3a4293..4029abf 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,7 @@ #!/bin/bash # Some PKGBUILDs need CARCH to be set -CARCH="x86_64" +CARCH=$(. $(librelib conf.sh); load_files makepkg; echo $CARCH) # Useful functions UMASK="" diff --git a/test/lib/common.inc b/test/lib/common.inc index be039fa..2d6140f 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -5,6 +5,12 @@ set -E TMPDIR="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" . "$(dirname ${BASH_SOURCE[0]})/../../db-functions" +arches() ( + . $(librelib conf.sh) + load_files libretools + printf '%s\n' "${ARCHES[*]}" +) + signpkg() { if [[ -r '/etc/makepkg.conf' ]]; then source '/etc/makepkg.conf' @@ -24,6 +30,7 @@ oneTimeSetUp() { local p local d local a + local arches=($(arches)) local pkgname local pkgarch local pkgversion @@ -49,14 +56,16 @@ oneTimeSetUp() { sudo libremakepkg || die 'libremakepkg failed' else for a in ${pkgarch[@]}; do - sudo setarch $a libremakepkg -n $a || die "setarch ${a} libremakepkg -n ${a} failed" + if in_array $a ${arches[@]}; then + sudo setarch $a libremakepkg -n $a || die "setarch ${a} libremakepkg -n ${a} failed" + for p in ${pkgname[@]}; do + cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/${d##*/} + done + else + warning "skipping arch %s" "$a" + fi done fi - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/${d##*/} - done - done fi popd >/dev/null done @@ -161,7 +170,7 @@ checkAnyPackageDB() { [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" fi - for arch in i686 x86_64; do + for arch in $(arches); do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" @@ -268,7 +277,7 @@ checkRemovedAnyPackageDB() { local db for db in ${DBEXT} ${FILESEXT}; do - for arch in i686 x86_64; do + for arch in $(arches); do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 49734c4..8df7030 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testCreateSimpleFileLists() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch @@ -26,7 +26,7 @@ testCreateSimpleFileLists() { } testCreateAnyFileLists() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase local arch @@ -46,7 +46,7 @@ testCreateAnyFileLists() { } testCreateSplitFileLists() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -75,7 +75,7 @@ testCreateSplitFileLists() { testCleanupFileLists() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch diff --git a/test/test.d/db-move.sh b/test/test.d/db-move.sh index 890ffc4..3cf355b 100755 --- a/test/test.d/db-move.sh +++ b/test/test.d/db-move.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testMoveSimplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -28,7 +28,7 @@ testMoveSimplePackages() { } testMoveMultiplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -52,7 +52,7 @@ testMoveMultiplePackages() { } testMoveEpochPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-epoch') local pkgbase local arch @@ -90,7 +90,7 @@ testMoveAnyPackages() { } testMoveSplitPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase diff --git a/test/test.d/db-remove.sh b/test/test.d/db-remove.sh index 416e693..d79605e 100755 --- a/test/test.d/db-remove.sh +++ b/test/test.d/db-remove.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testRemovePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') local pkgbase local arch @@ -31,7 +31,7 @@ testRemovePackages() { } testRemoveMultiplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') local pkgbase local arch diff --git a/test/test.d/db-repo-add.sh b/test/test.d/db-repo-add.sh index 8603104..09fc52f 100755 --- a/test/test.d/db-repo-add.sh +++ b/test/test.d/db-repo-add.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testAddSimplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -27,7 +27,7 @@ testAddSimplePackages() { } testAddMultiplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch diff --git a/test/test.d/db-repo-remove.sh b/test/test.d/db-repo-remove.sh index 315d63d..eec0109 100755 --- a/test/test.d/db-repo-remove.sh +++ b/test/test.d/db-repo-remove.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testRemovePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch @@ -31,7 +31,7 @@ testRemovePackages() { } testRemoveMultiplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 4b3008e..4e77140 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testAddSimplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -52,7 +52,7 @@ testAddAnyPackages() { } testAddSplitPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -132,7 +132,7 @@ testUpdateSameAnyPackageToDifferentRepositories() { ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) local arch - for arch in i686 x86_64; do + for arch in $(arches); do ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ && fail "${pkgbase} should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" @@ -141,7 +141,7 @@ testUpdateSameAnyPackageToDifferentRepositories() { testAddIncompleteSplitPackage() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local repo='extra' local pkgbase='pkg-split-a' local arch diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh index bfea7ea..630b88f 100755 --- a/test/test.d/ftpdir-cleanup.sh +++ b/test/test.d/ftpdir-cleanup.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testCleanupSimplePackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -35,7 +35,7 @@ testCleanupSimplePackages() { } testCleanupEpochPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-epoch') local pkgbase local arch @@ -85,7 +85,7 @@ testCleanupAnyPackages() { } testCleanupSplitPackages() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase diff --git a/test/test.d/sourceballs.sh b/test/test.d/sourceballs.sh index fdcf08c..472cb30 100755 --- a/test/test.d/sourceballs.sh +++ b/test/test.d/sourceballs.sh @@ -4,7 +4,7 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" testSourceballs() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch @@ -38,7 +38,7 @@ testAnySourceballs() { } testSplitSourceballs() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -59,7 +59,7 @@ testSplitSourceballs() { } testSourceballsCleanup() { - local arches=('i686' 'x86_64') + local arches=(`arches`) local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch -- cgit v1.2.3 From 9ea7be252eb359fa8d42eb74897b216158736f56 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 12 Jun 2013 19:52:00 -0600 Subject: fix TMPDIR --- db-functions | 2 +- test/lib/common.inc | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/db-functions b/db-functions index 4029abf..01629bc 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d "${TMPDIR}/${0##*/}.XXXXXXXXXX") +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") if [ -n "${SVNUSER}" ]; then setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" setfacl -m d:u:"${USER}":rwx "${WORKDIR}" diff --git a/test/lib/common.inc b/test/lib/common.inc index 2d6140f..bf9d28e 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -1,8 +1,16 @@ set -E -. "$(dirname ${BASH_SOURCE[0]})/../../config" # override the default TMPDIR -TMPDIR="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" +init_tmpdir() { + [[ -n $MASTER_TMPDIR ]] || export MASTER_TMPDIR="$(mktemp -dqt ${0##*/}.XXXXXXXXXX)" + export TMPDIR=$MASTER_TMPDIR +} + +_TMPDIR=$TMPDIR +. "$(dirname ${BASH_SOURCE[0]})/../../config" +TMPDIR=$_TMPDIR +init_tmpdir + . "$(dirname ${BASH_SOURCE[0]})/../../db-functions" arches() ( @@ -35,7 +43,7 @@ oneTimeSetUp() { local pkgarch local pkgversion local build - pkgdir="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" + pkgdir="$(mktemp -dqt ${0##*/}.XXXXXXXXXX)" cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" msg 'Building packages...' for d in "${pkgdir}"/*; do @@ -82,7 +90,8 @@ setUp() { local a [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" - TMP="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" + init_tmpdir + TMP="$(mktemp -dqt ${0##*/}.XXXXXXXXXX)" #msg "Using ${TMP}" PKGREPOS=('core' 'extra' 'testing') -- cgit v1.2.3 From 5c72f37618b77e7c7ebcab2c998e7be468c34e02 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 15 Jun 2013 13:57:06 -0600 Subject: db-functions: use common.sh --- db-functions | 58 +++++++--------------------------------------------------- 1 file changed, 7 insertions(+), 51 deletions(-) diff --git a/db-functions b/db-functions index 01629bc..523e872 100644 --- a/db-functions +++ b/db-functions @@ -24,57 +24,13 @@ if [ -n "${SVNUSER}" ]; then fi LOCKS=() -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW -if [[ -t 2 ]]; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" -fi -readonly ALL_OFF BOLD BLUE GREEN RED YELLOW - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> WARNING:${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> ERROR${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -## -# usage : in_array( $needle, $haystack ) -# return : 0 - found -# 1 - not found -## -in_array() { - local needle=$1; shift - [[ -z $1 ]] && return 1 # Not Found - local item - for item in "$@"; do - [[ $item = $needle ]] && return 0 # Found - done - return 1 # Not Found -} +# Used: plain, msg, msg2, warning, error, in_array +# Overwritten: get_full_version, +# cleanup, abort, die +# Ignored: stat_busy, stat_done, +# setup_workdir, trab_abort, trap_exit, +# lock, slock, lock_close +. $(librelib common) ## # usage : get_full_version( $epoch, $pkgver, $pkgrel ) -- cgit v1.2.3 From d0e9909ce3419c36e134227318a29f2f1fd782a2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 15 Jun 2013 13:57:35 -0600 Subject: fix tests - properly set SRCPOOL - have SVNREPO point to the checkout, not the server repo - in case TMPDIR has a symlink in it, use `readlink -e` on both sides of inspecting symlinks. - use `grep {pattern} &>/dev/null` instead of `grep -q {pattern}`. Because `grep -q` is able to bail early, a program being piped in to it may spit out a message about a write error to stderr. --- test/lib/common.inc | 20 +++++++++++--------- test/test.d/create-filelists.sh | 10 +++++----- test/test.d/db-update.sh | 4 ++-- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/test/lib/common.inc b/test/lib/common.inc index bf9d28e..fbdd30b 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -96,6 +96,7 @@ setUp() { PKGREPOS=('core' 'extra' 'testing') PKGPOOL='pool/packages' + SRCPOOL='pool/sources' mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} for r in ${PKGREPOS[@]}; do @@ -121,9 +122,10 @@ setUp() { cat < "$(dirname ${BASH_SOURCE[0]})/../../config.local" FTP_BASE="${TMP}/ftp" - SVNREPO="file://${TMP}/svn-packages-repo" + SVNREPO="${TMP}/svn-packages-copy" PKGREPOS=(${PKGREPOS[@]}) PKGPOOL="${PKGPOOL}" + SRCPOOL="${SRCPOOL}" TESTING_REPO='testing' STABLE_REPOS=('core' 'extra') CLEANUP_DESTDIR="${TMP}/package-cleanup" @@ -181,18 +183,18 @@ checkAnyPackageDB() { for arch in $(arches); do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkg}")" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" if ${REQUIRE_SIGNATURE}; then [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkg}.sig")" ] \ || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" fi for db in ${DBEXT} ${FILESEXT}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep ${pkg} &>/dev/null) \ || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" done done @@ -222,7 +224,7 @@ checkPackageDB() { [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkg}")" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" if ${REQUIRE_SIGNATURE}; then @@ -230,13 +232,13 @@ checkPackageDB() { [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkg}.sig")" ] \ || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" fi for db in ${DBEXT} ${FILESEXT}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep ${pkg} &>/dev/null) \ || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" done } @@ -262,7 +264,7 @@ checkRemovedPackageDB() { for db in ${DBEXT} ${FILESEXT}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep ${pkgbase} &>/dev/null) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" done } @@ -288,7 +290,7 @@ checkRemovedAnyPackageDB() { for db in ${DBEXT} ${FILESEXT}; do for arch in $(arches); do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep ${pkgbase} &>/dev/null) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" done done diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 8df7030..e78bde8 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -18,7 +18,7 @@ testCreateSimpleFileLists() { for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/${pkgbase}" &>/dev/null; then fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}" fi done @@ -38,7 +38,7 @@ testCreateAnyFileLists() { for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/share/${pkgbase}/test" &>/dev/null; then fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}" fi done @@ -65,7 +65,7 @@ testCreateSplitFileLists() { pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) for pkgname in ${pkgnames[@]}; do for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/${pkgname}" &>/dev/null; then fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}" fi done @@ -92,10 +92,10 @@ testCleanupFileLists() { done for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/pkg-simple-b" &>/dev/null; then fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}" fi - if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-a"; then + if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/pkg-simple-a" &>/dev/null; then fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}" fi done diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 4e77140..7f1874b 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -134,7 +134,7 @@ testUpdateSameAnyPackageToDifferentRepositories() { local arch for arch in $(arches); do ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep ${pkgbase} &>/dev/null) \ && fail "${pkgbase} should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" done } @@ -157,7 +157,7 @@ testAddIncompleteSplitPackage() { for arch in ${arches[@]}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep ${pkgbase} &>/dev/null) \ && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" done } -- cgit v1.2.3 From 33b8cb611363102e23972cf3914a03d65cb3cad8 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 17 Jun 2013 23:59:15 -0600 Subject: use xbs, not svn --- config | 2 -- cron-jobs/sourceballs | 4 ++-- db-functions | 49 +++++++++++-------------------------------------- db-move | 48 ++++++++++++++---------------------------------- db-remove | 15 +++++---------- db-update | 4 ++-- test/lib/common.inc | 15 ++++++++++++++- testing2x | 10 ++++------ 8 files changed, 52 insertions(+), 95 deletions(-) diff --git a/config b/config index 3df6c95..e1e705b 100644 --- a/config +++ b/config @@ -1,6 +1,4 @@ FTP_BASE="/srv/ftp" -SVNREPO='' -SVNUSER='' PKGREPOS=() PKGPOOL='' SRCPOOL='' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6f75ccc..9572a30 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -77,9 +77,9 @@ for repo in ${PKGREPOS[@]}; do continue fi - # Get the sources from svn + # Get the sources from xbs mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + cp -a "$(xbs releasepath "${pkgbase}" "${repo}" "${pkgarch}")" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" diff --git a/db-functions b/db-functions index 523e872..c98b26f 100644 --- a/db-functions +++ b/db-functions @@ -17,11 +17,6 @@ restore_umask () { # set up general environment WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -if [ -n "${SVNUSER}" ]; then - setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" - setfacl -m d:u:"${USER}":rwx "${WORKDIR}" - setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" -fi LOCKS=() # Used: plain, msg, msg2, warning, error, in_array @@ -292,7 +287,7 @@ check_pkgfile() { fi } -check_pkgsvn() { +check_pkgxbs() { local pkgfile="${1}" local _pkgbase="$(getpkgbase ${pkgfile})" [ $? -ge 1 ] && return 1 @@ -306,18 +301,11 @@ check_pkgsvn() { in_array "${repo}" ${PKGREPOS[@]} || return 1 - if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then - mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 - fi - - local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel}) )" - [ "${svnver}" == "${_pkgver}" ] || return 1 + local xbsver="$(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel}) )" + [ "${xbsver}" == "${_pkgver}" ] || return 1 - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) - in_array "${_pkgname}" ${svnnames[@]} || return 1 + local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo ${pkgname[@]})) + in_array "${_pkgname}" ${xbsnames[@]} || return 1 return 0 } @@ -328,7 +316,7 @@ check_splitpkgs() { local pkgfiles=(${@}) local pkgfile local pkgdir - local svnname + local xbsname mkdir -p "${WORKDIR}/check_splitpkgs/" pushd "${WORKDIR}/check_splitpkgs" >/dev/null @@ -341,16 +329,9 @@ check_splitpkgs() { mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" - if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then - mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 - fi - - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) - for svnname in ${svnnames[@]}; do - echo "${svnname}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" + local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo ${pkgname[@]})) + for xbsname in ${xbsnames[@]}; do + echo "${xbsname}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" done done popd >/dev/null @@ -358,8 +339,8 @@ check_splitpkgs() { for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do [ ! -d "${pkgdir}" ] && continue sort -u "${pkgdir}/staging" -o "${pkgdir}/staging" - sort -u "${pkgdir}/svn" -o "${pkgdir}/svn" - if [ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/svn")" ]; then + sort -u "${pkgdir}/xbs" -o "${pkgdir}/xbs" + if [ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/xbs")" ]; then return 1 fi done @@ -465,11 +446,3 @@ arch_repo_remove() { || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } - -arch_svn() { - if [ -z "${SVNUSER}" ]; then - /usr/bin/svn "${@}" - else - sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}" - fi -} diff --git a/db-move b/db-move index 1fa44d4..045f33c 100755 --- a/db-move +++ b/db-move @@ -24,19 +24,17 @@ for pkgarch in ${ARCHES[@]}; do repo_lock ${repo_from} ${pkgarch} || exit 1 done -# check if packages to be moved exist in svn and ftp dir -arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null +# check if packages to be moved exist in xbs and ftp dir for pkgbase in ${args[@]:2}; do - arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null for pkgarch in ${ARCHES[@]} 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - if [ -r "${svnrepo_from}/PKGBUILD" ]; then - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) + xbsrepo_from="$(xbs releasepath ${pkgbase} ${repo_from} ${pkgarch})" + if [ -r "${xbsrepo_from}/PKGBUILD" ]; then + pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; echo ${pkgname[@]})) if [ ${#pkgnames[@]} -lt 1 ]; then die "Could not read pkgname" fi - pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgver=$(. "${xbsrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) if [ -z "${pkgver}" ]; then die "Could not read pkgver" fi @@ -63,35 +61,19 @@ msg "Moving packages from [${repo_from}] to [${repo_to}]..." declare -A add_pkgs declare -A remove_pkgs for pkgbase in ${args[@]:2}; do - tag_list="" - for pkgarch in ${ARCHES[@]} 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - - if [ -f "${svnrepo_from}/PKGBUILD" ]; then + # move the package in xbs + arches=($(xbs move ${repo_from} ${repo_to} ${pkgbase})) + # move the package in ftp + for pkgarch in ${arches[@]}; do + dir_to="$(xbs releasepath $pkgbase $repo_to $pkgarch)" + if true; then # to add in indent level to make merging easier if [ "${pkgarch}" == 'any' ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi - msg2 "${pkgbase} ($(echo ${tarches[@]}))" - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) - pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - - if [ -d "${svnrepo_to}" ]; then - for file in $(arch_svn ls "${svnrepo_to}"); do - arch_svn rm -q "${svnrepo_to}/$file@" - done - else - mkdir "${svnrepo_to}" - arch_svn add -q "${svnrepo_to}" - fi - - for file in $(arch_svn ls "${svnrepo_from}"); do - arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" - done - arch_svn rm --force -q "${svnrepo_from}" - tag_list="$tag_list, $pkgarch" + pkgnames=($(. "${dir_to}/PKGBUILD"; echo ${pkgname[@]})) + pkgver=$(. "${dir_to}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do @@ -108,8 +90,6 @@ for pkgbase in ${args[@]:2}; do done fi done - tag_list="${tag_list#, }" - arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" done for tarch in ${ARCHES[@]}; do diff --git a/db-remove b/db-remove index 25cb9a7..e632c47 100755 --- a/db-remove +++ b/db-remove @@ -12,9 +12,6 @@ repo="$1" arch="$2" pkgbases=(${@:3}) -ftppath="$FTP_BASE/$repo/os" -svnrepo="$repo-$arch" - if ! check_repo_permission $repo; then die "You don't have permission to remove packages from ${repo}" fi @@ -32,14 +29,12 @@ done remove_pkgs=() for pkgbase in ${pkgbases[@]}; do msg "Removing $pkgbase from [$repo]..." - arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null - - if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then - remove_pkgs=(${remove_pkgs[@]} $(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) - arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" - arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" + path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" + if [ -d "$path" ]; then + remove_pkgs+=($(. "$path/PKGBUILD"; echo ${pkgname[@]})) + xbs unrelease "$pkgbase" "$repo" "$arch" else - warning "$pkgbase not found in $svnrepo" + warning "$pkgbase not found in $repo-$arch" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" remove_pkgs[${#remove_pkgs[*]}]=$pkgbase diff --git a/db-update b/db-update index 576fe2b..28fcabe 100755 --- a/db-update +++ b/db-update @@ -45,8 +45,8 @@ for repo in ${repos[@]}; do if ${REQUIRE_SIGNATURE} && ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then die "Package ${repo}/${pkg##*/} does not have a valid signature" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/${pkg##*/} is not consistent with svn repository" + if ! check_pkgxbs "${pkg}" "${repo}"; then + die "Package ${repo}/${pkg##*/} is not consistent with xbs" fi if ! check_pkgrepos "${pkg}"; then die "Package ${repo}/${pkg##*/} already exists in another repository" diff --git a/test/lib/common.inc b/test/lib/common.inc index fbdd30b..396f12f 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -19,6 +19,10 @@ arches() ( printf '%s\n' "${ARCHES[*]}" ) +arch_svn() { + /usr/bin/svn "$@" +} + signpkg() { if [[ -r '/etc/makepkg.conf' ]]; then source '/etc/makepkg.conf' @@ -120,6 +124,15 @@ setUp() { arch_svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done + mkdir -p "${TMP}/home/.config/libretools" + export XDG_CONFIG_HOME="${TMP}/home/.config" + printf '%s\n' \ + 'SVNURL=foo' \ + "SVNREPO=\"${TMP}/svn-packages-copy\"" \ + "ARCHES=($(arches))" \ + > $XDG_CONFIG_HOME/libretools/xbs-abs.conf + printf '%s\n' 'BUILDSYSTEM=abs' > $XDG_CONFIG_HOME/xbs.conf + cat < "$(dirname ${BASH_SOURCE[0]})/../../config.local" FTP_BASE="${TMP}/ftp" SVNREPO="${TMP}/svn-packages-copy" @@ -155,7 +168,7 @@ releasePackage() { local pkgname pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null - archrelease ${repo}-${arch} >/dev/null 2>&1 + xbs release ${repo} ${arch} >/dev/null 2>&1 pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) pkgname=($(. PKGBUILD; echo ${pkgname[@]})) popd >/dev/null diff --git a/testing2x b/testing2x index 369857f..a76e972 100755 --- a/testing2x +++ b/testing2x @@ -20,12 +20,10 @@ declare -A pkgs for pkgbase in $*; do if [ ! -d "${WORKDIR}/${pkgbase}" ]; then - arch_svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null - found_source=false for pkgarch in ${ARCHES[@]} 'any'; do - svnrepo_from="${WORKDIR}/${pkgbase}/${TESTING_REPO}-${pkgarch}" - if [ -r "${svnrepo_from}/PKGBUILD" ]; then + xbsrepo_from="$(xbs releasepath ${pkgbase} ${TESTING_REPO} ${pkgarch})" + if [ -r "${xbsrepo_from}/PKGBUILD" ]; then found_source=true break fi @@ -34,8 +32,8 @@ for pkgbase in $*; do found_target=false for pkgarch in ${ARCHES[@]} 'any'; do for repo in ${STABLE_REPOS[@]}; do - svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" - if [ -r "${svnrepo_to}/PKGBUILD" ]; then + xbsrepo_to="$(xbs releasepath ${pkgbase} ${repo} ${pkgarch})" + if [ -r "${xbsrepo_to}/PKGBUILD" ]; then found_target=true pkgs[${repo}]+="${pkgbase} " break 2 -- cgit v1.2.3 From 761b56dcffedc6e484d38c74267810e491add951 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 16 Jun 2013 22:06:51 -0600 Subject: touch up --- cron-jobs/sourceballs | 2 +- db-functions | 6 +++--- db-remove | 10 +++++----- testing2x | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9572a30..cd70916 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -95,7 +95,7 @@ for repo in ${PKGREPOS[@]}; do echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" else - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") cat "${WORKDIR}/${pkgbase}.log" >> "${WORKDIR}/makepkg-fail.log" fi popd >/dev/null diff --git a/db-functions b/db-functions index c98b26f..dabb22b 100644 --- a/db-functions +++ b/db-functions @@ -369,8 +369,8 @@ check_pkgrepos() { #usage: chk_license ${license[@]}" chk_license() { local l - for l in ${@}; do - in_array ${l} ${ALLOWED_LICENSES[@]} && return 0 + for l in "${@}"; do + in_array "${l}" "${ALLOWED_LICENSES[@]}" && return 0 done return 1 @@ -382,7 +382,7 @@ check_repo_permission() { [ ${#PKGREPOS[@]} -eq 0 ] && return 1 [ -z "${PKGPOOL}" ] && return 1 - in_array "${repo}" ${PKGREPOS[@]} || return 1 + in_array "${repo}" "${PKGREPOS[@]}" || return 1 [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 diff --git a/db-remove b/db-remove index e632c47..ccab5ea 100755 --- a/db-remove +++ b/db-remove @@ -17,12 +17,12 @@ if ! check_repo_permission $repo; then fi if [ "$arch" == "any" ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do +for tarch in "${tarches[@]}"; do repo_lock $repo $tarch || exit 1 done @@ -37,11 +37,11 @@ for pkgbase in ${pkgbases[@]}; do warning "$pkgbase not found in $repo-$arch" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" - remove_pkgs[${#remove_pkgs[*]}]=$pkgbase + remove_pkgs+=($pkgbase) fi done -for tarch in ${tarches[@]}; do - arch_repo_remove "${repo}" "${tarch}" ${remove_pkgs[@]} +for tarch in "${tarches[@]}"; do + arch_repo_remove "${repo}" "${tarch}" "${remove_pkgs[@]}" repo_unlock $repo $tarch done diff --git a/testing2x b/testing2x index a76e972..d93c5f8 100755 --- a/testing2x +++ b/testing2x @@ -8,7 +8,7 @@ if [ $# -lt 1 ]; then exit 1 fi -# Lock everything to reduce possibility of interfering task between the different repo-updates +# Lock everything to reduce possibility of interfering task between the different repo-updates script_lock for repo in ${TESTING_REPO} ${STABLE_REPOS[@]}; do for pkgarch in ${ARCHES[@]}; do -- cgit v1.2.3 From d744cae51bf5ea3e8c01d70ddbec564ad66b036c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 18 Jun 2013 00:06:38 -0600 Subject: use /etc/makepkg.conf for sourceballs --- config | 4 +- cron-jobs/makepkg.conf | 121 ------------------------------------------------- cron-jobs/sourceballs | 2 +- 3 files changed, 3 insertions(+), 124 deletions(-) delete mode 100644 cron-jobs/makepkg.conf diff --git a/config b/config index e1e705b..34869ab 100644 --- a/config +++ b/config @@ -25,8 +25,8 @@ TMPDIR="/var/tmp" ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" -SRCEXT=".src.tar.gz" +PKGEXT="$(. $(librelib conf); load_files makepkg; echo "$PKGEXT")" +SRCEXT="$(. $(librelib conf); load_files makepkg; echo "$SRCEXT")" # Allowed licenses: get sourceballs only for licenses in this array ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') diff --git a/cron-jobs/makepkg.conf b/cron-jobs/makepkg.conf deleted file mode 100644 index 603edba..0000000 --- a/cron-jobs/makepkg.conf +++ /dev/null @@ -1,121 +0,0 @@ -# -# /etc/makepkg.conf -# - -######################################################################### -# SOURCE ACQUISITION -######################################################################### -# -#-- The download utilities that makepkg should use to acquire sources -# Format: 'protocol::agent' -DLAGENTS=('ftp::/usr/bin/curl -sS -fC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' - 'http::/usr/bin/curl -sS -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'https::/usr/bin/curl -sS -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'rsync::/usr/bin/rsync -q --no-motd -z %u %o' - 'scp::/usr/bin/scp -q -C %u %o') - -# Other common tools: -# /usr/bin/snarf -# /usr/bin/lftpget -c -# /usr/bin/wget - -######################################################################### -# ARCHITECTURE, COMPILE FLAGS -######################################################################### -# -CARCH="x86_64" -CHOST="x86_64-unknown-linux-gnu" - -#-- Compiler and Linker Flags -# -march (or -mcpu) builds exclusively for an architecture -# -mtune optimizes for an architecture, but builds for whole processor family -CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2" -CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2" -LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,--hash-style=gnu" -#-- Make Flags: change this for DistCC/SMP systems -#MAKEFLAGS="-j2" - -######################################################################### -# BUILD ENVIRONMENT -######################################################################### -# -# Defaults: BUILDENV=(fakeroot !distcc color !ccache check !sign) -# A negated environment option will do the opposite of the comments below. -# -#-- fakeroot: Allow building packages as a non-root user -#-- distcc: Use the Distributed C/C++/ObjC compiler -#-- color: Colorize output messages -#-- ccache: Use ccache to cache compilation -#-- check: Run the check() function if present in the PKGBUILD -#-- sign: Generate PGP signature file -# -BUILDENV=(fakeroot !distcc color !ccache check !sign) -# -#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, -#-- specify a space-delimited list of hosts running in the DistCC cluster. -#DISTCC_HOSTS="" -# -#-- Specify a directory for package building. -#BUILDDIR=/tmp/makepkg - -######################################################################### -# GLOBAL PACKAGE OPTIONS -# These are default values for the options=() settings -######################################################################### -# -# Default: OPTIONS=(strip docs libtool emptydirs zipman purge !upx) -# A negated option will do the opposite of the comments below. -# -#-- strip: Strip symbols from binaries/libraries -#-- docs: Save doc directories specified by DOC_DIRS -#-- libtool: Leave libtool (.la) files in packages -#-- emptydirs: Leave empty directories in packages -#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip -#-- purge: Remove files specified by PURGE_TARGETS -#-- upx: Compress binary executable files using UPX -# -OPTIONS=(strip docs libtool emptydirs zipman purge !upx) - -#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 -INTEGRITY_CHECK=(md5) -#-- Options to be used when stripping binaries. See `man strip' for details. -STRIP_BINARIES="--strip-all" -#-- Options to be used when stripping shared libraries. See `man strip' for details. -STRIP_SHARED="--strip-unneeded" -#-- Options to be used when stripping static libraries. See `man strip' for details. -STRIP_STATIC="--strip-debug" -#-- Manual (man and info) directories to compress (if zipman is specified) -MAN_DIRS=({usr{,/local}{,/share},opt/*}/{man,info}) -#-- Doc directories to remove (if !docs is specified) -DOC_DIRS=(usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}) -#-- Files to be removed from all packages (if purge is specified) -PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) - -######################################################################### -# PACKAGE OUTPUT -######################################################################### -# -# Default: put built package and cached source in build directory -# -#-- Destination: specify a fixed directory where all packages will be placed -#PKGDEST=/home/packages -#-- Source cache: specify a fixed directory where source files will be cached -#SRCDEST=/home/sources -#-- Source packages: specify a fixed directory where all src packages will be placed -#SRCPKGDEST=/home/srcpackages -#-- Packager: name/email of the person or organization building packages -#PACKAGER="John Doe " -#-- Specify a key to use for package signing -#GPGKEY="" - -######################################################################### -# EXTENSION DEFAULTS -######################################################################### -# -# WARNING: Do NOT modify these variables unless you know what you are -# doing. -# -PKGEXT='.pkg.tar.xz' -SRCEXT='.src.tar.gz' - -# vim: set ft=sh ts=2 sw=2 et: diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index cd70916..34de2a6 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -88,7 +88,7 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch --skippgpcheck --config "${dirname}/makepkg.conf" >"${WORKDIR}/${pkgbase}.log" 2>&1 + SRCPKGDEST=. makepkg --nocolor --allsource --ignorearch --skippgpcheck >"${WORKDIR}/${pkgbase}.log" 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch -- cgit v1.2.3 From 7034eed1f7ceec379b34f8dd782fb986c7165ad4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 29 Jun 2013 00:45:43 -0600 Subject: use the get_full_version from libremessages (actually, common.sh) It works properly when split packages have different versions. --- db-functions | 20 +++----------------- db-move | 10 +++++----- test/lib/common.inc | 4 ++-- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/db-functions b/db-functions index dabb22b..f5e2b3d 100644 --- a/db-functions +++ b/db-functions @@ -19,27 +19,13 @@ restore_umask () { WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") LOCKS=() -# Used: plain, msg, msg2, warning, error, in_array -# Overwritten: get_full_version, -# cleanup, abort, die +# Used: plain, msg, msg2, warning, error, in_array, get_full_version +# Overwritten: cleanup, abort, die # Ignored: stat_busy, stat_done, # setup_workdir, trab_abort, trap_exit, # lock, slock, lock_close . $(librelib common) -## -# usage : get_full_version( $epoch, $pkgver, $pkgrel ) -# return : full version spec, including epoch (if necessary), pkgver, pkgrel -## -get_full_version() { - if [[ $1 -eq 0 ]]; then - # zero epoch case, don't include it in version - echo $2-$3 - else - echo $1:$2-$3 - fi -} - script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then @@ -301,7 +287,7 @@ check_pkgxbs() { in_array "${repo}" ${PKGREPOS[@]} || return 1 - local xbsver="$(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel}) )" + local xbsver="$(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; get_full_version "${_pkgname}")" [ "${xbsver}" == "${_pkgver}" ] || return 1 local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo ${pkgname[@]})) diff --git a/db-move b/db-move index 045f33c..6e8796d 100755 --- a/db-move +++ b/db-move @@ -34,10 +34,6 @@ for pkgbase in ${args[@]:2}; do die "Could not read pkgname" fi - pkgver=$(. "${xbsrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - if [ -z "${pkgver}" ]; then - die "Could not read pkgver" - fi if [ "${pkgarch}" == 'any' ]; then tarches=(${ARCHES[@]}) @@ -46,6 +42,10 @@ for pkgbase in ${args[@]:2}; do fi for pkgname in ${pkgnames[@]}; do + pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version ${pkgname}) + if [ -z "${pkgver}" ]; then + die "Could not read pkgver" + fi for tarch in ${tarches[@]}; do getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null done @@ -73,9 +73,9 @@ for pkgbase in ${args[@]:2}; do tarches=("${pkgarch}") fi pkgnames=($(. "${dir_to}/PKGBUILD"; echo ${pkgname[@]})) - pkgver=$(. "${dir_to}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) for pkgname in ${pkgnames[@]}; do + pkgver=$(. "${dir_to}/PKGBUILD"; get_full_version ${pkgname}) for tarch in ${tarches[@]}; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) pkgfile="${pkgpath##*/}" diff --git a/test/lib/common.inc b/test/lib/common.inc index 396f12f..2f308dd 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -54,7 +54,7 @@ oneTimeSetUp() { pushd $d >/dev/null pkgname=($(. PKGBUILD; echo ${pkgname[@]})) pkgarch=($(. PKGBUILD; echo ${arch[@]})) - pkgversion=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgversion=$(. PKGBUILD; get_full_version) build=true for a in ${pkgarch[@]}; do @@ -169,7 +169,7 @@ releasePackage() { pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null xbs release ${repo} ${arch} >/dev/null 2>&1 - pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgver=$(. PKGBUILD; get_full_version) pkgname=($(. PKGBUILD; echo ${pkgname[@]})) popd >/dev/null cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ -- cgit v1.2.3 From ac6ebd38626a052049c17f166dd2ebee81a0168a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 28 Jul 2013 00:17:44 -0600 Subject: twiddle with config --- config | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config b/config index 34869ab..179a2b9 100644 --- a/config +++ b/config @@ -1,3 +1,5 @@ +#/bin/bash # as a hint to editors + FTP_BASE="/srv/ftp" PKGREPOS=() PKGPOOL='' @@ -20,13 +22,13 @@ REQUIRE_SIGNATURE=true LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$HOME/staging" +[ -z "${STAGING:-}" ] && STAGING="$HOME/staging" TMPDIR="/var/tmp" ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT="$(. $(librelib conf); load_files makepkg; echo "$PKGEXT")" -SRCEXT="$(. $(librelib conf); load_files makepkg; echo "$SRCEXT")" +PKGEXT=".pkg.tar.?z" +SRCEXT=".src.tar.gz" # Allowed licenses: get sourceballs only for licenses in this array ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') -- cgit v1.2.3 From 12aab28511d12c58864b0077262957985fa7f2a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 15 Aug 2013 12:00:21 -0300 Subject: Verbose mode is optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolás Reynolds --- db-sync | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/db-sync b/db-sync index d86d348..81dee24 100755 --- a/db-sync +++ b/db-sync @@ -12,15 +12,18 @@ # * Sync repo => repo # TODO -# * verbose mode # * make a tarball of files used for forensics # * get files db +# Run as `V=true db-sync` to get verbose output +VERBOSE=${V} +${VERBOSE} && extra="-v" + # Returns contents of a repo get_repos() { mkdir -p ${TMPDIR}/$0.$$.cache # Exclude everything but db files - rsync -vmrtlH --no-p --include="*/" \ + rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ --exclude="*" \ @@ -93,7 +96,7 @@ init() { # Sync excluding everything but whitelist # We delete here for cleanup - rsync -vrtlH \ + rsync ${extra} -rtlH \ --delete-after \ --delete-excluded \ --delay-updates \ @@ -106,7 +109,7 @@ init() { whitelists+=(/tmp/${_repo}-${_arch}.whitelist) msg "Putting databases back in place" - rsync -vrtlH \ + rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ @@ -128,7 +131,7 @@ init() { # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too for PKGPOOL in ${PKGPOOLS[@]}; do - rsync -vrtlH \ + rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ @@ -146,7 +149,7 @@ init() { # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too for SRCPOOL in ${SRCPOOLS[@]}; do - rsync -vrtlH \ + rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ -- cgit v1.2.3 From e0a2005c59d15ef462d2bf26404a5b2d14266f0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Fri, 16 Aug 2013 18:50:49 -0300 Subject: add coherence repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 7601527..42842de 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'coherence') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3 From 21a1c0b1c74e877209a0a467d2b9fade996cac92 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 19 Aug 2013 16:04:18 -0700 Subject: get-repos: parabolaweb wants *.files.tar.gz files, not *.db.tar.gz --- get-repos | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/get-repos b/get-repos index 4397d54..98a39bc 100755 --- a/get-repos +++ b/get-repos @@ -26,7 +26,7 @@ DBLIST=() # Repos for _repo in ${PKGREPOS[@]}; do for _arch in ${ARCHES[@]}; do - DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${DBEXT}") + DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${FILESEXT}") done done @@ -36,7 +36,7 @@ done # MAYBE run scripts on get-repos.d/ which should return db urls for _repo in ${RMTREPOS}; do _arch=i586 - DBLIST+=("http://www.connochaetos.org/os/${_arch}/${_repo}/${_repo}${DBEXT}") + DBLIST+=("http://www.connochaetos.org/os/${_arch}/${_repo}/${_repo}${FILESEXT}") done # Get them all @@ -52,7 +52,7 @@ wget --directory-prefix=${TMPDIR} \ arch_re="$(echo "(${ARCHES[@]} i586)" | tr ' ' '|')" msg "Adding to parabolaweb" -find "${TMPDIR}" -iname "*${DBEXT}" | while read _db; do +find "${TMPDIR}" -iname "*${FILESEXT}" | while read _db; do _arch=$(echo "${_db}" | egrep -o "${arch_re}") if [ -z "${_arch}" ]; then -- cgit v1.2.3 From 2bb8c19bdff6b45dbb2ce09d7f45ca42d56bcd4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Tue, 20 Aug 2013 00:10:01 -0300 Subject: adding nonprism repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 42842de..5bff50a 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'coherence') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'coherence' 'nonprism') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3 From f8051126147e57292eeaa730f2c499aea1b1a16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Tue, 20 Aug 2013 00:20:13 -0300 Subject: fixing PKGEXT variable --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 5bff50a..0eb851a 100644 --- a/config +++ b/config @@ -45,7 +45,7 @@ OURARCHES=(mips64el) ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" +PKGEXT=".pkg.tar.xz" SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" -- cgit v1.2.3 From 48995748c99dee29ca473463ffbf8b443b6c287a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Tue, 20 Aug 2013 00:28:04 -0300 Subject: fixing db-move for any package architecture --- db-move | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db-move b/db-move index 010d941..73c4517 100755 --- a/db-move +++ b/db-move @@ -47,7 +47,9 @@ for pkgbase in ${args[@]:2}; do for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null + getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null \ + || \ + getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-any${PKGEXT} >/dev/null done done continue 2 -- cgit v1.2.3 From e95d49c335723a15031083287d889c56a531ef1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Tue, 20 Aug 2013 00:34:28 -0300 Subject: revert back db-move --- db-move | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/db-move b/db-move index 73c4517..010d941 100755 --- a/db-move +++ b/db-move @@ -47,9 +47,7 @@ for pkgbase in ${args[@]:2}; do for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null \ - || \ - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-any${PKGEXT} >/dev/null + getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null done done continue 2 -- cgit v1.2.3 From 2a4a1c4015a4cc48f1613b7fd360148378775445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Wed, 21 Aug 2013 17:48:40 -0300 Subject: rename unfree to nonfree --- db-check-nonfree | 2 +- yf/PKGBUILD | 2 +- yf/your-freedom.install | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/db-check-nonfree b/db-check-nonfree index ab6491d..ecad3b9 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -33,7 +33,7 @@ for repo in ${ARCHREPOS[@]}; do fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Unfree: ${cleanpkgs[@]}" + msg2 "Nonfree: ${cleanpkgs[@]}" arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} fi done diff --git a/yf/PKGBUILD b/yf/PKGBUILD index cc6f07b..e737482 100644 --- a/yf/PKGBUILD +++ b/yf/PKGBUILD @@ -2,7 +2,7 @@ pkgname=your-freedom pkgver=$(LC_ALL=C date -u +%Y%m%d) pkgrel=1 -pkgdesc="This package conflicts with every unfree package known to date." +pkgdesc="This package conflicts with every nonfree package known to date." arch=('any') url="https://parabolagnulinux.org" license=('GPL') diff --git a/yf/your-freedom.install b/yf/your-freedom.install index 49ae045..731a575 100644 --- a/yf/your-freedom.install +++ b/yf/your-freedom.install @@ -3,18 +3,18 @@ pre_install() { cat < Date: Thu, 22 Aug 2013 12:54:10 -0300 Subject: remove coherence repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 0eb851a..2c1fa43 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'coherence' 'nonprism') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3 From 3e085d0d529a749386f826ecb2dd189df55b4833 Mon Sep 17 00:00:00 2001 From: Icarious Date: Mon, 26 Aug 2013 05:26:11 +0530 Subject: removed unnecessary code --- cron-jobs/sourceballs2 | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 5e228fc..5644268 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -60,32 +60,3 @@ for repo in ${PKGREPOS[@]}; do popd >/dev/null done # end repos - -# Cleanup old source packages -cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" -old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) - -if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old source packages..." - ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" - if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" - touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" - fi - done -fi - -old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) -if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old source packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" - ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" - done -fi - -script_unlock - -- cgit v1.2.3 From f185efe31d518b08b45e07288a4239eddaac11b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Mon, 26 Aug 2013 16:36:21 -0300 Subject: remove deprecated repos --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 2c1fa43..f390bea 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'nonprism') +PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3 From 97ed9a26bd00c8a4c03a331c076e9f2a9c7ac57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Mon, 26 Aug 2013 16:47:10 -0300 Subject: remove fauno repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index f390bea..0882c86 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') +USERREPOS=('~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -- cgit v1.2.3 From c2513af7d1a4ab9180cbcfd34135cd04add73c0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 28 Aug 2013 18:18:53 -0300 Subject: remove mtjm repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 0882c86..6c652a7 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') +USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -- cgit v1.2.3 From 93229619764ff3c59fd097c72fb7890a7f542668 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Oct 2013 22:32:27 +0000 Subject: config: only set staging if it is not set from the environment --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 6c652a7..03a80bd 100644 --- a/config +++ b/config @@ -38,7 +38,7 @@ REQUIRE_SIGNATURE=true LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$FTP_BASE/staging" +[ -n "${STAGING:-}" ] || STAGING="$FTP_BASE/staging" TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) OURARCHES=(mips64el) -- cgit v1.2.3 From ce88f47d19cb11ebcc02565156deddc6b48df38c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 14 Oct 2013 18:41:54 -0400 Subject: db-sync: also generate ${repo}.files databases for imported repos --- db-sync | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/db-sync b/db-sync index 81dee24..684f7e6 100755 --- a/db-sync +++ b/db-sync @@ -26,6 +26,8 @@ get_repos() { rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ + --include="*.db" \ + --include="*${FILESEXT}" \ --exclude="*" \ --delete-after \ rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache @@ -46,9 +48,7 @@ get_blacklist() { # repo # arch get_repo_file() { -# shopt -s nullglob - - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}" + echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" } # Process the databases and get the libre packages @@ -69,21 +69,27 @@ init() { for _arch in ${ARCHARCHES[@]}; do msg "Processing ${_repo}-${_arch}" - repo_file=$(get_repo_file ${_repo} ${_arch}) + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - if [ ! -f "${repo_file}" ]; then - warning "${repo_file} doesn't exist, skipping this repo-arch" + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" continue fi # Remove blacklisted packages and count them # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages: $( - LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ - grep "\-> Removing" 2>/dev/null| wc -l)" + msg2 "Removing blacklisted packages from .db database..." + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" + msg2 "Removing blacklisted packages from .files database..." + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" # Get db contents - db=($(get_repo_content ${repo_file})) + db=($(get_repo_content ${db_file})) msg2 "Process clean db for syncing..." -- cgit v1.2.3 From 45bfa89440e93cd722e1c14dcd58b60a52808fa4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 14 Oct 2013 18:43:20 -0400 Subject: oops --- db-sync | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-sync b/db-sync index 684f7e6..e8481d6 100755 --- a/db-sync +++ b/db-sync @@ -26,7 +26,7 @@ get_repos() { rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ - --include="*.db" \ + --include="*.files" \ --include="*${FILESEXT}" \ --exclude="*" \ --delete-after \ -- cgit v1.2.3 From fbce7db101feaba0d69bf185b54270c1aa6d65ab Mon Sep 17 00:00:00 2001 From: "Jan Alexander Steffens (heftig)" Date: Tue, 19 Mar 2013 19:53:38 +0100 Subject: sourceballs: fix ACL issues on nymeria ACLs from the WORKDIR were used, leading to problems as these weren't world-readable. Fix follows ftpdir-cleanup. Signed-off-by: Jan Alexander Steffens (heftig) Signed-off-by: Pierre Schmitz --- cron-jobs/ftpdir-cleanup | 9 --------- cron-jobs/sourceballs | 4 ++-- db-functions | 9 +++++++++ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index e1294bd..8f5cb3c 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -3,15 +3,6 @@ . "$(dirname $0)/../config" . "$(dirname $0)/../db-functions" -# just like mv -f, but we touch the file and then copy the content so -# default ACLs in the target dir will be applied -mv_acl() { - rm -f "$2" - touch "$2" - cat "$1" >"$2" || return 1 - rm -f "$1" -} - clean_pkg() { local pkg local target diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6f75ccc..be66e99 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -90,7 +90,7 @@ for repo in ${PKGREPOS[@]}; do pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch --skippgpcheck --config "${dirname}/makepkg.conf" >"${WORKDIR}/${pkgbase}.log" 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -127,7 +127,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/db-functions b/db-functions index b3a4293..4a9a42a 100644 --- a/db-functions +++ b/db-functions @@ -15,6 +15,15 @@ restore_umask () { umask $UMASK >/dev/null } +# just like mv -f, but we touch the file and then copy the content so +# default ACLs in the target dir will be applied +mv_acl() { + rm -f "$2" + touch "$2" + cat "$1" >"$2" || return 1 + rm -f "$1" +} + # set up general environment WORKDIR=$(mktemp -d "${TMPDIR}/${0##*/}.XXXXXXXXXX") if [ -n "${SVNUSER}" ]; then -- cgit v1.2.3 From 0b43e8cdee1ee46ea79e4d089136e76e767b4d5b Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Wed, 23 Jan 2013 23:03:35 +0100 Subject: Add lastupdate file This allows for faster checking if an update might be needed and helps to let reporead run when something changed. Signed-off-by: Florian Pritz --- db-functions | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/db-functions b/db-functions index 4a9a42a..bb49894 100644 --- a/db-functions +++ b/db-functions @@ -32,6 +32,7 @@ if [ -n "${SVNUSER}" ]; then setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" fi LOCKS=() +REPO_MODIFIED=0 # check if messages are to be printed using color unset ALL_OFF BOLD BLUE GREEN RED YELLOW @@ -142,6 +143,11 @@ cleanup() { script_unlock fi rm -rf "$WORKDIR" + + if (( REPO_MODIFIED )); then + date +%s > "${FTP_BASE}/lastupdate" + fi + [ "$1" ] && exit $1 } @@ -499,6 +505,8 @@ arch_repo_add() { || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 } arch_repo_remove() { @@ -517,6 +525,8 @@ arch_repo_remove() { /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 } arch_svn() { -- cgit v1.2.3 From 280421c67b7042ea94fc2ec94f04278cb72e2a06 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 30 Nov 2013 03:17:18 +0000 Subject: Change the default WEB_DIR to reflect the new server --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 03a80bd..3d15c1b 100644 --- a/config +++ b/config @@ -52,4 +52,4 @@ MAKEPKGCONF="~/.makepkg.conf" BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" # parabolaweb root -WEB_DIR=/srv/http/web +WEB_DIR=/srv/http/parabolagnulinux.org/web -- cgit v1.2.3 From 8a693d7f3a9324de445b5a151c1f46194405f0eb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 30 Nov 2013 03:17:57 +0000 Subject: Drop remote (connos) repo support --- config | 1 - get-repos | 9 --------- 2 files changed, 10 deletions(-) diff --git a/config b/config index 3d15c1b..e323d00 100644 --- a/config +++ b/config @@ -12,7 +12,6 @@ USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jor # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/get-repos b/get-repos index 98a39bc..bfc08ff 100755 --- a/get-repos +++ b/get-repos @@ -30,15 +30,6 @@ for _repo in ${PKGREPOS[@]}; do done done -# Remote repos -# TODO don't hardcode the remote architecture -# TODO don't hardcode the remote url -# MAYBE run scripts on get-repos.d/ which should return db urls -for _repo in ${RMTREPOS}; do - _arch=i586 - DBLIST+=("http://www.connochaetos.org/os/${_arch}/${_repo}/${_repo}${FILESEXT}") -done - # Get them all msg "Retrieving ${#DBLIST[@]} databases" wget --directory-prefix=${TMPDIR} \ -- cgit v1.2.3 From a88674528c5df02f03bf30449982932b6cc378dc Mon Sep 17 00:00:00 2001 From: Drtan Samos Date: Tue, 17 Dec 2013 10:24:33 +0100 Subject: Added a user repository for me in order to test packages before they come to the offical repositories. --- config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config b/config index e323d00..9baf816 100644 --- a/config +++ b/config @@ -8,7 +8,8 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') +USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' +'~coadde' '~drtan') # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -- cgit v1.2.3 From ea21f1388681d4fed43201bda54efc9d63aadc88 Mon Sep 17 00:00:00 2001 From: Drtan Samos Date: Tue, 17 Dec 2013 10:36:31 +0100 Subject: Removes the extra new line added by nano for unknown reasons. --- config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config b/config index 9baf816..8b8117f 100644 --- a/config +++ b/config @@ -8,8 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' -'~coadde' '~drtan') +USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -- cgit v1.2.3 From 77fa9fc45569325a508893e16bcbb404fe12ffbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:19:45 -0300 Subject: Original abslibre sync script used on the dead repo --- abslibre | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/abslibre b/abslibre index 9485986..e49d627 100755 --- a/abslibre +++ b/abslibre @@ -30,18 +30,21 @@ function get_blacklist() { return 1 } +# Prevent using an empty blacklist + [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 + printf "[OK]\n" } function sync_abs_libre() { # Clone ABSLibre git repo - if [ -d /tmp/abslibre/.git ]; then - pushd /tmp/abslibre >/dev/null 2>&1 + if [ -d /var/tmp/abslibre/.git ]; then + pushd /var/tmp/abslibre >/dev/null 2>&1 git pull popd >/dev/null 2>&1 else - git clone /srv/git/repositories/abslibre.git /tmp/abslibre + git clone /srv/git/abslibre.git /var/tmp/abslibre fi # Sync from ABS and then sync from ABSLibre @@ -51,7 +54,7 @@ function sync_abs_libre() { ${ABSROOT} \ ${ABSLIBRE} \ && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { printf "[FAILED]\n" return 1 } @@ -62,7 +65,21 @@ function sync_abs_libre() { sync_pre_mips64el() { pushd /home/parabola/abslibre-pre-mips64el >/dev/null - rsync ${SYNCARGS} --exclude=.git* ${ABSLIBRE}/x86_64/ /home/parabola/abslibre-pre-mips64el/ && git add * && git commit -m "$(date)" + rsync ${SYNCARGS} \ + --exclude=.git* \ + --exclude=community-staging \ + --exclude=community-testing \ + --exclude=gnome-unstable \ + --exclude=kde-unstable \ + --exclude=multilib \ + --exclude=multilib-testing \ + --exclude=multilib-staging \ + --exclude=staging \ + --exclude=testing \ + ${ABSLIBRE}/x86_64/ \ + /home/parabola/abslibre-pre-mips64el/ && \ + git add . && \ + git commit -m "$(date)" -a } # Create .abs.tar.gz tarballs @@ -88,6 +105,9 @@ get_blacklist || exit 1 sync_abs_libre || exit 1 # This is being done at repo server now sync_pre_mips64el || exit 1 -create_tarballs || exit 1 +#create_tarballs || exit 1 + +echo "Exclusion list used" +cat ${BLFILE} exit 0 -- cgit v1.2.3 From 187b0eb8d443d181dc533b1d1d0fce9aa38b4c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:25:11 -0300 Subject: Remove abs dependency --- abslibre | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/abslibre b/abslibre index e49d627..70e4be3 100755 --- a/abslibre +++ b/abslibre @@ -8,7 +8,13 @@ BLACKLIST='http://repo.parabolagnulinux.org/docs/blacklist.txt' SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' BLFILE=/tmp/blacklist.txt -. /etc/abs.conf +# Variables from abs.conf +ABSROOT="/var/abs/" +# DON'T CHANGE. WE NEED IT FOR ABSLIBRE +SYNCSERVER="rsync.archlinux.org" +ARCH="i686" +MIRRORLIST="/etc/pacman.d/mirrorlist" +REPOS=(core extra community testing community-testing !staging !community-staging) # Steps # * Sync abs -- cgit v1.2.3 From 2e756472a8c75eafc290bc8823013f3ac4335a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:26:13 -0300 Subject: Use the blacklist from git --- abslibre | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abslibre b/abslibre index 70e4be3..fd2e900 100755 --- a/abslibre +++ b/abslibre @@ -4,7 +4,7 @@ ABSLIBRE=/var/abslibre ABSGIT=/srv/git/repositories/abslibre.git # Remote # ABSGIT=http://projects.parabolagnulinux.org/abslibre.git -BLACKLIST='http://repo.parabolagnulinux.org/docs/blacklist.txt' +BLACKLIST='https://projects.parabolagnulinux.org/blacklist.git/plain/blacklist.txt' SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' BLFILE=/tmp/blacklist.txt -- cgit v1.2.3 From 863af4a1c724e9ef13d5011c84f7b6ea8e940c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:33:47 -0300 Subject: Adapt to the current setup --- abslibre | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/abslibre b/abslibre index fd2e900..bb7fd12 100755 --- a/abslibre +++ b/abslibre @@ -68,8 +68,11 @@ function sync_abs_libre() { printf "[OK]\n" } +# This part is very hacky and particular to the current setup :P sync_pre_mips64el() { - pushd /home/parabola/abslibre-pre-mips64el >/dev/null + pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null + + sudo -u fauno sh -c " rsync ${SYNCARGS} \ --exclude=.git* \ @@ -83,9 +86,11 @@ sync_pre_mips64el() { --exclude=staging \ --exclude=testing \ ${ABSLIBRE}/x86_64/ \ - /home/parabola/abslibre-pre-mips64el/ && \ + /home/fauno/Repos/abslibre-pre-mips64el/ && \ git add . && \ - git commit -m "$(date)" -a + git commit -m \"$(date)\" -a + git push origin master + " } # Create .abs.tar.gz tarballs -- cgit v1.2.3 From ffe9ada1c43d5fd779863c4be2782f7b690a5fbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:41:02 -0300 Subject: Changed to /srv --- abslibre | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/abslibre b/abslibre index bb7fd12..fa777ac 100755 --- a/abslibre +++ b/abslibre @@ -1,6 +1,6 @@ #!/bin/bash -ABSLIBRE=/var/abslibre +ABSLIBRE=/srv/abslibre ABSGIT=/srv/git/repositories/abslibre.git # Remote # ABSGIT=http://projects.parabolagnulinux.org/abslibre.git @@ -9,7 +9,7 @@ SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' BLFILE=/tmp/blacklist.txt # Variables from abs.conf -ABSROOT="/var/abs/" +ABSROOT="/srv/abs/" # DON'T CHANGE. WE NEED IT FOR ABSLIBRE SYNCSERVER="rsync.archlinux.org" ARCH="i686" -- cgit v1.2.3 From 63810122211aa60b056895989984eaccf6242c58 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 8 Dec 2013 15:12:55 -0500 Subject: clarify comment on bogus shebangs --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 179a2b9..f327cd4 100644 --- a/config +++ b/config @@ -1,4 +1,4 @@ -#/bin/bash # as a hint to editors +#/bin/bash # as a hint to text editors FTP_BASE="/srv/ftp" PKGREPOS=() -- cgit v1.2.3 From c1bbe81b1e14f06ed49d9e0292ec9cd60903b104 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 14:11:53 -0500 Subject: add db-import --- db-import | 293 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ db-import.conf | 11 +++ 2 files changed, 304 insertions(+) create mode 100755 db-import create mode 100644 db-import.conf diff --git a/db-import b/db-import new file mode 100755 index 0000000..7bc822f --- /dev/null +++ b/db-import @@ -0,0 +1,293 @@ +#!/bin/bash -euE +# Imports Arch-like repos, running them through a blacklist +# License: GPLv3 + +. $(dirname $0)/config +. $(dirname $0)/db-import.conf +. $(librelib messages) +. $(librelib blacklist) + +# DBs = pacman DataBases + +# This replaces two scripts: +# - abslibre : imported ABS tree from Arch +# - db-sync : imported pacman DBs from Arch + +# The flow here is: +# 1. "${IMPORTDIR}/cache/${name}/dbs/" # Download the pacman databases +# 2. "${IMPORTDIR}/cache/${name}/abs/" # Download the ABS tree +# 3. "${IMPORTDIR}/clean/${name}/dbs/" # Run the pacman DBs through the blacklist +# 4. Download all the package files mentioned in "clean/${name/dbs/" +# 5. "${STAGING}-importer-${tag}" Copy all the package files we just downloaded to here +# 6. Run db-update on "${STAGING}-importer-${tag}" + +SYNCARGS='-mrtvlH --no-motd --no-p --no-o --no-g' + +main() { + blacklist-update + + local importStr + for importStr in "${imports[@]}"; do + local importAry=($importStr) + local name=${importAry[0]} + local pkgmirror=${importAry[1]} + local absmirror=${importAry[2]} + local tags=("${importAry[@]:3}") + + msg "Fetching remote package source: %s" "$name" + fetch_dbs "$name" "$pkgmirror" + fetch_abs "$name" "$absmirror" "${tags[@]}" + msg "Filtering blacklisted packages from remote package source: %s" "$name" + clean_dbs "$name" "${tags[@]}" + msg "Publishing changes from remote package source: %s" "$name" + publish "$name" "${tags[@]}" + done + return $r +} + +fetch_dbs() { + local name=$1 + local pkgmirror=$2 + + msg2 'Synchronizing package databases...' + + mkdir -p -- "${IMPORTDIR}/cache/${name}/dbs" + # Grab just the .db files from $pkgmirror + rsync $SYNCARGS --delete-after \ + --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --exclude="*" \ + "rsync://${pkgmirror}/" "${IMPORTDIR}/cache/${name}/dbs" +} + +fetch_abs() { + local name=$1 + local absmirror=$2 + local tags=("${@:3}") + + local _HOME=$HOME + + local absroot + + # Sync the ABS tree from $absmirror + local arch + for arch in $(list_arches "${tags[@]}"); do + msg2 'Synchronizing %s ABS tree...' "$arch" + + absroot="${IMPORTDIR}/cache/${name}/abs/${arch}" + mkdir -p -- "$absroot" + + # Configure `abs` for this mirror + export HOME="${IMPORTDIR}/homes/${name}/${arch}" + mkdir -p -- "$HOME" + { + printf "ABSROOT='%s'\n" "$absroot" + printf "SYNCSERVER='%s'\n" "$absmirror" + printf "ARCH='%s'\n" "$arch" + printf 'REPOS=(\n' + list_repos "$arch" "${tags[@]}" + printf ')\n' + } > ~/.abs.conf + + # Run `abs` + abs + done + + export HOME=$_HOME +} + +clean_dbs() { + local name=$1 + local tags=("${@:2}") + + rm -rf -- "${IMPORTDIR}/clean/$name" + + local tag + for tag in "${tags[@]}"; do + msg2 'Creating clean version of %s package database...' "$tag" + + local cache="${IMPORTDIR}/cache/$name/dbs/$(db_file "$tag")" + local clean="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + install -Dm644 "$cache" "$clean" + + repo-remove "$clean" $(blacklist-cat|blacklist-get-pkg) + done +} + +fetch_pkgs() { + local name=$1 + local tags=("${@:2}") + + local repo arch dbfile whitelist + + local tag + for tag in "${tags[@]}"; do + msg2 'Syncronizing package files for %s...' "$tag" + repo=${tag%-*} + arch=${tag##*-} + + dbfile="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + whitelist="${IMPORTDIR}/clean/$name/dbs/$tag.whitelist" + + list_pkgs "$dbfile" > "$whitelist" + + # fetch the architecture-specific packages + rsync $SYNCARGS --delete-after --delete-excluded \ + --delay-updates \ + --include-from=<(sed "s|\$|-$arch.tar.?z|" "$whitelist") \ + --exclude='*' \ + "rsync://${pkgmirror}/$(db_dir "$tag")/" \ + "${IMPORTDIR}/clean/${name}/pkgs/${tag}/" + + # fetch the architecture-independent packages + rsync $SYNCARGS --delete-after --delete-excluded \ + --delay-updates \ + --include-from=<(sed "s|\$|-any.tar.?z|" "$whitelist") \ + --exclude='*' \ + "rsync://${pkgmirror}/$(db_dir "$tag")/" \ + "${IMPORTDIR}/clean/${name}/pkgs/${repo}-any/" + done +} + +publish() { + local name=$1 + local tags=("${@:2}") + + local tag + for tag in "${tags[@]}"; do + msg2 'Publishing changes to %s...' "$tag" + publish_tag "$name" "$tag" + done +} + +publish_tag() { + local name=$1 + local tag=$2 + + local repo=${tag%-*} + local arch=${tag##*-} + local dir="${IMPORTDIR}/clean/${name}/pkgs/${tag}" + + local found + local error=false + local files=() + + local pkgid pkgarch + for pkgid in $(list_added_pkgs "$name" "$tag"); do + found=false + + for pkgarch in "${arch}" any; do + file="${dir}/${pkgid}-${arch}".pkg.tar.?z + if ! $found && [[ -r $file ]]; then + files+=("$file") + found=true + fi + done + + if ! $found; then + error 'Could not find package file for %s' "$pkgid" + error=true + fi + done + + if $error; then + error 'Quitting...' + return 1 + fi + + local _STAGING=$STAGING + export STAGING="$STAGING-importer-$tag" + + mkdir -p -- "${STAGING}/${repo}" + cp -a -- "${files[@]}" "${STAGING}/${repo}/" + db-update + + # XXX: db-remove wants pkgbase, not pkgname + db-remove "$repo" "$arch" $(list_removed_pkgs "$name" "$tag") + + STAGING=$_STAGING +} + +################################################################################ + +# Usage: list_arches repo-arch... +# Returns a list of the architectures mentioned in a list of "repo-arch" pairs. +list_arches() { + local tags=("$@") + printf '%s\n' "${tags[@]##*-}" | sort -u +} + +# Usage: list_repos arch repo-arch... +# Returns a list of all the repositories mentioned for a given architecture in a +# list of "repo-arch" pairs. +list_repos() { + local arch=$1 + local tags=("${@:2}") + printf '%s\n' "${tags[@]}" | sed -n "s/-$arch\$//p" +} + +# Usage: db_dir repo-arch +db_dir() { + local tag=$1 + local repo=${tag%-*} + local arch=${tag##*-} + echo "${repo}/os/${arch}" +} + +# Usage; db_file repo-arch +db_file() { + local tag=$1 + local repo=${tag%-*} + local arch=${tag##*-} + echo "${repo}/os/${arch}/${repo}${DBEXT}" +} + +# Usage: list_pkgs dbfile +# Prints "$pkgname-$(get_full_version "$pkgname")" for every package in $dbfile +list_pkgs() { + local dbfile=$1 + bsdtar tf "$dbfile" | cut -d/ -f1 +} + +# Usage: list_pkgs | sep_ver +# Separates the pkgname from the version (replaces the '-' with ' ') for the +# list provided on stdin. +sep_ver() { + sed -r 's/-([^-]*-[^-]*)$/ \1/' +} + +# Usage: list_removed_pkgs importsrc repo-arch +# Prints "$pkgname-$(get_full_version "$pkgname")" for every removed package. +list_removed_pkgs() { + local name=$1 + local tag=$2 + + local old="${FTP_BASE}/$(db_file "$tag")" + local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + + # make a list of: + # pkgname oldver[ newver] + # It will include removed or updated packages (changed packages) + join -a1 \ + <(list_pkgs "$old"|sep_ver|sort) \ + <(list_pkgs "$new"|sep_ver|sort) + | grep -v ' .* ' # remove updated packages + | sed 's/ /-/' # re-combine the pkgname and version +} + +# Usage: list_added_pkgs importsrc repo-arch +# slightly a misnomer; added and updated +# Prints "$pkgname-$(get_full_version "$pkgname")" for every added or updated +# package. +list_added_pkgs() { + local name=$1 + local tag=$2 + + local old="${FTP_BASE}/$(db_file "$tag")" + local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" + + comm -13 <(list_pkgs "$old") <(list_pkgs "$new") +} + +main "$@" diff --git a/db-import.conf b/db-import.conf new file mode 100644 index 0000000..ef96568 --- /dev/null +++ b/db-import.conf @@ -0,0 +1,11 @@ +#/bin/bash # as a hint to text editors + +IMPORTDIR=/srv/repo/import + +_archrepos=( + {core,extra,testing}-{i686,x86_64} + community{,-testing}-{i686,x86_64} + multilib{,-testing}-x86_64 +) +# name pkgmirror absmirror repo-arch... +imports=("archlinux archlinux.c3sl.ufpr.br/archlinux rsync.archlinux.org ${_archrepos[*]}") -- cgit v1.2.3 From 563cf021c96446aa15a41c1f532067d13d880b41 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 8 Dec 2013 15:13:23 -0500 Subject: db-import: detect best package mirror --- db-import.conf | 5 ++++- db-pick-mirror | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100755 db-pick-mirror diff --git a/db-import.conf b/db-import.conf index ef96568..1ba50ba 100644 --- a/db-import.conf +++ b/db-import.conf @@ -7,5 +7,8 @@ _archrepos=( community{,-testing}-{i686,x86_64} multilib{,-testing}-x86_64 ) + +_archpkgmirror=$(db-pick-mirror rsync https://www.archlinux.org/mirrors/status/json/) + # name pkgmirror absmirror repo-arch... -imports=("archlinux archlinux.c3sl.ufpr.br/archlinux rsync.archlinux.org ${_archrepos[*]}") +imports=("archlinux ${_archpkgmirror} rsync.archlinux.org ${_archrepos[*]}") diff --git a/db-pick-mirror b/db-pick-mirror new file mode 100755 index 0000000..7cbc032 --- /dev/null +++ b/db-pick-mirror @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby + +require 'json' +require 'rest_client' + +protocol = ARGV[0] +jsonurl = ARGV[1] + +data = JSON::parse(RestClient.get(jsonurl)) + +if data["version"] != 3 + print "Data format version != 3" + exit 1 +end + +urls = data["urls"] +rsync_urls = urls.select{|a| a["protocol"]==protocol} + +# By score ( (delay+speed)/completion ) +#best = rsync_urls.sort{|a,b| (a["score"] || Float::INFINITY) <=> (b["score"] || Float::INFINITY) }.first +# By delay/completion ; hopefully this gives us a tier 1 mirror +best = rsync_urls.sort{|a,b| a["delay"]/a["completion_pct"] <=> b["delay"]/b["completion_pct"] }.first + +puts best["url"] -- cgit v1.2.3 From e6294556d3197b7d87f7659355d0e189fad613d6 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 8 Dec 2013 17:33:25 -0500 Subject: `readlink -e` all "$0"s --- cron-jobs/ftpdir-cleanup | 4 ++-- cron-jobs/sourceballs | 2 +- cron-jobs/update-web-db | 4 ++-- db-move | 4 ++-- db-remove | 4 ++-- db-repo-add | 4 ++-- db-repo-remove | 4 ++-- db-update | 4 ++-- testing2x | 4 ++-- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 8f5cb3c..e42a1a8 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../config" -. "$(dirname $0)/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" clean_pkg() { local pkg diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 103898d..2362024 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname $(readlink -e $0))" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../config" . "${dirname}/../db-functions" pushd "${WORKDIR}" >/dev/null diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 195d1fc..713e75e 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../config" -. "$(dirname $0)/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" # setup paths SPATH="/srv/http/archweb" diff --git a/db-move b/db-move index 6e8796d..1b34404 100755 --- a/db-move +++ b/db-move @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-remove b/db-remove index ccab5ea..993574f 100755 --- a/db-remove +++ b/db-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-repo-add b/db-repo-add index 5d5b653..a6355a1 100755 --- a/db-repo-add +++ b/db-repo-add @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-repo-remove b/db-repo-remove index 2a693f4..7077d62 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-update b/db-update index 28fcabe..9c6a56a 100755 --- a/db-update +++ b/db-update @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is no longer supported" diff --git a/testing2x b/testing2x index d93c5f8..14610c2 100755 --- a/testing2x +++ b/testing2x @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 1 ]; then msg "usage: ${0##*/} ..." -- cgit v1.2.3 From bcfc6c88a6e23ac15f08b37d55cf1f5378b4120e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 14:19:10 -0500 Subject: db-import: run `readlink -e` on $0 --- db-import | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-import b/db-import index 7bc822f..79dcfd5 100755 --- a/db-import +++ b/db-import @@ -2,10 +2,10 @@ # Imports Arch-like repos, running them through a blacklist # License: GPLv3 -. $(dirname $0)/config -. $(dirname $0)/db-import.conf -. $(librelib messages) -. $(librelib blacklist) +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-import.conf" +. "$(librelib messages)" +. "$(librelib blacklist)" # DBs = pacman DataBases -- cgit v1.2.3 From 8b6653a2c6c7e55f2c312735a31e0f42d7260bbd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 9 Dec 2013 03:20:17 -0500 Subject: clean up db-import --- db-import | 45 ++++++++++++++++++++------------------------- db-import.conf | 6 ++++-- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/db-import b/db-import index 79dcfd5..a8a073d 100755 --- a/db-import +++ b/db-import @@ -1,9 +1,10 @@ -#!/bin/bash -euE +#!/bin/bash +set -euE # Imports Arch-like repos, running them through a blacklist # License: GPLv3 -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-import.conf" +. "$(dirname "$(readlink -e "$0")")/config" # for: FTP_BASE DBEXT +. "$(dirname "$(readlink -e "$0")")/db-import.conf" # for: IMPORTDIR IMPORTS . "$(librelib messages)" . "$(librelib blacklist)" @@ -17,17 +18,18 @@ # 1. "${IMPORTDIR}/cache/${name}/dbs/" # Download the pacman databases # 2. "${IMPORTDIR}/cache/${name}/abs/" # Download the ABS tree # 3. "${IMPORTDIR}/clean/${name}/dbs/" # Run the pacman DBs through the blacklist -# 4. Download all the package files mentioned in "clean/${name/dbs/" -# 5. "${STAGING}-importer-${tag}" Copy all the package files we just downloaded to here -# 6. Run db-update on "${STAGING}-importer-${tag}" +# 4. "${IMPORTDIR}/clean/${name}/pkgs/" # Download the pkg files mentioned in "clean/${name}/dbs/" +# 5. "${IMPORTDIR}/staging/${tag}" # Copy all the package files we just downloaded to here +# 6. Run `db-update on` with STAGING="${IMPORTDIR}/staging/${tag}" +# generic arguments to pass to rsync, borrowed from `abs` SYNCARGS='-mrtvlH --no-motd --no-p --no-o --no-g' main() { blacklist-update local importStr - for importStr in "${imports[@]}"; do + for importStr in "${IMPORTS[@]}"; do local importAry=($importStr) local name=${importAry[0]} local pkgmirror=${importAry[1]} @@ -39,6 +41,7 @@ main() { fetch_abs "$name" "$absmirror" "${tags[@]}" msg "Filtering blacklisted packages from remote package source: %s" "$name" clean_dbs "$name" "${tags[@]}" + fetch_pkgs "$name" "${tags[@]}" msg "Publishing changes from remote package source: %s" "$name" publish "$name" "${tags[@]}" done @@ -66,8 +69,7 @@ fetch_abs() { local absmirror=$2 local tags=("${@:3}") - local _HOME=$HOME - + local fake_home local absroot # Sync the ABS tree from $absmirror @@ -79,8 +81,8 @@ fetch_abs() { mkdir -p -- "$absroot" # Configure `abs` for this mirror - export HOME="${IMPORTDIR}/homes/${name}/${arch}" - mkdir -p -- "$HOME" + fake_home="${IMPORTDIR}/homes/${name}/${arch}" + mkdir -p -- "$fake_home" { printf "ABSROOT='%s'\n" "$absroot" printf "SYNCSERVER='%s'\n" "$absmirror" @@ -88,13 +90,11 @@ fetch_abs() { printf 'REPOS=(\n' list_repos "$arch" "${tags[@]}" printf ')\n' - } > ~/.abs.conf + } > "${fake_home}/.abs.conf" # Run `abs` - abs + HOME=$fake_home abs done - - export HOME=$_HOME } clean_dbs() { @@ -111,7 +111,7 @@ clean_dbs() { local clean="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" install -Dm644 "$cache" "$clean" - repo-remove "$clean" $(blacklist-cat|blacklist-get-pkg) + blacklist-cat | blacklist-get-pkg | xargs -d '\n' repo-remove "$clean" done } @@ -196,17 +196,12 @@ publish_tag() { return 1 fi - local _STAGING=$STAGING - export STAGING="$STAGING-importer-$tag" - - mkdir -p -- "${STAGING}/${repo}" - cp -a -- "${files[@]}" "${STAGING}/${repo}/" - db-update + mkdir -p -- "${IMPORTDIR}/staging/${tag}/${repo}" + cp -al -- "${files[@]}" "${IMPORTDIR}/staging/${tag}/${repo}/" + STAGING="${IMPORTDIR}/staging/${tag}" db-update # XXX: db-remove wants pkgbase, not pkgname - db-remove "$repo" "$arch" $(list_removed_pkgs "$name" "$tag") - - STAGING=$_STAGING + list_removed_pkgs "$name" "$tag" | xargs -d '\n' db-remove "$repo" "$arch" } ################################################################################ diff --git a/db-import.conf b/db-import.conf index 1ba50ba..5a6049d 100644 --- a/db-import.conf +++ b/db-import.conf @@ -8,7 +8,9 @@ _archrepos=( multilib{,-testing}-x86_64 ) -_archpkgmirror=$(db-pick-mirror rsync https://www.archlinux.org/mirrors/status/json/) +_archpkgmirror=$(db-pick-mirror rsync https://www.archlinux.org/mirrors/status/tier/1/json/) # name pkgmirror absmirror repo-arch... -imports=("archlinux ${_archpkgmirror} rsync.archlinux.org ${_archrepos[*]}") +IMPORTS=("archlinux ${_archpkgmirror} rsync.archlinux.org ${_archrepos[*]}") + +unset _archrepos _archpkgmirror -- cgit v1.2.3 From d42f5c6ab02a861ef548ae12fe403f61e2b02e4f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 13:54:34 -0500 Subject: db-import.conf: add {gnome,kde}-unstable staging repos --- db-import.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/db-import.conf b/db-import.conf index 5a6049d..2728077 100644 --- a/db-import.conf +++ b/db-import.conf @@ -4,6 +4,7 @@ IMPORTDIR=/srv/repo/import _archrepos=( {core,extra,testing}-{i686,x86_64} + {gnome,kde}-unstable-{i686,x86_64} community{,-testing}-{i686,x86_64} multilib{,-testing}-x86_64 ) -- cgit v1.2.3 From f65c477e42f46a3e22b5fbc5da790f9561483770 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 14:50:37 -0500 Subject: clean up --- TODO | 2 +- config.orig | 54 --------------------------------- cron-jobs/ftpdir-cleanup | 7 +++++ cron-jobs/sourceballs | 8 ++--- db-functions | 19 +++++------- db-update | 9 +++--- git-pbs | 44 --------------------------- libremessages | 77 ------------------------------------------------ 8 files changed, 23 insertions(+), 197 deletions(-) delete mode 100644 config.orig delete mode 100755 git-pbs delete mode 100755 libremessages diff --git a/TODO b/TODO index 3219b1c..9dd4b52 100644 --- a/TODO +++ b/TODO @@ -7,4 +7,4 @@ * Fix db-move - - Make it use abslibre \ No newline at end of file + - Make it use abslibre diff --git a/config.orig b/config.orig deleted file mode 100644 index a32f82f..0000000 --- a/config.orig +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -<<<<<<< HEAD -FTP_BASE="/srv/http/repo/public" -ARCH_BASE="/srv/http/repo/public" -SVNREPO="/srv/http/repo/abslibre" -======= -FTP_BASE="/srv/http/repo/public/temprepo" -ARCH_BASE="/srv/http/repo/public/temprepo" -SVNREPO="/var/abs" ->>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e - -# Repos from Arch -ARCHREPOS=('core' 'testing') #'extra' 'community' 'testing' 'multilib') -# Official Parabola repos -OURREPOS=('libre' 'libre-testing') -# User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') -# Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') -PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' - -CLEANUP_DESTDIR="$FTP_BASE/old/packages" -CLEANUP_DRYRUN=true -# Time in days to keep moved packages -CLEANUP_KEEP=30 - -SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" -SOURCE_CLEANUP_DRYRUN=true -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=30 - -REQUIRE_SIGNATURE=true - -LOCK_DELAY=10 -LOCK_TIMEOUT=300 - -STAGING="$FTP_BASE/staging" -TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) -OURARCHES=(mips64el) -ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) -DBEXT=".db.tar.gz" -FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" -SRCEXT=".src.tar.gz" - -<<<<<<< HEAD -MAKEPKGCONF="~/.makepkg.conf" -======= -MAKEPKGCONF="/etc/makepkg.conf" ->>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index a2823d4..e42a1a8 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -22,6 +22,13 @@ clean_pkg() { fi } +script_lock + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done ${CLEANUP_DRYRUN} && warning 'dry run mode is active' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 91bc3d6..1542499 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -59,7 +59,7 @@ for repo in ${PKGREPOS[@]}; do pkgarch=${pkginfo[2]} pkglicense=(${pkginfo[@]:3}) - # Should this packages be skipped? + # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi diff --git a/db-functions b/db-functions index 072f43d..0b59a53 100644 --- a/db-functions +++ b/db-functions @@ -135,7 +135,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(stat -c %U $LOCKDIR)" + _owner="$(/usr/bin/stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -250,9 +250,6 @@ getpkgfile() { } getpkgfiles() { -# Ignore anything that doesn't glob to PKGEXT - shopt -s nullglob - local f if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then error 'Duplicate packages found!' @@ -270,8 +267,6 @@ getpkgfiles() { done echo ${@} - - shopt -u nullglob } check_pkgfile() { @@ -330,7 +325,7 @@ check_splitpkgs() { for pkgfile in ${pkgfiles[@]}; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" - msg2 "Checking $_pkgbase" + msg2 "Checking %s" "$_pkgbase" local _pkgname="$(getpkgname ${pkgfile})" local _pkgarch="$(getpkgarch ${pkgfile})" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" @@ -411,7 +406,7 @@ set_repo_permission() { local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ -w "${dbfile}" ]; then - local group=$(stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" @@ -428,9 +423,9 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -449,9 +444,9 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" - repo-remove -q "${filesfile}" ${pkgs[@]} \ + /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" diff --git a/db-update b/db-update index 74d3f45..ca18ca3 100755 --- a/db-update +++ b/db-update @@ -52,10 +52,9 @@ for repo in ${repos[@]}; do die "Package ${repo}/${pkg##*/} already exists in another repository" fi done - # This is fucking obnoxious -# if ! check_splitpkgs ${repo} ${pkgs[@]}; then -# die "Missing split packages for ${repo}" -# fi + if ! check_splitpkgs ${repo} ${pkgs[@]}; then + die "Missing split packages for ${repo}" + fi else die "Could not read ${STAGING}" fi @@ -82,7 +81,7 @@ for repo in ${repos[@]}; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi - add_pkgs[${#add_pkgs[*]}]=${pkgfile} + add_pkgs[${#add_pkgs[*]}]=${pkgfile} done if [ ${#add_pkgs[@]} -ge 1 ]; then arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} diff --git a/git-pbs b/git-pbs deleted file mode 100755 index b815863..0000000 --- a/git-pbs +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -_pkg=$1 - -mkdir -p $_pkg -pushd $_pkg - - -if [ ! -d .git ]; then -# Start a git repo for the package -# Add the remote origin -# Pull the package branch onto an unmodified branch - git init - git remote add arch git://projects.archlinux.org/svntogit/packages.git - -# Export the repository - touch .git/git-daemon-export-ok - -# Pass the -b flag to checkout to create the branches - extra="-b" -fi - -git checkout ${extra} upstream -git pull arch packages/$_pkg - -# Move PKGBUILD and files to the basedir -# Remove everything else from the repo -git checkout ${extra} master - -# This produces a lot of merging conflicts -git merge upstream - -# This apparently solves them -git mv trunk/* . -git rm -rf repos - -# Remove the actual files -rm -rf trunk repos - -# Commit everything -git commit -a -m "Converted to PBS" - -# Return to the repo -popd >/dev/null diff --git a/libremessages b/libremessages deleted file mode 100755 index 9fbbc2b..0000000 --- a/libremessages +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2006-2010 Pacman Development Team -# Copyright (c) 2002-2006 by Judd Vinet -# Copyright (c) 2005 by Aurelien Foret -# Copyright (c) 2006 by Miklos Vajna -# Copyright (c) 2005 by Christian Hamar -# Copyright (c) 2006 by Alex Smith -# Copyright (c) 2006 by Andras Voroskoi -# Copyright (c) 2011 by Joshua Haase -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# gettext initialization -export TEXTDOMAIN='libretools' -export TEXTDOMAINDIR='/usr/share/locale' - -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW - -if tput setaf 0 &>/dev/null; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" - PURPLE="${ALL_OFF}$(tput setaf 5)" -else - ALL_OFF="\033[1;0m" - BOLD="\033[1;1m" - BLUE="${BOLD}\033[1;34m" - GREEN="${BOLD}\033[1;32m" - RED="${BOLD}\033[1;31m" - YELLOW="${BOLD}\033[1;33m" - PURPLE="${BOLD}\033[1;30;40m" -fi - -stdnull() { - local action=$1; - eval "${action} >/dev/null 2>&1" -} - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -- cgit v1.2.3 From e210b136790f740a6eb42913b30f6d0c8557c81d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 14:53:28 -0500 Subject: get rid of $ARCH_BASE --- config | 1 - createrepos | 4 ++-- cron-jobs/sourceballs2 | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/config b/config index 496b3ef..3b7c13a 100644 --- a/config +++ b/config @@ -1,6 +1,5 @@ #/bin/bash # as a hint to text editors FTP_BASE="/srv/http/repo/public" -ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch diff --git a/createrepos b/createrepos index 4ee057b..eec26e0 100755 --- a/createrepos +++ b/createrepos @@ -3,6 +3,6 @@ source $(dirname $0)/config -mkdir -p ${FTP_BASE}/{${PKGPOOL},${SRCPOOL}} ${ARCH_BASE} ${CLEANUP_DESTDIR} ${SOURCE_CLEANUP_DESTDIR} ${STAGING} +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR} ${STAGING}" -$(dirname $0)/create-repo ${PKGREPOS[@]} +$(dirname $0)/create-repo "${PKGREPOS[@]}" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 5644268..5b14d07 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -17,7 +17,7 @@ script_lock renice +10 -p $$ > /dev/null # Create a list of all available source package file names -find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" pushd "${SVNREPO}" >/dev/null -- cgit v1.2.3 From 7a49d26565b1bdbd73d946e3dca3dae7b1073572 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 15:30:23 -0500 Subject: db-sync: use tab indent --- db-sync | 258 ++++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 129 insertions(+), 129 deletions(-) diff --git a/db-sync b/db-sync index e8481d6..39e72b9 100755 --- a/db-sync +++ b/db-sync @@ -22,157 +22,157 @@ ${VERBOSE} && extra="-v" # Returns contents of a repo get_repos() { mkdir -p ${TMPDIR}/$0.$$.cache -# Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache + # Exclude everything but db files + rsync ${extra} -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache } get_repo_content() { -# Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u + # Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u } # Prints blacklisted packages get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" + cut -d ':' -f 1 "${BLACKLIST_FILE}" } # repo # arch get_repo_file() { - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" + echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" } # Process the databases and get the libre packages init() { -# Get the blacklisted packages - blacklist=($(get_blacklist)) -# Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - -# Sync the repos databases - get_repos - -# Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - -# Remove blacklisted packages and count them -# TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from .db database..." - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" - msg2 "Removing blacklisted packages from .files database..." - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" - -# Get db contents - db=($(get_repo_content ${db_file})) - - msg2 "Process clean db for syncing..." - -# Create a whitelist, add * wildcard to end -# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if -# we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - -# Sync excluding everything but whitelist -# We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Cleanup - unset db - done - done - - - msg "Syncing package pool" -# Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - -# Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from .db database..." + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" + msg2 "Removing blacklisted packages from .files database..." + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" + + # Get db contents + db=($(get_repo_content ${db_file})) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync ${extra} -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done -# Cleanup - unset blacklist whitelists _arch _repo repo_file + # Cleanup + unset blacklist whitelists _arch _repo repo_file } trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } -- cgit v1.2.3 From 9b69d5863bdf398b78ce46661dabb9abec09385e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 15:31:33 -0500 Subject: db-sync: use mktemp/trap --- db-sync | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db-sync b/db-sync index 39e72b9..0acfbc1 100755 --- a/db-sync +++ b/db-sync @@ -19,9 +19,11 @@ VERBOSE=${V} ${VERBOSE} && extra="-v" +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/$0.$$.cache # Exclude everything but db files rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ @@ -30,7 +32,7 @@ get_repos() { --include="*${FILESEXT}" \ --exclude="*" \ --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache + rsync://${mirror}/${mirrorpath}/ "$WORKDIR" } get_repo_content() { @@ -48,7 +50,7 @@ get_blacklist() { # repo # arch get_repo_file() { - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" + echo "${WORKDIR}/${1}/os/${2}/${1}" } # Process the databases and get the libre packages @@ -118,7 +120,7 @@ init() { rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ - ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ + "${WORKDIR}/${_repo}/os/${_arch}/" \ ${FTP_BASE}/${_repo}/os/${_arch}/ # Cleanup @@ -188,5 +190,3 @@ trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init - -rm -r ${TMPDIR}/$0.$$.cache -- cgit v1.2.3 From 6a093f1dc6bd5398115544dd229b520f9432e122 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 15:49:06 -0500 Subject: use `readlink -e` on $0 --- any-to-ours | 6 +++--- create-repo | 6 +++--- createrepos | 4 ++-- cron-jobs/integrity-check | 2 +- cron-jobs/repo-sanity-check | 4 ++-- cron-jobs/sourceballs2 | 2 +- cron-jobs/update-abs-tarballs | 2 +- db-check-nonfree | 4 ++-- db-cleanup | 6 +++--- db-list-unsigned-packages | 6 +++--- db-sync | 6 +++--- get-repos | 6 +++--- migrate-repo | 2 +- mkrepo | 4 ++-- repo-restore-to-normal | 4 ++-- testing2x | 2 +- yf-update | 12 ++++++------ 17 files changed, 39 insertions(+), 39 deletions(-) diff --git a/any-to-ours b/any-to-ours index a1d6686..31fea5f 100755 --- a/any-to-ours +++ b/any-to-ours @@ -7,9 +7,9 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/create-repo b/create-repo index 58842c3..f77fa24 100755 --- a/create-repo +++ b/create-repo @@ -1,8 +1,8 @@ #!/bin/bash # Creates repository structure -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -eq 0 ]; then msg "Usage: $0 repo1 [repo2 ... repoX]" @@ -21,4 +21,4 @@ for _repo in $@; do done done -msg "Don't forget to add them to the PKGREPOS array on $(dirname $0)/config" +msg "Don't forget to add them to the PKGREPOS array on %s/config" "$(dirname "$(readlink -e "$0")")" diff --git a/createrepos b/createrepos index eec26e0..cd17e4e 100755 --- a/createrepos +++ b/createrepos @@ -1,8 +1,8 @@ #!/bin/bash # Creates the repo structure defined in config -source $(dirname $0)/config +source "$(dirname "$(readlink -e "$0")")/config" mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR} ${STAGING}" -$(dirname $0)/create-repo "${PKGREPOS[@]}" +"$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index f6c26cf..86a8f1d 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname $0)" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../config" . "${dirname}/../db-functions" diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 1ba90a6..2aa7892 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" # Traverse all repos for _repo in ${PKGREPOS[@]}; do diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 5b14d07..bbe227d 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -4,7 +4,7 @@ # Makepkg --allsource every package # Remove the old sourceballs -dirname="$(dirname $(readlink -e $0))" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../db-functions" . "${dirname}/../config" . "${MAKEPKGCONF}" diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs index 824ac34..901cc4b 100755 --- a/cron-jobs/update-abs-tarballs +++ b/cron-jobs/update-abs-tarballs @@ -1,6 +1,6 @@ #!/bin/bash -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../config" rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ diff --git a/db-check-nonfree b/db-check-nonfree index ecad3b9..ba5f5aa 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -ge 1 ]; then warning "Calling $(basename $0) with a specific repository is not supported" diff --git a/db-cleanup b/db-cleanup index 904c06e..c6e4f1c 100755 --- a/db-cleanup +++ b/db-cleanup @@ -15,9 +15,9 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 3b5a5bd..4e90d42 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -20,8 +20,8 @@ set -e # unsigned packages available for architecture $1 and specified for # architecture $2 (usually $1 or any, default is to list all). -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 1 ]; then msg "usage: $(basename $0) " @@ -34,5 +34,5 @@ shift for repo in ${PKGREPOS[@]} do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - [ -f "$db" ] && "$(dirname $0)/db-list-unsigned-packages.py" "$repo" "$@" < "$db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" done diff --git a/db-sync b/db-sync index 0acfbc1..e4b6966 100755 --- a/db-sync +++ b/db-sync @@ -178,9 +178,9 @@ trap_exit() { } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/get-repos b/get-repos index bfc08ff..40d7205 100755 --- a/get-repos +++ b/get-repos @@ -9,9 +9,9 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/migrate-repo b/migrate-repo index 2e44adb..751d5bd 100755 --- a/migrate-repo +++ b/migrate-repo @@ -1,6 +1,6 @@ #!/bin/bash -source $(dirname $0)/config +source "$(dirname "$(readlink -e "$0")")/config" #dryrun="--dry-run" diff --git a/mkrepo b/mkrepo index 5f704cc..10d014b 100755 --- a/mkrepo +++ b/mkrepo @@ -3,8 +3,8 @@ # License: GPLv3+ # Description: A script to quickly create new [repos] -source $(dirname $0)/config -source $(dirname $0)/local_config +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" # TODO it would be simpler to expand arrays to {element1,element2,etc} for repo in $@; do diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 9463731..3636920 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore PKGREPOS=(community) diff --git a/testing2x b/testing2x index 14610c2..6646179 100755 --- a/testing2x +++ b/testing2x @@ -52,7 +52,7 @@ for repo in ${STABLE_REPOS[@]}; do repo_unlock ${repo} ${pkgarch} done if [ -n "${pkgs[${repo}]}" ]; then - "$(dirname $0)/db-move" ${TESTING_REPO} "${repo}" ${pkgs[${repo}]} + "$(dirname "$(readlink -e "$0")")/db-move" ${TESTING_REPO} "${repo}" ${pkgs[${repo}]} fi done diff --git a/yf-update b/yf-update index 9c2131e..ee5d3eb 100755 --- a/yf-update +++ b/yf-update @@ -1,17 +1,17 @@ #!/bin/bash -source $(dirname $0)/local_config -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) -last_bl_mtime=$(cat $(dirname $0)/yftime) +last_bl_mtime=$(< "$(dirname "$(readlink -e "$0")")/yftime") if [ $blacklist_mtime -gt $last_bl_mtime ]; then - pushd $(dirname $0)/yf + pushd "$(dirname "$(readlink -e "$0")")/yf" makepkg -f find . -name "*${PKGEXT}" -exec mv {} ${STAGING}/libre \; popd - echo ${blacklist_mtime} > $(dirname $0)/yftime + echo ${blacklist_mtime} > "$(dirname "$(readlink -e "$0")")/yftime" msg2 "built and staged" else msg2 "nothing to do" -- cgit v1.2.3 From a9d3b75c29cc82346c17787456fed009392f2029 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 16:04:28 -0500 Subject: local_config.example: remove unused variables, update comments --- local_config.example | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/local_config.example b/local_config.example index 2280cc2..daa6874 100644 --- a/local_config.example +++ b/local_config.example @@ -1,26 +1,15 @@ -# Mirror options +#/bin/bash # as a hint to text editors +_paraboladir=~/parabolagnulinux.org + +# db-sync mirror="mirrors.eu.kernel.org" mirrorpath="mirrors/archlinux" -# Directories: they should end without / -paraboladir=~/parabolagnulinux.org -tempdir=~/tmp -archdb=${tempdir}/db -docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo -licenses_dir=${docs_dir}/pending_licenses -# End Directories - -# Files -logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log -rsout_file=${tempdir}/rsout -rsync_not_needed=${tempdir}/rsync_not_needed - -rsync_blacklist=${docs_dir}/rsyncBlacklist +# mkrepo +repodir=${_paraboladir}/repo -blacklist=${docs_dir}/blacklist.txt -whitelist=${docs_dir}/whitelist.txt +# yf-update +blacklist=${_paraboladir}/docs/blacklist.txt +whitelist=${_paraboladir}/docs/whitelist.txt -# Rsync commands -rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " -rsync_update_command="rsync -rptgoL --exclude='*.abs.tar.*' --no-motd " +unset _paraboladir -- cgit v1.2.3 From 877eaef7357c5ca171e15de16ffce055da68af2f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 16:06:19 -0500 Subject: remove unnecessary loadings of local_config --- any-to-ours | 1 - db-cleanup | 1 - get-repos | 1 - 3 files changed, 3 deletions(-) diff --git a/any-to-ours b/any-to-ours index 31fea5f..a1697c7 100755 --- a/any-to-ours +++ b/any-to-ours @@ -8,7 +8,6 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg diff --git a/db-cleanup b/db-cleanup index c6e4f1c..364f400 100755 --- a/db-cleanup +++ b/db-cleanup @@ -16,7 +16,6 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg diff --git a/get-repos b/get-repos index 40d7205..2921dd0 100755 --- a/get-repos +++ b/get-repos @@ -10,7 +10,6 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg -- cgit v1.2.3 From baf990333197d42fc25e3220b2bbc97ea03d88a4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 17:09:29 -0500 Subject: get-repos: clean up temp directory handling - rename $TMPDIR to $WORKDIR - respect environmental $TMPDIR - use a trap to clean up,instead of a command at the end --- get-repos | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/get-repos b/get-repos index 2921dd0..5096433 100755 --- a/get-repos +++ b/get-repos @@ -19,7 +19,9 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -TMPDIR="$(mktemp -d /tmp/$(basename $0).XXXX)" +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + DBLIST=() # Repos @@ -31,7 +33,7 @@ done # Get them all msg "Retrieving ${#DBLIST[@]} databases" -wget --directory-prefix=${TMPDIR} \ +wget --directory-prefix=${WORKDIR} \ --no-verbose \ --force-directories \ --no-host-directories \ @@ -42,7 +44,7 @@ wget --directory-prefix=${TMPDIR} \ arch_re="$(echo "(${ARCHES[@]} i586)" | tr ' ' '|')" msg "Adding to parabolaweb" -find "${TMPDIR}" -iname "*${FILESEXT}" | while read _db; do +find "${WORKDIR}" -iname "*${FILESEXT}" | while read _db; do _arch=$(echo "${_db}" | egrep -o "${arch_re}") if [ -z "${_arch}" ]; then @@ -52,7 +54,3 @@ find "${TMPDIR}" -iname "*${FILESEXT}" | while read _db; do "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" || true done - -rm -r ${TMPDIR} - -exit $? -- cgit v1.2.3 From 12507975408257ff24f1f367b8b3b842fa779f1f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 17:12:16 -0500 Subject: use ${0##*/} instead of basename in "usage:" text --- create-repo | 2 +- db-check-nonfree | 2 +- db-list-unsigned-packages | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/create-repo b/create-repo index f77fa24..24b890d 100755 --- a/create-repo +++ b/create-repo @@ -5,7 +5,7 @@ . "$(dirname "$(readlink -e "$0")")/config" if [ $# -eq 0 ]; then - msg "Usage: $0 repo1 [repo2 ... repoX]" + msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" exit 1 fi diff --git a/db-check-nonfree b/db-check-nonfree index ba5f5aa..661daa6 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/config" if [ $# -ge 1 ]; then - warning "Calling $(basename $0) with a specific repository is not supported" + warning "Calling ${0##*/} with a specific repository is not supported" exit 1 fi diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 4e90d42..985d1c0 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -24,7 +24,7 @@ set -e . "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 1 ]; then - msg "usage: $(basename $0) " + msg "usage: ${0##*/} " exit 1 fi -- cgit v1.2.3 From bb69db69c7d319491f67be2db77889b70aa634f0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 1 Jan 2014 21:54:55 -0500 Subject: rm repo-remove -- in Oct 2012 fauno removed repo-add, but left this The customized repo-{add,remove} did license extraction for packages. They were based on the versions from pacman 3.5.0 Here is a diff between the stock versions from 3.5.0, and the modified version that we had: --- repo-add.sh.in 2013-12-31 18:02:13.546351038 -0500 +++ repo-remove.in 2013-12-31 18:13:19.957948677 -0500 @@ -20,6 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +LICENSESDIR=/home/parabolavnx/licenses + # gettext initialization export TEXTDOMAIN='pacman' export TEXTDOMAINDIR='@localedir@' @@ -309,6 +311,22 @@ fi fi + # Extracts licenses to a common license dir + msg "Extracting license" + if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + # Change dir to licenses, and extract them stripping the first part of the path + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ + --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + warning "This package doesn't contain a license dir" + fi + fi + return 0 } # end db_write_entry @@ -328,6 +346,12 @@ rm -rf $pkgentry pkgentry=$(find_pkgentry $pkgname) done + + msg "Removing license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + return $notfound } # end db_remove_entry --- repo-remove | 561 ------------------------------------------------------------ 1 file changed, 561 deletions(-) delete mode 100755 repo-remove diff --git a/repo-remove b/repo-remove deleted file mode 100755 index c4bf96f..0000000 --- a/repo-remove +++ /dev/null @@ -1,561 +0,0 @@ -#!/bin/bash -# -# repo-add - add a package to a given repo database file -# repo-remove - remove a package entry from a given repo database file -# Generated from repo-add.in; do not edit by hand. -# -# Copyright (c) 2006-2008 Aaron Griffin -# Copyright (c) 2007-2008 Dan McGee -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -LICENSESDIR=/home/parabolavnx/licenses - -# gettext initialization -export TEXTDOMAIN='pacman' -export TEXTDOMAINDIR='/usr/share/locale' - -myver='3.5.0' -confdir='/home/parabolavnx/etc' - -QUIET=0 -DELTA=0 -WITHFILES=0 -REPO_DB_FILE= -LOCKFILE= -CLEAN_LOCK=0 - -# ensure we have a sane umask set -umask 0022 - -msg() { - (( QUIET )) && return - local mesg=$1; shift - printf "==> ${mesg}\n" "$@" >&1 -} - -msg2() { - (( QUIET )) && return - local mesg=$1; shift - printf " -> ${mesg}\n" "$@" >&1 -} - -warning() { - local mesg=$1; shift - printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 -} - -# print usage instructions -usage() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" - printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" - printf "$(gettext "\ -repo-add will update a package database by reading a package file.\n\ -Multiple packages to add can be specified on the command line.\n\n")" - printf "$(gettext "\ -repo-remove will update a package database by removing the package name\n\ -specified on the command line from the given repo database. Multiple\n\ -packages to remove can be specified on the command line.\n\n")" - printf "$(gettext "\ -Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ -and errors.\n\n")" - printf "$(gettext "\ -Use the -d/--delta flag to automatically generate and add a delta file\n\ -between the old entry and the new one, if the old package file is found\n\ -next to the new one.\n\n")" - printf "$(gettext "\ -Use the -f/--files flag to update a database including file entries.\n\n")" - echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" - echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" -} - -version() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "\ -Copyright (C) 2006-2008 Aaron Griffin .\n\ -Copyright (c) 2007-2008 Dan McGee .\n\n\ -This is free software; see the source for copying conditions.\n\ -There is NO WARRANTY, to the extent permitted by law.\n")" -} - -# write a list entry -# arg1 - Entry name -# arg2 - List -# arg3 - File to write to -write_list_entry() { - if [[ -n $2 ]]; then - echo "%$1%" >>$3 - echo -e $2 >>$3 - fi -} - -find_pkgentry() -{ - local pkgname=$1 - local pkgentry - for pkgentry in $tmpdir/$pkgname*; do - name=${pkgentry##*/} - if [[ ${name%-*-*} = $pkgname ]]; then - echo $pkgentry - return 0 - fi - done - return 1 -} - -# Get the package name from the delta filename -get_delta_pkgname() { - local tmp - - tmp=${1##*/} - echo ${tmp%-*-*_to*} -} - -# write a delta entry -# arg1 - path to delta file -db_write_delta() -{ - deltafile="$1" - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - error "$(gettext "No database entry for package '%s'.")" "$pkgname" - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - echo -e "%DELTAS%" >$deltas - fi - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$deltafile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$deltafile") - - oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') - newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') - - if grep -q "$oldfile.*$newfile" $deltas; then - sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup - fi - msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" - echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas - - return 0 -} # end db_write_delta - -# remove a delta entry -# arg1 - path to delta file -db_remove_delta() -{ - deltafile="$1" - filename=${deltafile##*/} - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - return 1 - fi - if grep -q "$filename" $deltas; then - sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup - msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" - return 0 - fi - - return 1 -} # end db_remove_delta - -# write an entry to the pacman database -# arg1 - path to package -db_write_entry() -{ - # blank out all variables - local pkgfile="$1" - local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ - _groups _licenses _replaces _depends _conflicts _provides _optdepends - - local OLDIFS="$IFS" - # IFS (field separator) is only the newline character - IFS=" -" - - # read info from the zipped package - local line var val - for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | - grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do - # bash awesomeness here- var is always one word, val is everything else - var=${line%% *} - val=${line#* } - declare $var="$val" - case "$var" in - group) _groups="$_groups$group\n" ;; - license) _licenses="$_licenses$license\n" ;; - replaces) _replaces="$_replaces$replaces\n" ;; - depend) _depends="$_depends$depend\n" ;; - conflict) _conflicts="$_conflicts$conflict\n" ;; - provides) _provides="$_provides$provides\n" ;; - optdepend) _optdepends="$_optdepends$optdepend\n" ;; - esac - done - - IFS=$OLDIFS - - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$pkgfile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$pkgfile") - - # ensure $pkgname and $pkgver variables were found - if [[ -z $pkgname || -z $pkgver ]]; then - error "$(gettext "Invalid package file '%s'.")" "$pkgfile" - return 1 - fi - - pushd "$tmpdir" >/dev/null - if [[ -d $pkgname-$pkgver ]]; then - warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" - else - if (( DELTA )); then - pkgentry=$(find_pkgentry $pkgname) - if [[ -n $pkgentry ]]; then - local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) - local oldfile="$(dirname $1)/$oldfilename" - fi - fi - fi - - # remove an existing entry if it exists, ignore failures - db_remove_entry "$pkgname" - - # create package directory - mkdir "$pkgname-$pkgver" - pushd "$pkgname-$pkgver" >/dev/null - - # restore an eventual deltas file - [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas - - # create desc entry - msg2 "$(gettext "Creating '%s' db entry...")" 'desc' - echo -e "%FILENAME%\n$(basename "$1")\n" >>desc - echo -e "%NAME%\n$pkgname\n" >>desc - [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc - echo -e "%VERSION%\n$pkgver\n" >>desc - [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc - write_list_entry "GROUPS" "$_groups" "desc" - [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc - [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc - - # compute checksums - msg2 "$(gettext "Computing md5 checksums...")" - echo -e "%MD5SUM%\n$md5sum\n" >>desc - - [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc - write_list_entry "LICENSE" "$_licenses" "desc" - [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc - [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc - [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc - write_list_entry "REPLACES" "$_replaces" "desc" - - # create depends entry - msg2 "$(gettext "Creating '%s' db entry...")" 'depends' - # create the file even if it will remain empty - touch "depends" - write_list_entry "DEPENDS" "$_depends" "depends" - write_list_entry "CONFLICTS" "$_conflicts" "depends" - write_list_entry "PROVIDES" "$_provides" "depends" - write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - - popd >/dev/null - popd >/dev/null - - # create files file if wanted - if (( WITHFILES )); then - msg2 "$(gettext "Creating '%s' db entry...")" 'files' - local files_path="$tmpdir/$pkgname-$pkgver/files" - echo "%FILES%" >$files_path - bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path - fi - - # create a delta file - if (( DELTA )); then - if [[ -n $oldfilename ]]; then - if [[ -f $oldfile ]]; then - delta=$(pkgdelta -q $oldfile $1) - if [[ -f $delta ]]; then - db_write_delta $delta - fi - else - warning "$(gettext "Old package file not found: %s")" "$oldfilename" - fi - fi - fi - - # Extracts licenses to a common license dir - msg "Extracting license" - if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - # Change dir to licenses, and extract them stripping the first part of the path - bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ - --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 - - if [ $? -ne 0 ]; then - warning "This package doesn't contain a license dir" - fi - fi - - return 0 -} # end db_write_entry - -# remove existing entries from the DB -# arg1 - package name -db_remove_entry() { - local pkgname=$1 - local notfound=1 - local pkgentry=$(find_pkgentry $pkgname) - while [[ -n $pkgentry ]]; do - notfound=0 - if [[ -f $pkgentry/deltas ]]; then - mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" - fi - msg2 "$(gettext "Removing existing entry '%s'...")" \ - "$(basename $pkgentry)" - rm -rf $pkgentry - pkgentry=$(find_pkgentry $pkgname) - done - - msg "Removing license" - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - return $notfound -} # end db_remove_entry - -check_repo_db() -{ - # check lock file - if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then - CLEAN_LOCK=1 - else - error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" - exit 1 - fi - - if [[ -f $REPO_DB_FILE ]]; then - # there are two situations we can have here- a DB with some entries, - # or a DB with no contents at all. - if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then - # check empty case - if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then - error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" - exit 1 - fi - fi - msg "$(gettext "Extracting database to a temporary location...")" - bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" - else - case "$cmd" in - repo-remove) - error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" - exit 1 - ;; - repo-add) - # check if the file can be created (write permission, directory existence, etc) - if ! touch "$REPO_DB_FILE"; then - error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" - exit 1 - fi - rm -f "$REPO_DB_FILE" - ;; - esac - fi -} - -add() -{ - if [[ ! -f $1 ]]; then - error "$(gettext "File '%s' not found.")" "$1" - return 1 - fi - - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Adding delta '%s'")" "$deltafile" - if ! type xdelta3 &>/dev/null; then - error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" - exit 1 - fi - if db_write_delta "$deltafile"; then - return 0 - else - return 1 - fi - fi - - pkgfile=$1 - if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then - error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" - return 1 - fi - - msg "$(gettext "Adding package '%s'")" "$pkgfile" - - db_write_entry "$pkgfile" -} - -remove() -{ - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Searching for delta '%s'...")" "$deltafile" - if db_remove_delta "$deltafile"; then - return 0 - else - error "$(gettext "Delta matching '%s' not found.")" "$deltafile" - return 1 - fi - fi - - pkgname=$1 - msg "$(gettext "Searching for package '%s'...")" "$pkgname" - - if db_remove_entry "$pkgname"; then - rm -f "$tmpdir/$pkgname.deltas" - return 0 - else - error "$(gettext "Package matching '%s' not found.")" "$pkgname" - return 1 - fi -} - -trap_exit() -{ - echo - error "$@" - exit 1 -} - -clean_up() { - local exit_code=$? - - [[ -d $tmpdir ]] && rm -rf "$tmpdir" - (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" - - exit $exit_code -} - -# PROGRAM START - -# determine whether we have gettext; make it a no-op if we do not -if ! type gettext &>/dev/null; then - gettext() { - echo "$@" - } -fi - -case "$1" in - -h|--help) usage; exit 0;; - -V|--version) version; exit 0;; -esac - -# figure out what program we are -cmd="$(basename $0)" -if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then - error "$(gettext "Invalid command name '%s' specified.")" "$cmd" - exit 1 -fi - -tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ - error "$(gettext "Cannot create temp directory for database building.")"; \ - exit 1) - -trap 'clean_up' EXIT -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR - -success=0 -# parse arguments -for arg in "$@"; do - case "$arg" in - -q|--quiet) QUIET=1;; - -d|--delta) DELTA=1;; - -f|--files) WITHFILES=1;; - *) - if [[ -z $REPO_DB_FILE ]]; then - REPO_DB_FILE="$arg" - LOCKFILE="$REPO_DB_FILE.lck" - check_repo_db - else - case "$cmd" in - repo-add) add $arg && success=1 ;; - repo-remove) remove $arg && success=1 ;; - esac - fi - ;; - esac -done - -# if at least one operation was a success, re-zip database -if (( success )); then - msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" - - case "$REPO_DB_FILE" in - *tar.gz) TAR_OPT="z" ;; - *tar.bz2) TAR_OPT="j" ;; - *tar.xz) TAR_OPT="J" ;; - *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ - "$REPO_DB_FILE" ;; - esac - - filename=$(basename "$REPO_DB_FILE") - - pushd "$tmpdir" >/dev/null - if [[ -n $(ls) ]]; then - bsdtar -c${TAR_OPT}f "$filename" * - else - # we have no packages remaining? zip up some emptyness - warning "$(gettext "No packages remain, creating empty database.")" - bsdtar -c${TAR_OPT}f "$filename" -T /dev/null - fi - popd >/dev/null - - [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" - [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" - dblink="${REPO_DB_FILE%.tar.*}" - target=${REPO_DB_FILE##*/} - ln -sf "$target" "$dblink" 2>/dev/null || \ - ln -f "$target" "$dblink" 2>/dev/null || \ - cp "$REPO_DB_FILE" "$dblink" -else - msg "$(gettext "No packages modified, nothing to do.")" - exit 1 -fi - -exit 0 -# vim: set ts=2 sw=2 noet: -- cgit v1.2.3 From 00b1be5472acddf258f948719714ec1c6f3cdab1 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 1 Jan 2014 22:07:36 -0500 Subject: local_config.example: update to reflect what is on repo --- local_config.example | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/local_config.example b/local_config.example index daa6874..ec22d84 100644 --- a/local_config.example +++ b/local_config.example @@ -1,8 +1,13 @@ #/bin/bash # as a hint to text editors -_paraboladir=~/parabolagnulinux.org +_paraboladir=/srv/http/repo/public # db-sync -mirror="mirrors.eu.kernel.org" +#mirror="mirrors.uk2.net" +mirror="mirrors.kernel.org" +#mirror="mirror.umd.edu" +#mirror="archlinux.c3sl.ufpr.br" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.de.leaseweb.net" mirrorpath="mirrors/archlinux" # mkrepo -- cgit v1.2.3 From b57c78930fec71dc6c5ef8eaa5148a22f75a1fff Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 1 Jan 2014 22:08:39 -0500 Subject: go ahead and track local_config --- .gitignore | 3 +-- local_config | 20 ++++++++++++++++++++ local_config.example | 20 -------------------- 3 files changed, 21 insertions(+), 22 deletions(-) create mode 100644 local_config delete mode 100644 local_config.example diff --git a/.gitignore b/.gitignore index dd17455..98a5228 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,9 @@ *~ *.pyc -local_config /config.local test/packages/*/*.pkg.tar.?z \#*# .#* yftime src* -pkg* \ No newline at end of file +pkg* diff --git a/local_config b/local_config new file mode 100644 index 0000000..ec22d84 --- /dev/null +++ b/local_config @@ -0,0 +1,20 @@ +#/bin/bash # as a hint to text editors +_paraboladir=/srv/http/repo/public + +# db-sync +#mirror="mirrors.uk2.net" +mirror="mirrors.kernel.org" +#mirror="mirror.umd.edu" +#mirror="archlinux.c3sl.ufpr.br" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.de.leaseweb.net" +mirrorpath="mirrors/archlinux" + +# mkrepo +repodir=${_paraboladir}/repo + +# yf-update +blacklist=${_paraboladir}/docs/blacklist.txt +whitelist=${_paraboladir}/docs/whitelist.txt + +unset _paraboladir diff --git a/local_config.example b/local_config.example deleted file mode 100644 index ec22d84..0000000 --- a/local_config.example +++ /dev/null @@ -1,20 +0,0 @@ -#/bin/bash # as a hint to text editors -_paraboladir=/srv/http/repo/public - -# db-sync -#mirror="mirrors.uk2.net" -mirror="mirrors.kernel.org" -#mirror="mirror.umd.edu" -#mirror="archlinux.c3sl.ufpr.br" -#mirror="mirror.us.leaseweb.net" -#mirror="mirror.de.leaseweb.net" -mirrorpath="mirrors/archlinux" - -# mkrepo -repodir=${_paraboladir}/repo - -# yf-update -blacklist=${_paraboladir}/docs/blacklist.txt -whitelist=${_paraboladir}/docs/whitelist.txt - -unset _paraboladir -- cgit v1.2.3 From 7e93af9ef087cd048dd42cef71328ec7f75e885c Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 5 Jan 2014 04:21:24 +0000 Subject: fix use of $0 in db-cleanup --- db-cleanup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-cleanup b/db-cleanup index 904c06e..841800b 100755 --- a/db-cleanup +++ b/db-cleanup @@ -43,7 +43,7 @@ for _repo in ${PKGREPOS[@]}; do bsdtar tf "${dbfile}" | \ cut -d'/' -f1 | \ sort -u | \ - sed "s|$|*|" >> /tmp/$0.$$.filter + sed "s|$|*|" >> /tmp/${0##*/}.$$.filter done done @@ -54,7 +54,7 @@ for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do msg2 "${POOL}" rsync ${EXTRAFLAGS} -va --delete-excluded \ - --include-from="/tmp/$0.$$.filter" \ + --include-from="/tmp/${0##*/}.$$.filter" \ --exclude="*" \ ${FTP_BASE}/${POOL}/ \ ${FTP_BASE}/${POOL}/ -- cgit v1.2.3 From ba21dd9e342bd640b1bbaa82ebf015932390b18e Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 5 Jan 2014 04:21:52 +0000 Subject: db-update: add hook for publishing generated sources --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 0359697..2fa23af 100755 --- a/db-update +++ b/db-update @@ -83,4 +83,4 @@ for repo in ${repos[@]}; do done done - +cp -rviT "${STAGING}/other/" "${FTP_BASE}/other/" -- cgit v1.2.3 From d4f3035779dae2efc5c5baa527ebd82404207290 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 5 Jan 2014 04:22:51 +0000 Subject: go ahead and track local_config --- .gitignore | 3 +-- local_config | 31 +++++++++++++++++++++++++++++++ local_config.example | 26 -------------------------- 3 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 local_config delete mode 100644 local_config.example diff --git a/.gitignore b/.gitignore index dd17455..98a5228 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,9 @@ *~ *.pyc -local_config /config.local test/packages/*/*.pkg.tar.?z \#*# .#* yftime src* -pkg* \ No newline at end of file +pkg* diff --git a/local_config b/local_config new file mode 100644 index 0000000..8f3946a --- /dev/null +++ b/local_config @@ -0,0 +1,31 @@ +# Mirror options +#mirror="mirrors.uk2.net" +mirror="mirrors.kernel.org" +#mirror="mirror.umd.edu" +#mirror="archlinux.c3sl.ufpr.br" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.de.leaseweb.net" +mirrorpath="archlinux" + +# Directories: they should end without / +paraboladir=/srv/http/repo/public +tempdir=/tmp +archdb=${tempdir}/db +docs_dir=${paraboladir}/docs +repodir=${paraboladir} +licenses_dir=${docs_dir}/pending_licenses +# End Directories + +# Files +logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log +rsout_file=${tempdir}/rsout +rsync_not_needed=${tempdir}/rsync_not_needed + +rsync_blacklist=${docs_dir}/rsyncBlacklist + +blacklist=${docs_dir}/blacklist.txt +whitelist=${docs_dir}/whitelist.txt + +# Rsync commands +rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " +rsync_update_command="rsync -vrptgoL --exclude='*.abs.tar.*' --no-motd " diff --git a/local_config.example b/local_config.example deleted file mode 100644 index 2280cc2..0000000 --- a/local_config.example +++ /dev/null @@ -1,26 +0,0 @@ -# Mirror options -mirror="mirrors.eu.kernel.org" -mirrorpath="mirrors/archlinux" - -# Directories: they should end without / -paraboladir=~/parabolagnulinux.org -tempdir=~/tmp -archdb=${tempdir}/db -docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo -licenses_dir=${docs_dir}/pending_licenses -# End Directories - -# Files -logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log -rsout_file=${tempdir}/rsout -rsync_not_needed=${tempdir}/rsync_not_needed - -rsync_blacklist=${docs_dir}/rsyncBlacklist - -blacklist=${docs_dir}/blacklist.txt -whitelist=${docs_dir}/whitelist.txt - -# Rsync commands -rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " -rsync_update_command="rsync -rptgoL --exclude='*.abs.tar.*' --no-motd " -- cgit v1.2.3 From 91f6039acb4ba915a3ca3fc366a7159656ec0dc1 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 6 Jan 2014 04:30:22 +0000 Subject: fix things --- db-remove | 2 +- db-update | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/db-remove b/db-remove index a71bbab..f0785e5 100755 --- a/db-remove +++ b/db-remove @@ -33,7 +33,7 @@ for pkgbase in ${pkgbases[@]}; do if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) else - warning "$pkgbase not found in $svnrepo" + warning "$pkgbase not found in ABS(libre)" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" remove_pkgs[${#remove_pkgs[*]}]=$pkgbase diff --git a/db-update b/db-update index 2fa23af..2d4f28a 100755 --- a/db-update +++ b/db-update @@ -9,7 +9,7 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty ! -name other -printf '%f ' 2>/dev/null)) if [ $? -ge 1 ]; then die "Could not read ${STAGING}" fi @@ -39,10 +39,10 @@ for repo in ${repos[@]}; do die "Package ${repo}/$(basename ${pkg}) already exists in another repository" fi done - # This is fucking obnoxious -# if ! check_splitpkgs ${repo} ${pkgs[@]}; then -# die "Missing split packages for ${repo}" -# fi + # This is fucking obnoxious + #if ! check_splitpkgs ${repo} ${pkgs[@]}; then + # die "Missing split packages for ${repo}" + #fi else die "Could not read ${STAGING}" fi @@ -83,4 +83,13 @@ for repo in ${repos[@]}; do done done -cp -rviT "${STAGING}/other/" "${FTP_BASE}/other/" +cd "${STAGING}" +while read -r file; do + pub="${FTP_BASE}/${file}" + if [[ -f $pub ]]; then + warning "file already exists: %s" "${file}" + else + mkdir -p -- "${pub%/*}" + mv -vn "$file" "$pub" + fi +done < <(find other -type f) -- cgit v1.2.3 From 6682aefeafb8d30899e464122864f56318ad7640 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 8 Jan 2014 05:20:12 +0000 Subject: Don't error about permissions on empty staging repos (feature #460) --- db-update | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index 2d4f28a..b9b8015 100755 --- a/db-update +++ b/db-update @@ -23,11 +23,11 @@ done # check if packages are valid for repo in ${repos[@]}; do - if ! check_repo_permission "${repo}"; then - die "You don't have permission to update packages in ${repo}" - fi pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then + if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then + die "You don't have permission to update packages in ${repo}" + fi for pkg in ${pkgs[@]}; do if [ -h "${pkg}" ]; then die "Package ${repo}/$(basename ${pkg}) is a symbolic link" @@ -92,4 +92,4 @@ while read -r file; do mkdir -p -- "${pub%/*}" mv -vn "$file" "$pub" fi -done < <(find other -type f) +done < <(find other sources -type f) -- cgit v1.2.3 From 725ab5d12375dd593c375b1a494021bb96135d4f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 00:30:36 -0500 Subject: db-update: remove 'other' from repo blacklist now that issue #460 is added --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index b9b8015..1fddb8a 100755 --- a/db-update +++ b/db-update @@ -9,7 +9,7 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty ! -name other -printf '%f ' 2>/dev/null)) +repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) if [ $? -ge 1 ]; then die "Could not read ${STAGING}" fi -- cgit v1.2.3 From 08496a8bd13ffc7228feac53accb93b063c4b358 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 14:55:11 -0500 Subject: config: fix typo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 3b7c13a..c6fca1b 100644 --- a/config +++ b/config @@ -1,4 +1,4 @@ -#/bin/bash # as a hint to text editors +#!/bin/bash # as a hint to text editors FTP_BASE="/srv/http/repo/public" SVNREPO="/var/abs" -- cgit v1.2.3 From fedfd6f27bff4fddbaa86fb3881f591746a47fd2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 15:15:16 -0500 Subject: createrepos: fix quoting --- createrepos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/createrepos b/createrepos index cd17e4e..8da2455 100755 --- a/createrepos +++ b/createrepos @@ -3,6 +3,6 @@ source "$(dirname "$(readlink -e "$0")")/config" -mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR} ${STAGING}" +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" "$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" -- cgit v1.2.3 From 4a0e623b0c8162b466695be7f5091c76b8c0b63b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 16:14:55 -0500 Subject: db-functions: use ${array[*]} when appropriate --- db-functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-functions b/db-functions index 0b59a53..efb6172 100644 --- a/db-functions +++ b/db-functions @@ -424,9 +424,9 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ - || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" + || error "repo-add ${repo}${DBEXT} ${pkgs[*]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ - || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" + || error "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -445,9 +445,9 @@ arch_repo_remove() { return 1 fi /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ - || error "repo-remove ${dbfile} ${pkgs[@]}" + || error "repo-remove ${dbfile} ${pkgs[*]}" /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ - || error "repo-remove ${filesfile} ${pkgs[@]}" + || error "repo-remove ${filesfile} ${pkgs[*]}" set_repo_permission "${repo}" "${arch}" REPO_MODIFIED=1 -- cgit v1.2.3 From e7d2dcac7cf857fdccd82bec2bfc2a7d8e6b85c6 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 16:53:56 -0500 Subject: Normalize to load ./config before loading ./db-functions --- create-repo | 2 +- cron-jobs/repo-sanity-check | 2 +- cron-jobs/sourceballs2 | 2 +- db-check-nonfree | 2 +- db-list-unsigned-packages | 2 +- repo-restore-to-normal | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/create-repo b/create-repo index 24b890d..21a2a9c 100755 --- a/create-repo +++ b/create-repo @@ -1,8 +1,8 @@ #!/bin/bash # Creates repository structure -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -eq 0 ]; then msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 2aa7892..ee4c061 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname "$(readlink -e "$0")")/../db-functions" . "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" # Traverse all repos for _repo in ${PKGREPOS[@]}; do diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index bbe227d..1432bdf 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -5,8 +5,8 @@ # Remove the old sourceballs dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" . "${MAKEPKGCONF}" pushd "${WORKDIR}" >/dev/null diff --git a/db-check-nonfree b/db-check-nonfree index 661daa6..5cb7f6f 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is not supported" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 985d1c0..5105096 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -20,8 +20,8 @@ set -e # unsigned packages available for architecture $1 and specified for # architecture $2 (usually $1 or any, default is to list all). -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 1 ]; then msg "usage: ${0##*/} " diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 3636920..3fe4816 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore PKGREPOS=(community) -- cgit v1.2.3 From 39fbf0d8d3cdc666912c597d41d5b6a70fc0c725 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 20:53:38 -0500 Subject: Fix some array quoting. --- create-repo | 4 ++-- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 22 ++++++++++---------- cron-jobs/ftpdir-cleanup | 12 +++++------ cron-jobs/repo-sanity-check | 4 ++-- cron-jobs/sourceballs | 26 ++++++++++++------------ cron-jobs/sourceballs2 | 4 ++-- db-check-nonfree | 22 ++++++++++---------- db-functions | 30 ++++++++++++++-------------- db-list-unsigned-packages | 2 +- db-move | 27 ++++++++++++------------- db-remove | 6 +++--- db-sync | 5 ++--- db-update | 26 ++++++++++++------------ get-repos | 6 +++--- repo-restore-to-normal | 6 +++--- 15 files changed, 100 insertions(+), 102 deletions(-) diff --git a/create-repo b/create-repo index 21a2a9c..1ec9798 100755 --- a/create-repo +++ b/create-repo @@ -10,12 +10,12 @@ if [ $# -eq 0 ]; then fi msg "Creating repos..." -for _repo in $@; do +for _repo in "$@"; do msg2 "Creating [${_repo}]" mkdir -p "${FTP_BASE}/staging/${_repo}" || \ error "Failed creating staging dir" - for _arch in ${ARCHES[@]}; do + for _arch in "${ARCHES[@]}"; do mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ error "Failed creating ${_arch} dir" done diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 3f92169..c8d8618 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -6,18 +6,18 @@ exit() { return; } splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') -variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' "${splitpkg_overrides[@]}") readonly -a variables splitpkg_overrides backup_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" eval "${indirect}=(\${$var[@]})" done } restore_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then eval "${var}=(\${$indirect[@]})" @@ -42,31 +42,31 @@ print_info() { if [ -n "$arch" ]; then echo "%ARCH%" - for i in ${arch[@]}; do echo $i; done + for i in "${arch[@]}"; do echo $i; done echo "" fi if [ -n "$depends" ]; then echo "%DEPENDS%" - for i in ${depends[@]}; do + for i in "${depends[@]}"; do echo $i done echo "" fi if [ -n "$makedepends" ]; then echo "%MAKEDEPENDS%" - for i in ${makedepends[@]}; do + for i in "${makedepends[@]}"; do echo $i done echo "" fi if [ -n "$conflicts" ]; then echo "%CONFLICTS%" - for i in ${conflicts[@]}; do echo $i; done + for i in "${conflicts[@]}"; do echo $i; done echo "" fi if [ -n "$provides" ]; then echo "%PROVIDES%" - for i in ${provides[@]}; do echo $i; done + for i in "${provides[@]}"; do echo $i; done echo "" fi } @@ -75,7 +75,7 @@ source_pkgbuild() { ret=0 dir=$1 pkgbuild=$dir/PKGBUILD - for var in ${variables[@]}; do + for var in "${variables[@]}"; do unset ${var} done source $pkgbuild &>/dev/null || ret=$? @@ -88,7 +88,7 @@ source_pkgbuild() { if [ "${#pkgname[@]}" -gt "1" ]; then pkgbase=${pkgbase:-${pkgname[0]}} - for pkg in ${pkgname[@]}; do + for pkg in "${pkgname[@]}"; do if [ "$(type -t package_${pkg})" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" return 1 @@ -98,7 +98,7 @@ source_pkgbuild() { while IFS= read -r line; do var=${line%%=*} var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters - for realvar in ${variables[@]}; do + for realvar in "${variables[@]}"; do if [ "$var" == "$realvar" ]; then eval $line break diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index e42a1a8..ad2e7f9 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -24,16 +24,16 @@ clean_pkg() { script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_lock ${repo} ${arch} || exit 1 done done ${CLEANUP_DRYRUN} && warning 'dry run mode is active' -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi @@ -87,8 +87,8 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_unlock ${repo} ${arch} done done diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index ee4c061..9d351df 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -5,7 +5,7 @@ . "$(dirname "$(readlink -e "$0")")/../db-functions" # Traverse all repos -for _repo in ${PKGREPOS[@]}; do +for _repo in "${PKGREPOS[@]}"; do msg "Cleaning up [${_repo}]" # Find all pkgnames on this repo's abs @@ -19,7 +19,7 @@ for _repo in ${PKGREPOS[@]}; do >/dev/null 2>&1 # also cleanup package functions - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${pkg} >/dev/null 2>&1 done diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 1542499..329e135 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -7,8 +7,8 @@ pushd "${WORKDIR}" >/dev/null script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_lock ${repo} ${arch} || exit 1 done done @@ -18,8 +18,8 @@ renice +10 -p $$ > /dev/null # Create a readable file for each repo with the following format # - [ ] -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do # Repo does not exist; skip it if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue @@ -39,8 +39,8 @@ for repo in ${PKGREPOS[@]}; do done | sort -u > "${WORKDIR}/db-${repo}" done -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_unlock ${repo} ${arch} done done @@ -49,15 +49,15 @@ done find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do newpkgs=() failedpkgs=() while read line; do - pkginfo=(${line}) + pkginfo=("${line}") pkgbase=${pkginfo[0]} pkgver=${pkginfo[1]} pkgarch=${pkginfo[2]} - pkglicense=(${pkginfo[@]:3}) + pkglicense=("${pkginfo[@]:3}") # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then @@ -105,13 +105,13 @@ for repo in ${PKGREPOS[@]}; do if [ ${#newpkgs[@]} -ge 1 ]; then msg "Adding source packages for [${repo}]..." - for new_pkg in ${newpkgs[@]}; do + for new_pkg in "${newpkgs[@]}"; do msg2 "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then msg "Failed to create source packages for [${repo}]..." - for failed_pkg in ${failedpkgs[@]}; do + for failed_pkg in "${failedpkgs[@]}"; do msg2 "${failed_pkg}" done fi @@ -125,7 +125,7 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" @@ -137,7 +137,7 @@ fi old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 1432bdf..2a26e6a 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -21,7 +21,7 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do msg "Sourceballing [${repo}]" pushd $repo >/dev/null @@ -40,7 +40,7 @@ for repo in ${PKGREPOS[@]}; do unset build package url pkgdesc source md5sums depends makedepends \ optdepends license arch options check mksource - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${_pkg} >/dev/null 2>&1 done diff --git a/db-check-nonfree b/db-check-nonfree index 5cb7f6f..6e2dc17 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -9,16 +9,16 @@ if [ $# -ge 1 ]; then fi # TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo} ${pkgarch} || exit 1 done done msg "Check nonfree in repo:" nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) -for repo in ${ARCHREPOS[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${ARCHREPOS[@]}"; do + for pkgarch in "${ARCHES[@]}"; do msg2 "$repo $pkgarch" if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue @@ -27,20 +27,20 @@ for repo in ${ARCHREPOS[@]}; do unset cleanpkgs cleanpkgs=() dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in ${dbpkgs[@]}; do - if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs+=(${pkgname}) + for pkgname in "${dbpkgs[@]}"; do + if in_array "${pkgname}" "${nonfree[@]}"; then + cleanpkgs+=("${pkgname}") fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: ${cleanpkgs[@]}" - arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + msg2 "Nonfree: ${cleanpkgs[*]}" + arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" fi done done -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo} ${pkgarch} done done diff --git a/db-functions b/db-functions index efb6172..5b10e05 100644 --- a/db-functions +++ b/db-functions @@ -67,7 +67,7 @@ cleanup() { local arch trap - EXIT INT QUIT TERM - for l in ${LOCKS[@]}; do + for l in "${LOCKS[@]}"; do repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then @@ -256,7 +256,7 @@ getpkgfiles() { exit 1 fi - for f in ${@}; do + for f in "${@}"; do if [ ! -f "${f}" ]; then error "Package ${f} not found!" exit 1 @@ -266,7 +266,7 @@ getpkgfiles() { fi done - echo ${@} + echo "${@}" } check_pkgfile() { @@ -279,7 +279,7 @@ check_pkgfile() { local pkgarch="$(getpkgarch ${pkgfile})" [ $? -ge 1 ] && return 1 - in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 + in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then return 0 @@ -300,13 +300,13 @@ check_pkgxbs() { [ $? -ge 1 ] && return 1 local repo="${2}" - in_array "${repo}" ${PKGREPOS[@]} || return 1 + in_array "${repo}" "${PKGREPOS[@]}" || return 1 local xbsver="$(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; get_full_version "${_pkgname}")" [ "${xbsver}" == "${_pkgver}" ] || return 1 local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo ${pkgname[@]})) - in_array "${_pkgname}" ${xbsnames[@]} || return 1 + in_array "${_pkgname}" "${xbsnames[@]}" || return 1 return 0 } @@ -314,7 +314,7 @@ check_pkgxbs() { check_splitpkgs() { local repo="${1}" shift - local pkgfiles=(${@}) + local pkgfiles=("${@}") local pkgfile local pkgdir local xbsname @@ -322,7 +322,7 @@ check_splitpkgs() { mkdir -p "${WORKDIR}/check_splitpkgs/" pushd "${WORKDIR}/check_splitpkgs" >/dev/null - for pkgfile in ${pkgfiles[@]}; do + for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" msg2 "Checking %s" "$_pkgbase" @@ -332,7 +332,7 @@ check_splitpkgs() { echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo ${pkgname[@]})) - for xbsname in ${xbsnames[@]}; do + for xbsname in "${xbsnames[@]}"; do echo "${xbsname}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" done done @@ -419,13 +419,13 @@ set_repo_permission() { arch_repo_add() { local repo=$1 local arch=$2 - local pkgs=(${@:3}) + local pkgs=("${@:3}") # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[*]}" - /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -436,7 +436,7 @@ arch_repo_add() { arch_repo_remove() { local repo=$1 local arch=$2 - local pkgs=(${@:3}) + local pkgs=("${@:3}") local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" @@ -444,9 +444,9 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[*]}" - /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ + /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ || error "repo-remove ${filesfile} ${pkgs[*]}" set_repo_permission "${repo}" "${arch}" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 5105096..f593686 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -31,7 +31,7 @@ fi arch=$1 shift -for repo in ${PKGREPOS[@]} +for repo in "${PKGREPOS[@]}" do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" diff --git a/db-move b/db-move index 1b34404..c66b088 100755 --- a/db-move +++ b/db-move @@ -8,7 +8,7 @@ if [ $# -lt 3 ]; then exit 1 fi -args=(${@}) +args=("${@}") repo_from="${args[0]}" repo_to="${args[1]}" ftppath_from="${FTP_BASE}/${repo_from}/os/" @@ -19,14 +19,14 @@ if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then fi # TODO: this might lock too much (architectures) -for pkgarch in ${ARCHES[@]}; do +for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo_to} ${pkgarch} || exit 1 repo_lock ${repo_from} ${pkgarch} || exit 1 done # check if packages to be moved exist in xbs and ftp dir -for pkgbase in ${args[@]:2}; do - for pkgarch in ${ARCHES[@]} 'any'; do +for pkgbase in "${args[@]:2}"; do + for pkgarch in "${ARCHES[@]}" 'any'; do xbsrepo_from="$(xbs releasepath ${pkgbase} ${repo_from} ${pkgarch})" if [ -r "${xbsrepo_from}/PKGBUILD" ]; then pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; echo ${pkgname[@]})) @@ -34,19 +34,18 @@ for pkgbase in ${args[@]:2}; do die "Could not read pkgname" fi - if [ "${pkgarch}" == 'any' ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi - for pkgname in ${pkgnames[@]}; do + for pkgname in "${pkgnames[@]}"; do pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version ${pkgname}) if [ -z "${pkgver}" ]; then die "Could not read pkgver" fi - for tarch in ${tarches[@]}; do + for tarch in "${tarches[@]}"; do getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null done done @@ -60,11 +59,11 @@ msg "Moving packages from [${repo_from}] to [${repo_to}]..." declare -A add_pkgs declare -A remove_pkgs -for pkgbase in ${args[@]:2}; do +for pkgbase in "${args[@]:2}"; do # move the package in xbs arches=($(xbs move ${repo_from} ${repo_to} ${pkgbase})) # move the package in ftp - for pkgarch in ${arches[@]}; do + for pkgarch in "${arches[@]}"; do dir_to="$(xbs releasepath $pkgbase $repo_to $pkgarch)" if true; then # to add in indent level to make merging easier if [ "${pkgarch}" == 'any' ]; then @@ -74,9 +73,9 @@ for pkgbase in ${args[@]:2}; do fi pkgnames=($(. "${dir_to}/PKGBUILD"; echo ${pkgname[@]})) - for pkgname in ${pkgnames[@]}; do + for pkgname in "${pkgnames[@]}"; do pkgver=$(. "${dir_to}/PKGBUILD"; get_full_version ${pkgname}) - for tarch in ${tarches[@]}; do + for tarch in "${tarches[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) pkgfile="${pkgpath##*/}" @@ -92,14 +91,14 @@ for pkgbase in ${args[@]:2}; do done done -for tarch in ${ARCHES[@]}; do +for tarch in "${ARCHES[@]}"; do if [ -n "${add_pkgs[${tarch}]}" ]; then arch_repo_add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} arch_repo_remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} fi done -for pkgarch in ${ARCHES[@]}; do +for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo_from} ${pkgarch} repo_unlock ${repo_to} ${pkgarch} done diff --git a/db-remove b/db-remove index 1c25e5c..33d0933 100755 --- a/db-remove +++ b/db-remove @@ -10,7 +10,7 @@ fi repo="$1" arch="$2" -pkgbases=(${@:3}) +pkgbases=("${@:3}") if ! check_repo_permission $repo; then die "You don't have permission to remove packages from ${repo}" @@ -27,7 +27,7 @@ for tarch in "${tarches[@]}"; do done remove_pkgs=() -for pkgbase in ${pkgbases[@]}; do +for pkgbase in "${pkgbases[@]}"; do msg "Removing $pkgbase from [$repo]..." path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" if [ -d "$path" ]; then @@ -37,7 +37,7 @@ for pkgbase in ${pkgbases[@]}; do warning "$pkgbase not found in XBS $repo-$arch" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" - remove_pkgs+=($pkgbase) + remove_pkgs+=("$pkgbase") fi done diff --git a/db-sync b/db-sync index e4b6966..3595876 100755 --- a/db-sync +++ b/db-sync @@ -67,8 +67,8 @@ init() { get_repos # Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do + for _repo in "${ARCHREPOS[@]}"; do + for _arch in "${ARCHARCHES[@]}"; do msg "Processing ${_repo}-${_arch}" db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} @@ -165,7 +165,6 @@ init() { rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ ${FTP_BASE}/${SRCPOOL}/ done - # Cleanup unset blacklist whitelists _arch _repo repo_file diff --git a/db-update b/db-update index 8cf61fc..c4bd33b 100755 --- a/db-update +++ b/db-update @@ -16,26 +16,26 @@ fi repos=() for staging_repo in ${staging_repos[@]##*/}; do - if in_array ${staging_repo} ${PKGREPOS[@]}; then - repos+=(${staging_repo}) + if in_array "${staging_repo}" "${PKGREPOS[@]}"; then + repos+=("${staging_repo}") fi done # TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo} ${pkgarch} || exit 1 done done # check if packages are valid -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then die "You don't have permission to update packages in ${repo}" fi - for pkg in ${pkgs[@]}; do + for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then die "Package ${repo}/${pkg##*/} is a symbolic link" fi @@ -60,13 +60,13 @@ for repo in ${repos[@]}; do fi done -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do msg "Updating [${repo}]..." any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) - for pkgarch in ${ARCHES[@]}; do + for pkgarch in "${ARCHES[@]}"; do add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) - for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do + for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run @@ -81,16 +81,16 @@ for repo in ${repos[@]}; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi - add_pkgs[${#add_pkgs[*]}]=${pkgfile} + add_pkgs+=("${pkgfile}") done if [ ${#add_pkgs[@]} -ge 1 ]; then - arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} + arch_repo_add "${repo}" "${pkgarch}" "${add_pkgs[@]}" fi done done -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo} ${pkgarch} done done diff --git a/get-repos b/get-repos index 5096433..b8d2ccb 100755 --- a/get-repos +++ b/get-repos @@ -25,8 +25,8 @@ trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT DBLIST=() # Repos -for _repo in ${PKGREPOS[@]}; do - for _arch in ${ARCHES[@]}; do +for _repo in "${PKGREPOS[@]}"; do + for _arch in "${ARCHES[@]}"; do DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${FILESEXT}") done done @@ -37,7 +37,7 @@ wget --directory-prefix=${WORKDIR} \ --no-verbose \ --force-directories \ --no-host-directories \ - ${DBLIST[@]} || true + "${DBLIST[@]}" || true # Always return true, some databases are expect to be missing # Create the arches regexp arch1|arch2 diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 3fe4816..063aacf 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -12,7 +12,7 @@ PKGREPOS=(community) # sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/")) # Traverse all repos -for _repo in ${PKGREPOS[@]}; do +for _repo in "${PKGREPOS[@]}"; do msg "Restoring [${_repo}]" # Find all pkgnames on this repo's abs @@ -27,7 +27,7 @@ for _repo in ${PKGREPOS[@]}; do >/dev/null 2>&1 # also cleanup package functions - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${pkg} >/dev/null 2>&1 # this fills the on_abs array echo ${_pkg}-${pkgver}-${pkgrel} @@ -49,7 +49,7 @@ for _repo in ${PKGREPOS[@]}; do msg2 "Restoring the following packages:" # plain "$(echo ${restore[@]} | tr ' ' "\n")" - for _pkg in ${on_abs[@]}; do + for _pkg in "${on_abs[@]}"; do find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \; done -- cgit v1.2.3 From 607b95c40e8ff5fbb452a1594d96ec6cac43df40 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 20:56:24 -0500 Subject: fix local_config:mirrorpath for the selected mirror --- local_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local_config b/local_config index ec22d84..d329e17 100644 --- a/local_config +++ b/local_config @@ -8,7 +8,7 @@ mirror="mirrors.kernel.org" #mirror="archlinux.c3sl.ufpr.br" #mirror="mirror.us.leaseweb.net" #mirror="mirror.de.leaseweb.net" -mirrorpath="mirrors/archlinux" +mirrorpath="archlinux" # mkrepo repodir=${_paraboladir}/repo -- cgit v1.2.3 From cad2d8b7075cb255f76e79c12c23acee94074514 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:02:13 -0500 Subject: repo-sanity-check: take advantage of printf --- cron-jobs/repo-sanity-check | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 9d351df..8b0758f 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -41,8 +41,9 @@ for _repo in "${PKGREPOS[@]}"; do )) # Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ - <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) + remove=($(comm -13 \ + <(printf '%s\n' "${on_abs[@]}" | sort -u) \ + <(printf '%s\n' "${on_repo[@]}" | sort -u) )) # Remove them from databases, ftpdir-cleanup will take care of the rest find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ -- cgit v1.2.3 From 766a076891a37cebc7b448241c8c183c32f49d9e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 18:05:41 -0500 Subject: rm repo-remove -- in Oct 2012 fauno removed repo-add, but left this The customized repo-{add,remove} did license extraction for packages. They were based on the versions from pacman 3.5.0 Here is a diff between the stock versions from 3.5.0, and the modified version that we had: --- repo-add.sh.in 2013-12-31 18:02:13.546351038 -0500 +++ repo-remove.in 2013-12-31 18:13:19.957948677 -0500 @@ -20,6 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +LICENSESDIR=/home/parabolavnx/licenses + # gettext initialization export TEXTDOMAIN='pacman' export TEXTDOMAINDIR='@localedir@' @@ -309,6 +311,22 @@ fi fi + # Extracts licenses to a common license dir + msg "Extracting license" + if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + # Change dir to licenses, and extract them stripping the first part of the path + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ + --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + warning "This package doesn't contain a license dir" + fi + fi + return 0 } # end db_write_entry @@ -328,6 +346,12 @@ rm -rf $pkgentry pkgentry=$(find_pkgentry $pkgname) done + + msg "Removing license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + return $notfound } # end db_remove_entry --- repo-remove | 561 ------------------------------------------------------------ 1 file changed, 561 deletions(-) delete mode 100755 repo-remove diff --git a/repo-remove b/repo-remove deleted file mode 100755 index c4bf96f..0000000 --- a/repo-remove +++ /dev/null @@ -1,561 +0,0 @@ -#!/bin/bash -# -# repo-add - add a package to a given repo database file -# repo-remove - remove a package entry from a given repo database file -# Generated from repo-add.in; do not edit by hand. -# -# Copyright (c) 2006-2008 Aaron Griffin -# Copyright (c) 2007-2008 Dan McGee -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -LICENSESDIR=/home/parabolavnx/licenses - -# gettext initialization -export TEXTDOMAIN='pacman' -export TEXTDOMAINDIR='/usr/share/locale' - -myver='3.5.0' -confdir='/home/parabolavnx/etc' - -QUIET=0 -DELTA=0 -WITHFILES=0 -REPO_DB_FILE= -LOCKFILE= -CLEAN_LOCK=0 - -# ensure we have a sane umask set -umask 0022 - -msg() { - (( QUIET )) && return - local mesg=$1; shift - printf "==> ${mesg}\n" "$@" >&1 -} - -msg2() { - (( QUIET )) && return - local mesg=$1; shift - printf " -> ${mesg}\n" "$@" >&1 -} - -warning() { - local mesg=$1; shift - printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 -} - -# print usage instructions -usage() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" - printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" - printf "$(gettext "\ -repo-add will update a package database by reading a package file.\n\ -Multiple packages to add can be specified on the command line.\n\n")" - printf "$(gettext "\ -repo-remove will update a package database by removing the package name\n\ -specified on the command line from the given repo database. Multiple\n\ -packages to remove can be specified on the command line.\n\n")" - printf "$(gettext "\ -Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ -and errors.\n\n")" - printf "$(gettext "\ -Use the -d/--delta flag to automatically generate and add a delta file\n\ -between the old entry and the new one, if the old package file is found\n\ -next to the new one.\n\n")" - printf "$(gettext "\ -Use the -f/--files flag to update a database including file entries.\n\n")" - echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" - echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" -} - -version() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "\ -Copyright (C) 2006-2008 Aaron Griffin .\n\ -Copyright (c) 2007-2008 Dan McGee .\n\n\ -This is free software; see the source for copying conditions.\n\ -There is NO WARRANTY, to the extent permitted by law.\n")" -} - -# write a list entry -# arg1 - Entry name -# arg2 - List -# arg3 - File to write to -write_list_entry() { - if [[ -n $2 ]]; then - echo "%$1%" >>$3 - echo -e $2 >>$3 - fi -} - -find_pkgentry() -{ - local pkgname=$1 - local pkgentry - for pkgentry in $tmpdir/$pkgname*; do - name=${pkgentry##*/} - if [[ ${name%-*-*} = $pkgname ]]; then - echo $pkgentry - return 0 - fi - done - return 1 -} - -# Get the package name from the delta filename -get_delta_pkgname() { - local tmp - - tmp=${1##*/} - echo ${tmp%-*-*_to*} -} - -# write a delta entry -# arg1 - path to delta file -db_write_delta() -{ - deltafile="$1" - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - error "$(gettext "No database entry for package '%s'.")" "$pkgname" - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - echo -e "%DELTAS%" >$deltas - fi - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$deltafile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$deltafile") - - oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') - newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') - - if grep -q "$oldfile.*$newfile" $deltas; then - sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup - fi - msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" - echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas - - return 0 -} # end db_write_delta - -# remove a delta entry -# arg1 - path to delta file -db_remove_delta() -{ - deltafile="$1" - filename=${deltafile##*/} - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - return 1 - fi - if grep -q "$filename" $deltas; then - sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup - msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" - return 0 - fi - - return 1 -} # end db_remove_delta - -# write an entry to the pacman database -# arg1 - path to package -db_write_entry() -{ - # blank out all variables - local pkgfile="$1" - local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ - _groups _licenses _replaces _depends _conflicts _provides _optdepends - - local OLDIFS="$IFS" - # IFS (field separator) is only the newline character - IFS=" -" - - # read info from the zipped package - local line var val - for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | - grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do - # bash awesomeness here- var is always one word, val is everything else - var=${line%% *} - val=${line#* } - declare $var="$val" - case "$var" in - group) _groups="$_groups$group\n" ;; - license) _licenses="$_licenses$license\n" ;; - replaces) _replaces="$_replaces$replaces\n" ;; - depend) _depends="$_depends$depend\n" ;; - conflict) _conflicts="$_conflicts$conflict\n" ;; - provides) _provides="$_provides$provides\n" ;; - optdepend) _optdepends="$_optdepends$optdepend\n" ;; - esac - done - - IFS=$OLDIFS - - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$pkgfile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$pkgfile") - - # ensure $pkgname and $pkgver variables were found - if [[ -z $pkgname || -z $pkgver ]]; then - error "$(gettext "Invalid package file '%s'.")" "$pkgfile" - return 1 - fi - - pushd "$tmpdir" >/dev/null - if [[ -d $pkgname-$pkgver ]]; then - warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" - else - if (( DELTA )); then - pkgentry=$(find_pkgentry $pkgname) - if [[ -n $pkgentry ]]; then - local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) - local oldfile="$(dirname $1)/$oldfilename" - fi - fi - fi - - # remove an existing entry if it exists, ignore failures - db_remove_entry "$pkgname" - - # create package directory - mkdir "$pkgname-$pkgver" - pushd "$pkgname-$pkgver" >/dev/null - - # restore an eventual deltas file - [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas - - # create desc entry - msg2 "$(gettext "Creating '%s' db entry...")" 'desc' - echo -e "%FILENAME%\n$(basename "$1")\n" >>desc - echo -e "%NAME%\n$pkgname\n" >>desc - [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc - echo -e "%VERSION%\n$pkgver\n" >>desc - [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc - write_list_entry "GROUPS" "$_groups" "desc" - [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc - [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc - - # compute checksums - msg2 "$(gettext "Computing md5 checksums...")" - echo -e "%MD5SUM%\n$md5sum\n" >>desc - - [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc - write_list_entry "LICENSE" "$_licenses" "desc" - [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc - [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc - [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc - write_list_entry "REPLACES" "$_replaces" "desc" - - # create depends entry - msg2 "$(gettext "Creating '%s' db entry...")" 'depends' - # create the file even if it will remain empty - touch "depends" - write_list_entry "DEPENDS" "$_depends" "depends" - write_list_entry "CONFLICTS" "$_conflicts" "depends" - write_list_entry "PROVIDES" "$_provides" "depends" - write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - - popd >/dev/null - popd >/dev/null - - # create files file if wanted - if (( WITHFILES )); then - msg2 "$(gettext "Creating '%s' db entry...")" 'files' - local files_path="$tmpdir/$pkgname-$pkgver/files" - echo "%FILES%" >$files_path - bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path - fi - - # create a delta file - if (( DELTA )); then - if [[ -n $oldfilename ]]; then - if [[ -f $oldfile ]]; then - delta=$(pkgdelta -q $oldfile $1) - if [[ -f $delta ]]; then - db_write_delta $delta - fi - else - warning "$(gettext "Old package file not found: %s")" "$oldfilename" - fi - fi - fi - - # Extracts licenses to a common license dir - msg "Extracting license" - if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - # Change dir to licenses, and extract them stripping the first part of the path - bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ - --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 - - if [ $? -ne 0 ]; then - warning "This package doesn't contain a license dir" - fi - fi - - return 0 -} # end db_write_entry - -# remove existing entries from the DB -# arg1 - package name -db_remove_entry() { - local pkgname=$1 - local notfound=1 - local pkgentry=$(find_pkgentry $pkgname) - while [[ -n $pkgentry ]]; do - notfound=0 - if [[ -f $pkgentry/deltas ]]; then - mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" - fi - msg2 "$(gettext "Removing existing entry '%s'...")" \ - "$(basename $pkgentry)" - rm -rf $pkgentry - pkgentry=$(find_pkgentry $pkgname) - done - - msg "Removing license" - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - return $notfound -} # end db_remove_entry - -check_repo_db() -{ - # check lock file - if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then - CLEAN_LOCK=1 - else - error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" - exit 1 - fi - - if [[ -f $REPO_DB_FILE ]]; then - # there are two situations we can have here- a DB with some entries, - # or a DB with no contents at all. - if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then - # check empty case - if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then - error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" - exit 1 - fi - fi - msg "$(gettext "Extracting database to a temporary location...")" - bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" - else - case "$cmd" in - repo-remove) - error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" - exit 1 - ;; - repo-add) - # check if the file can be created (write permission, directory existence, etc) - if ! touch "$REPO_DB_FILE"; then - error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" - exit 1 - fi - rm -f "$REPO_DB_FILE" - ;; - esac - fi -} - -add() -{ - if [[ ! -f $1 ]]; then - error "$(gettext "File '%s' not found.")" "$1" - return 1 - fi - - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Adding delta '%s'")" "$deltafile" - if ! type xdelta3 &>/dev/null; then - error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" - exit 1 - fi - if db_write_delta "$deltafile"; then - return 0 - else - return 1 - fi - fi - - pkgfile=$1 - if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then - error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" - return 1 - fi - - msg "$(gettext "Adding package '%s'")" "$pkgfile" - - db_write_entry "$pkgfile" -} - -remove() -{ - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Searching for delta '%s'...")" "$deltafile" - if db_remove_delta "$deltafile"; then - return 0 - else - error "$(gettext "Delta matching '%s' not found.")" "$deltafile" - return 1 - fi - fi - - pkgname=$1 - msg "$(gettext "Searching for package '%s'...")" "$pkgname" - - if db_remove_entry "$pkgname"; then - rm -f "$tmpdir/$pkgname.deltas" - return 0 - else - error "$(gettext "Package matching '%s' not found.")" "$pkgname" - return 1 - fi -} - -trap_exit() -{ - echo - error "$@" - exit 1 -} - -clean_up() { - local exit_code=$? - - [[ -d $tmpdir ]] && rm -rf "$tmpdir" - (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" - - exit $exit_code -} - -# PROGRAM START - -# determine whether we have gettext; make it a no-op if we do not -if ! type gettext &>/dev/null; then - gettext() { - echo "$@" - } -fi - -case "$1" in - -h|--help) usage; exit 0;; - -V|--version) version; exit 0;; -esac - -# figure out what program we are -cmd="$(basename $0)" -if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then - error "$(gettext "Invalid command name '%s' specified.")" "$cmd" - exit 1 -fi - -tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ - error "$(gettext "Cannot create temp directory for database building.")"; \ - exit 1) - -trap 'clean_up' EXIT -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR - -success=0 -# parse arguments -for arg in "$@"; do - case "$arg" in - -q|--quiet) QUIET=1;; - -d|--delta) DELTA=1;; - -f|--files) WITHFILES=1;; - *) - if [[ -z $REPO_DB_FILE ]]; then - REPO_DB_FILE="$arg" - LOCKFILE="$REPO_DB_FILE.lck" - check_repo_db - else - case "$cmd" in - repo-add) add $arg && success=1 ;; - repo-remove) remove $arg && success=1 ;; - esac - fi - ;; - esac -done - -# if at least one operation was a success, re-zip database -if (( success )); then - msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" - - case "$REPO_DB_FILE" in - *tar.gz) TAR_OPT="z" ;; - *tar.bz2) TAR_OPT="j" ;; - *tar.xz) TAR_OPT="J" ;; - *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ - "$REPO_DB_FILE" ;; - esac - - filename=$(basename "$REPO_DB_FILE") - - pushd "$tmpdir" >/dev/null - if [[ -n $(ls) ]]; then - bsdtar -c${TAR_OPT}f "$filename" * - else - # we have no packages remaining? zip up some emptyness - warning "$(gettext "No packages remain, creating empty database.")" - bsdtar -c${TAR_OPT}f "$filename" -T /dev/null - fi - popd >/dev/null - - [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" - [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" - dblink="${REPO_DB_FILE%.tar.*}" - target=${REPO_DB_FILE##*/} - ln -sf "$target" "$dblink" 2>/dev/null || \ - ln -f "$target" "$dblink" 2>/dev/null || \ - cp "$REPO_DB_FILE" "$dblink" -else - msg "$(gettext "No packages modified, nothing to do.")" - exit 1 -fi - -exit 0 -# vim: set ts=2 sw=2 noet: -- cgit v1.2.3 From 3537841a9287ea1c7871545ffebb855561f7c1e0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:17:35 -0500 Subject: misc touch up - TODO: add trailing newline - config: add text editor hint - config.orig: remove - getrepos: quote, use -- - db-functions: hardcode some paths, remove needless nullglob --- TODO | 2 +- config | 2 +- config.orig | 54 ------------------------------------------------------ createrepos | 2 +- db-functions | 21 ++++++++------------- 5 files changed, 11 insertions(+), 70 deletions(-) delete mode 100644 config.orig diff --git a/TODO b/TODO index 3219b1c..9dd4b52 100644 --- a/TODO +++ b/TODO @@ -7,4 +7,4 @@ * Fix db-move - - Make it use abslibre \ No newline at end of file + - Make it use abslibre diff --git a/config b/config index 8b8117f..0152b69 100644 --- a/config +++ b/config @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash # as a hint to text editors FTP_BASE="/srv/http/repo/public" ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" diff --git a/config.orig b/config.orig deleted file mode 100644 index a32f82f..0000000 --- a/config.orig +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -<<<<<<< HEAD -FTP_BASE="/srv/http/repo/public" -ARCH_BASE="/srv/http/repo/public" -SVNREPO="/srv/http/repo/abslibre" -======= -FTP_BASE="/srv/http/repo/public/temprepo" -ARCH_BASE="/srv/http/repo/public/temprepo" -SVNREPO="/var/abs" ->>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e - -# Repos from Arch -ARCHREPOS=('core' 'testing') #'extra' 'community' 'testing' 'multilib') -# Official Parabola repos -OURREPOS=('libre' 'libre-testing') -# User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') -# Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') -PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' - -CLEANUP_DESTDIR="$FTP_BASE/old/packages" -CLEANUP_DRYRUN=true -# Time in days to keep moved packages -CLEANUP_KEEP=30 - -SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" -SOURCE_CLEANUP_DRYRUN=true -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=30 - -REQUIRE_SIGNATURE=true - -LOCK_DELAY=10 -LOCK_TIMEOUT=300 - -STAGING="$FTP_BASE/staging" -TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) -OURARCHES=(mips64el) -ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) -DBEXT=".db.tar.gz" -FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" -SRCEXT=".src.tar.gz" - -<<<<<<< HEAD -MAKEPKGCONF="~/.makepkg.conf" -======= -MAKEPKGCONF="/etc/makepkg.conf" ->>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" diff --git a/createrepos b/createrepos index 4ee057b..926c158 100755 --- a/createrepos +++ b/createrepos @@ -3,6 +3,6 @@ source $(dirname $0)/config -mkdir -p ${FTP_BASE}/{${PKGPOOL},${SRCPOOL}} ${ARCH_BASE} ${CLEANUP_DESTDIR} ${SOURCE_CLEANUP_DESTDIR} ${STAGING} +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${ARCH_BASE}" "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" $(dirname $0)/create-repo ${PKGREPOS[@]} diff --git a/db-functions b/db-functions index 4c247a7..458a370 100644 --- a/db-functions +++ b/db-functions @@ -87,7 +87,7 @@ get_full_version() { script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(stat -c %U $LOCKDIR)" + local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" error "Script $(basename $0) is already locked by $_owner." exit 1 else @@ -178,7 +178,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(stat -c %U $LOCKDIR)" + _owner="$(/usr/bin/stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -211,7 +211,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -293,9 +293,6 @@ getpkgfile() { } getpkgfiles() { -# Ignore anything that doesn't glob to PKGEXT - shopt -s nullglob - local f if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then error 'Duplicate packages found!' @@ -313,8 +310,6 @@ getpkgfiles() { done echo ${@} - - shopt -u nullglob } check_pkgfile() { @@ -485,7 +480,7 @@ set_repo_permission() { local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ -w "${dbfile}" ]; then - local group=$(stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" @@ -502,9 +497,9 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -521,9 +516,9 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" - repo-remove -q "${filesfile}" ${pkgs[@]} \ + /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } -- cgit v1.2.3 From 1bcc49058febc7e2723c455879da6ff3b9ca6576 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:18:36 -0500 Subject: Use `mktemp -t` to respect $TMPDIR --- db-functions | 2 +- get-repos | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db-functions b/db-functions index 458a370..f56ca19 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -dt "$(basename $0).XXXXXXXXXX") LOCKS=() # check if messages are to be printed using color diff --git a/get-repos b/get-repos index bfc08ff..796051c 100755 --- a/get-repos +++ b/get-repos @@ -20,7 +20,7 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -TMPDIR="$(mktemp -d /tmp/$(basename $0).XXXX)" +TMPDIR="$(mktemp -dt "$(basename $0).XXXX")" DBLIST=() # Repos -- cgit v1.2.3 From 0811dea8b9c695ed9e67b22d389142956bd1cdd8 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 17:40:26 -0500 Subject: Fix quoting on arrays. --- config | 2 +- createrepos | 2 +- db-functions | 22 +++++++++++----------- db-move | 2 +- db-remove | 14 +++++++------- db-update | 16 ++++++++-------- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/config b/config index 0152b69..9d72ebd 100644 --- a/config +++ b/config @@ -12,7 +12,7 @@ USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jor # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) +PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/createrepos b/createrepos index 926c158..1840c83 100755 --- a/createrepos +++ b/createrepos @@ -5,4 +5,4 @@ source $(dirname $0)/config mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${ARCH_BASE}" "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" -$(dirname $0)/create-repo ${PKGREPOS[@]} +$(dirname $0)/create-repo "${PKGREPOS[@]}" diff --git a/db-functions b/db-functions index f56ca19..59e9c29 100644 --- a/db-functions +++ b/db-functions @@ -445,8 +445,8 @@ check_pkgrepos() { #usage: chk_license ${license[@]}" chk_license() { local l - for l in ${@}; do - in_array ${l} ${ALLOWED_LICENSES[@]} && return 0 + for l in "${@}"; do + in_array "${l}" "${ALLOWED_LICENSES[@]}" && return 0 done return 1 @@ -458,7 +458,7 @@ check_repo_permission() { [ ${#PKGREPOS[@]} -eq 0 ] && return 1 [ -z "${PKGPOOL}" ] && return 1 - in_array "${repo}" ${PKGREPOS[@]} || return 1 + in_array "${repo}" "${PKGREPOS[@]}" || return 1 [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 @@ -497,10 +497,10 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ - || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ - || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" + /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ + || error "repo-add ${repo}${DBEXT} ${pkgs[*]}" + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ + || error "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" } @@ -516,9 +516,9 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ - || error "repo-remove ${dbfile} ${pkgs[@]}" - /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ - || error "repo-remove ${filesfile} ${pkgs[@]}" + /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ + || error "repo-remove ${dbfile} ${pkgs[*]}" + /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ + || error "repo-remove ${filesfile} ${pkgs[*]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-move b/db-move index 010d941..69f24ac 100755 --- a/db-move +++ b/db-move @@ -66,7 +66,7 @@ for pkgbase in ${args[@]:2}; do if [ -f "${svnrepo_from}/PKGBUILD" ]; then if [ "${pkgarch}" == 'any' ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi diff --git a/db-remove b/db-remove index f0785e5..255aa32 100755 --- a/db-remove +++ b/db-remove @@ -10,24 +10,24 @@ fi repo="$1" arch="$2" -pkgbases=(${@:3}) +pkgbases=("${@:3}") if ! check_repo_permission $repo; then die "You don't have permission to remove packages from ${repo}" fi if [ "$arch" == "any" ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do +for tarch in "${tarches[@]}"; do repo_lock $repo $tarch || exit 1 done remove_pkgs=() -for pkgbase in ${pkgbases[@]}; do +for pkgbase in "${pkgbases[@]}"; do msg "Removing $pkgbase from [$repo]..." if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then @@ -36,11 +36,11 @@ for pkgbase in ${pkgbases[@]}; do warning "$pkgbase not found in ABS(libre)" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" - remove_pkgs[${#remove_pkgs[*]}]=$pkgbase + remove_pkgs+=("$pkgbase") fi done -for tarch in ${tarches[@]}; do - arch_repo_remove "${repo}" "${tarch}" ${remove_pkgs[@]} +for tarch in "${tarches[@]}"; do + arch_repo_remove "${repo}" "${tarch}" "${remove_pkgs[@]}" repo_unlock $repo $tarch done diff --git a/db-update b/db-update index 1fddb8a..cdde63b 100755 --- a/db-update +++ b/db-update @@ -15,14 +15,14 @@ if [ $? -ge 1 ]; then fi # TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo} ${pkgarch} || exit 1 done done # check if packages are valid -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then @@ -48,7 +48,7 @@ for repo in ${repos[@]}; do fi done -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do msg "Updating [${repo}]..." any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) for pkgarch in ${ARCHES[@]}; do @@ -69,16 +69,16 @@ for repo in ${repos[@]}; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi - add_pkgs[${#add_pkgs[*]}]=${pkgfile} + add_pkgs+=("${pkgfile}") done if [ ${#add_pkgs[@]} -ge 1 ]; then - arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} + arch_repo_add "${repo}" "${pkgarch}" "${add_pkgs[@]}" fi done done -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo} ${pkgarch} done done -- cgit v1.2.3 From 1d08899c3ac82047fe614a7694e5a7235f808cfd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 14:52:20 -0500 Subject: Remove extra local_config loads. Some files were loading local_config even if they did not use any settings from it. --- any-to-ours | 1 - db-cleanup | 1 - get-repos | 1 - 3 files changed, 3 deletions(-) diff --git a/any-to-ours b/any-to-ours index a1d6686..b927b82 100755 --- a/any-to-ours +++ b/any-to-ours @@ -8,7 +8,6 @@ trap_exit() { } source $(dirname $0)/config -source $(dirname $0)/local_config source $(dirname $0)/libremessages # From makepkg diff --git a/db-cleanup b/db-cleanup index 841800b..abad390 100755 --- a/db-cleanup +++ b/db-cleanup @@ -16,7 +16,6 @@ trap_exit() { } source $(dirname $0)/config -source $(dirname $0)/local_config source $(dirname $0)/libremessages # From makepkg diff --git a/get-repos b/get-repos index 796051c..92db9a2 100755 --- a/get-repos +++ b/get-repos @@ -10,7 +10,6 @@ trap_exit() { } source $(dirname $0)/config -source $(dirname $0)/local_config source $(dirname $0)/libremessages # From makepkg -- cgit v1.2.3 From b6e8ebd66d22abf5439485985a7851e768c71e8a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:04:43 -0500 Subject: Be very careful about using $0. --- any-to-ours | 4 ++-- create-repo | 8 ++++---- createrepos | 4 ++-- cron-jobs/ftpdir-cleanup | 4 ++-- cron-jobs/integrity-check | 4 ++-- cron-jobs/repo-sanity-check | 4 ++-- cron-jobs/sourceballs | 2 +- cron-jobs/sourceballs2 | 2 +- cron-jobs/update-abs-tarballs | 2 +- cron-jobs/update-web-db | 6 +++--- db-check-nonfree | 6 +++--- db-cleanup | 4 ++-- db-functions | 14 +++++++------- db-list-unsigned-packages | 8 ++++---- db-move | 6 +++--- db-remove | 6 +++--- db-repo-add | 6 +++--- db-repo-remove | 6 +++--- db-sync | 16 ++++++++-------- db-update | 6 +++--- get-repos | 6 +++--- migrate-repo | 2 +- mkrepo | 4 ++-- repo-restore-to-normal | 4 ++-- testing2x | 8 ++++---- yf-update | 12 ++++++------ 26 files changed, 77 insertions(+), 77 deletions(-) diff --git a/any-to-ours b/any-to-ours index b927b82..a1697c7 100755 --- a/any-to-ours +++ b/any-to-ours @@ -7,8 +7,8 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/create-repo b/create-repo index 58842c3..24b890d 100755 --- a/create-repo +++ b/create-repo @@ -1,11 +1,11 @@ #!/bin/bash # Creates repository structure -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -eq 0 ]; then - msg "Usage: $0 repo1 [repo2 ... repoX]" + msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" exit 1 fi @@ -21,4 +21,4 @@ for _repo in $@; do done done -msg "Don't forget to add them to the PKGREPOS array on $(dirname $0)/config" +msg "Don't forget to add them to the PKGREPOS array on %s/config" "$(dirname "$(readlink -e "$0")")" diff --git a/createrepos b/createrepos index 1840c83..a8f93e8 100755 --- a/createrepos +++ b/createrepos @@ -1,8 +1,8 @@ #!/bin/bash # Creates the repo structure defined in config -source $(dirname $0)/config +source "$(dirname "$(readlink -e "$0")")/config" mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${ARCH_BASE}" "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" -$(dirname $0)/create-repo "${PKGREPOS[@]}" +"$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 83e6e17..8d691b5 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" clean_pkg() { local pkg diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index d4f9694..05a56a5 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname $0)" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../db-functions" . "${dirname}/../config" @@ -8,7 +8,7 @@ dirname="$(dirname $0)" script_lock if [ $# -ne 1 ]; then - die "usage: $(basename $0) " + die "usage: ${0##*/} " fi mailto=$1 diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 1ba90a6..2aa7892 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" # Traverse all repos for _repo in ${PKGREPOS[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index ee074bd..73d8432 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname $(readlink -e $0))" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../db-functions" . "${dirname}/../config" pushd "${WORKDIR}" >/dev/null diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 5644268..eb46579 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -4,7 +4,7 @@ # Makepkg --allsource every package # Remove the old sourceballs -dirname="$(dirname $(readlink -e $0))" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../db-functions" . "${dirname}/../config" . "${MAKEPKGCONF}" diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs index 824ac34..901cc4b 100755 --- a/cron-jobs/update-abs-tarballs +++ b/cron-jobs/update-abs-tarballs @@ -1,6 +1,6 @@ #!/bin/bash -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../config" rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 6ced4c1..825eea6 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" # setup paths SPATH="/srv/http/archweb" @@ -14,7 +14,7 @@ REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'te LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform -cmd="$(basename $0)" +cmd="${0##*/}" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then die "Invalid command name '$cmd' specified!" fi diff --git a/db-check-nonfree b/db-check-nonfree index ecad3b9..661daa6 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -ge 1 ]; then - warning "Calling $(basename $0) with a specific repository is not supported" + warning "Calling ${0##*/} with a specific repository is not supported" exit 1 fi diff --git a/db-cleanup b/db-cleanup index abad390..57ef36e 100755 --- a/db-cleanup +++ b/db-cleanup @@ -15,8 +15,8 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/db-functions b/db-functions index 59e9c29..26b23e4 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -dt "$(basename $0).XXXXXXXXXX") +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") LOCKS=() # check if messages are to be printed using color @@ -85,10 +85,10 @@ get_full_version() { } script_lock() { - local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - error "Script $(basename $0) is already locked by $_owner." + error "Script ${0##*/} is already locked by $_owner." exit 1 else set_umask @@ -97,9 +97,9 @@ script_lock() { } script_unlock() { - local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if [ ! -d "$LOCKDIR" ]; then - warning "Script $(basename $0) was not locked!" + warning "Script ${0##*/} was not locked!" restore_umask return 1 else @@ -123,8 +123,8 @@ cleanup() { repo_unlock $repo $arch fi done - if [ -d "$TMPDIR/.scriptlock.$(basename $0)" ]; then - msg "Removing left over lock from $(basename $0)" + if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then + msg "Removing left over lock from ${0##*/}" script_unlock fi rm -rf "$WORKDIR" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 3b5a5bd..985d1c0 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -20,11 +20,11 @@ set -e # unsigned packages available for architecture $1 and specified for # architecture $2 (usually $1 or any, default is to list all). -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 1 ]; then - msg "usage: $(basename $0) " + msg "usage: ${0##*/} " exit 1 fi @@ -34,5 +34,5 @@ shift for repo in ${PKGREPOS[@]} do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - [ -f "$db" ] && "$(dirname $0)/db-list-unsigned-packages.py" "$repo" "$@" < "$db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" done diff --git a/db-move b/db-move index 69f24ac..a3f1532 100755 --- a/db-move +++ b/db-move @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-remove b/db-remove index 255aa32..4e45881 100755 --- a/db-remove +++ b/db-remove @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-repo-add b/db-repo-add index b83fb77..2b7894a 100755 --- a/db-repo-add +++ b/db-repo-add @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-repo-remove b/db-repo-remove index 4f04ed1..09e34da 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-sync b/db-sync index e8481d6..c4d495e 100755 --- a/db-sync +++ b/db-sync @@ -21,7 +21,7 @@ ${VERBOSE} && extra="-v" # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/$0.$$.cache + mkdir -p ${TMPDIR}/${0##*/}.$$.cache # Exclude everything but db files rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ @@ -30,7 +30,7 @@ get_repos() { --include="*${FILESEXT}" \ --exclude="*" \ --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache } get_repo_content() { @@ -48,7 +48,7 @@ get_blacklist() { # repo # arch get_repo_file() { - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" + echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" } # Process the databases and get the libre packages @@ -118,7 +118,7 @@ init() { rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ - ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ + ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ ${FTP_BASE}/${_repo}/os/${_arch}/ # Cleanup @@ -176,9 +176,9 @@ trap_exit() { } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E @@ -189,4 +189,4 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init -rm -r ${TMPDIR}/$0.$$.cache +rm -r ${TMPDIR}/${0##*/}.$$.cache diff --git a/db-update b/db-update index cdde63b..86faec3 100755 --- a/db-update +++ b/db-update @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -ge 1 ]; then - warning "Calling $(basename $0) with a specific repository is no longer supported" + warning "Calling ${0##*/} with a specific repository is no longer supported" exit 1 fi diff --git a/get-repos b/get-repos index 92db9a2..a477cf2 100755 --- a/get-repos +++ b/get-repos @@ -9,8 +9,8 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E @@ -19,7 +19,7 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -TMPDIR="$(mktemp -dt "$(basename $0).XXXX")" +TMPDIR="$(mktemp -dt "${0##*/}.XXXX")" DBLIST=() # Repos diff --git a/migrate-repo b/migrate-repo index 2e44adb..751d5bd 100755 --- a/migrate-repo +++ b/migrate-repo @@ -1,6 +1,6 @@ #!/bin/bash -source $(dirname $0)/config +source "$(dirname "$(readlink -e "$0")")/config" #dryrun="--dry-run" diff --git a/mkrepo b/mkrepo index 5f704cc..10d014b 100755 --- a/mkrepo +++ b/mkrepo @@ -3,8 +3,8 @@ # License: GPLv3+ # Description: A script to quickly create new [repos] -source $(dirname $0)/config -source $(dirname $0)/local_config +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" # TODO it would be simpler to expand arrays to {element1,element2,etc} for repo in $@; do diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 9463731..3636920 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore PKGREPOS=(community) diff --git a/testing2x b/testing2x index 54cae11..14a45de 100755 --- a/testing2x +++ b/testing2x @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 1 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi @@ -54,7 +54,7 @@ for repo in 'core' 'extra'; do repo_unlock ${repo} ${pkgarch} done if [ -n "${pkgs[${repo}]}" ]; then - "$(dirname $0)/db-move" 'testing' "${repo}" ${pkgs[${repo}]} + "$(dirname "$(readlink -e "$0")")/db-move" 'testing' "${repo}" ${pkgs[${repo}]} fi done diff --git a/yf-update b/yf-update index 9c2131e..3bce4aa 100755 --- a/yf-update +++ b/yf-update @@ -1,17 +1,17 @@ #!/bin/bash -source $(dirname $0)/local_config -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) -last_bl_mtime=$(cat $(dirname $0)/yftime) +last_bl_mtime=$(cat "$(dirname "$(readlink -e "$0")")/yftime") if [ $blacklist_mtime -gt $last_bl_mtime ]; then - pushd $(dirname $0)/yf + pushd "$(dirname "$(readlink -e "$0")")/yf" makepkg -f find . -name "*${PKGEXT}" -exec mv {} ${STAGING}/libre \; popd - echo ${blacklist_mtime} > $(dirname $0)/yftime + echo ${blacklist_mtime} > "$(dirname "$(readlink -e "$0")")/yftime" msg2 "built and staged" else msg2 "nothing to do" -- cgit v1.2.3 From 8650ad4fdef09b1244ba138531b6a1e2bbd0975b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 18:00:12 -0500 Subject: Get rid of $ARCH_BASE; these days it is the same as $FTP_BASE --- config | 1 - createrepos | 2 +- cron-jobs/sourceballs | 10 +++++----- cron-jobs/sourceballs2 | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/config b/config index 9d72ebd..c3f9a55 100644 --- a/config +++ b/config @@ -1,6 +1,5 @@ #!/bin/bash # as a hint to text editors FTP_BASE="/srv/http/repo/public" -ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch diff --git a/createrepos b/createrepos index a8f93e8..8da2455 100755 --- a/createrepos +++ b/createrepos @@ -3,6 +3,6 @@ source "$(dirname "$(readlink -e "$0")")/config" -mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${ARCH_BASE}" "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" "$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 73d8432..4debcee 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -96,7 +96,7 @@ for repo in ${PKGREPOS[@]}; do pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -132,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index eb46579..bbe227d 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -17,7 +17,7 @@ script_lock renice +10 -p $$ > /dev/null # Create a list of all available source package file names -find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" pushd "${SVNREPO}" >/dev/null -- cgit v1.2.3 From 62142bcbac5f457b2172c7311741f90fedf0c776 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 15:20:16 -0500 Subject: fix comments, indentation --- cron-jobs/sourceballs | 12 ++++++------ db-functions | 14 +++++++------- db-move | 2 +- db-remove | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4debcee..6393472 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -59,11 +59,11 @@ for repo in ${PKGREPOS[@]}; do pkgarch=${pkginfo[2]} pkglicense=(${pkginfo[@]:3}) - # Should this packages be skipped? + # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi - # Commenting out, we'll sourceball everything + # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package # if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then # continue @@ -83,10 +83,10 @@ for repo in ${PKGREPOS[@]}; do #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 - # If it's on official repos, nor [libre], nor [libre-testing] - cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ - cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ - cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 + # If it's on official repos, nor [libre], nor [libre-testing] + cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" continue diff --git a/db-functions b/db-functions index 26b23e4..e681504 100644 --- a/db-functions +++ b/db-functions @@ -375,7 +375,7 @@ check_splitpkgs() { for pkgfile in ${pkgfiles[@]}; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" - msg2 "Checking $_pkgbase" + msg2 "Checking $_pkgbase" local _pkgname="$(getpkgname ${pkgfile})" local _pkgarch="$(getpkgarch ${pkgfile})" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" @@ -384,14 +384,14 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r ${SVNREPO}/libre/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r ${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/libre/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 [[ $? -ge 1 ]] && { - echo "Failed $_pkgbase-$_pkgver-$_pkgarch" - return 1 - } + echo "Failed $_pkgbase-$_pkgver-$_pkgarch" + return 1 + } fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) diff --git a/db-move b/db-move index a3f1532..41ef4c1 100755 --- a/db-move +++ b/db-move @@ -24,7 +24,7 @@ for pkgarch in ${ARCHES[@]}; do repo_lock ${repo_from} ${pkgarch} || exit 1 done -# No idea why we loop twice... -- fauno +# First loop is to check that all necessary files exist for pkgbase in ${args[@]:2}; do for pkgarch in ${ARCHES[@]} 'any'; do svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" diff --git a/db-remove b/db-remove index 4e45881..2a12dba 100755 --- a/db-remove +++ b/db-remove @@ -30,8 +30,8 @@ remove_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing $pkgbase from [$repo]..." - if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then - remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) + if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then + remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) else warning "$pkgbase not found in ABS(libre)" warning "Removing only $pkgbase from the repo" -- cgit v1.2.3 From eefb787983d2608511214bcd682f3d8271bac60c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 16:44:51 -0500 Subject: Normalize to load config then local_config then db-functions --- create-repo | 2 +- cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/integrity-check | 2 +- cron-jobs/repo-sanity-check | 2 +- cron-jobs/sourceballs | 2 +- cron-jobs/sourceballs2 | 2 +- cron-jobs/update-web-db | 2 +- db-check-nonfree | 2 +- db-list-unsigned-packages | 2 +- db-move | 2 +- db-remove | 2 +- db-repo-add | 2 +- db-repo-remove | 2 +- db-update | 2 +- repo-restore-to-normal | 2 +- testing2x | 2 +- yf-update | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/create-repo b/create-repo index 24b890d..21a2a9c 100755 --- a/create-repo +++ b/create-repo @@ -1,8 +1,8 @@ #!/bin/bash # Creates repository structure -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -eq 0 ]; then msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 8d691b5..b290138 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/../db-functions" . "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" clean_pkg() { local pkg diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 05a56a5..86a8f1d 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -2,8 +2,8 @@ dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" script_lock diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 2aa7892..ee4c061 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname "$(readlink -e "$0")")/../db-functions" . "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" # Traverse all repos for _repo in ${PKGREPOS[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6393472..c9aadba 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,8 +1,8 @@ #!/bin/bash dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" pushd "${WORKDIR}" >/dev/null script_lock diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index bbe227d..1432bdf 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -5,8 +5,8 @@ # Remove the old sourceballs dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" . "${MAKEPKGCONF}" pushd "${WORKDIR}" >/dev/null diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 825eea6..713e75e 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/../db-functions" . "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" # setup paths SPATH="/srv/http/archweb" diff --git a/db-check-nonfree b/db-check-nonfree index 661daa6..5cb7f6f 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is not supported" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 985d1c0..5105096 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -20,8 +20,8 @@ set -e # unsigned packages available for architecture $1 and specified for # architecture $2 (usually $1 or any, default is to list all). -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 1 ]; then msg "usage: ${0##*/} " diff --git a/db-move b/db-move index 41ef4c1..eed48eb 100755 --- a/db-move +++ b/db-move @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-remove b/db-remove index 2a12dba..46585ad 100755 --- a/db-remove +++ b/db-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-repo-add b/db-repo-add index 2b7894a..a6355a1 100755 --- a/db-repo-add +++ b/db-repo-add @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-repo-remove b/db-repo-remove index 09e34da..7077d62 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-update b/db-update index 86faec3..e3da232 100755 --- a/db-update +++ b/db-update @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is no longer supported" diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 3636920..3fe4816 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore PKGREPOS=(community) diff --git a/testing2x b/testing2x index 14a45de..b6828fc 100755 --- a/testing2x +++ b/testing2x @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 1 ]; then msg "usage: ${0##*/} ..." diff --git a/yf-update b/yf-update index 3bce4aa..b6dff14 100755 --- a/yf-update +++ b/yf-update @@ -1,6 +1,6 @@ #!/bin/bash -source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) -- cgit v1.2.3 From 9d9116bd23720cf6c5b27aa39e5cc4c71de1fb26 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:27:07 -0500 Subject: Fix some array quoting. --- create-repo | 4 ++-- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 22 +++++++++++----------- cron-jobs/ftpdir-cleanup | 10 +++++----- cron-jobs/repo-sanity-check | 4 ++-- cron-jobs/sourceballs | 28 ++++++++++++++-------------- cron-jobs/sourceballs2 | 4 ++-- db-check-nonfree | 22 +++++++++++----------- db-functions | 26 +++++++++++++------------- db-list-unsigned-packages | 2 +- db-move | 28 ++++++++++++++-------------- db-update | 8 ++++---- get-repos | 6 +++--- repo-restore-to-normal | 6 +++--- 13 files changed, 85 insertions(+), 85 deletions(-) diff --git a/create-repo b/create-repo index 21a2a9c..1ec9798 100755 --- a/create-repo +++ b/create-repo @@ -10,12 +10,12 @@ if [ $# -eq 0 ]; then fi msg "Creating repos..." -for _repo in $@; do +for _repo in "$@"; do msg2 "Creating [${_repo}]" mkdir -p "${FTP_BASE}/staging/${_repo}" || \ error "Failed creating staging dir" - for _arch in ${ARCHES[@]}; do + for _arch in "${ARCHES[@]}"; do mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ error "Failed creating ${_arch} dir" done diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 3f92169..c8d8618 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -6,18 +6,18 @@ exit() { return; } splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') -variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' "${splitpkg_overrides[@]}") readonly -a variables splitpkg_overrides backup_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" eval "${indirect}=(\${$var[@]})" done } restore_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then eval "${var}=(\${$indirect[@]})" @@ -42,31 +42,31 @@ print_info() { if [ -n "$arch" ]; then echo "%ARCH%" - for i in ${arch[@]}; do echo $i; done + for i in "${arch[@]}"; do echo $i; done echo "" fi if [ -n "$depends" ]; then echo "%DEPENDS%" - for i in ${depends[@]}; do + for i in "${depends[@]}"; do echo $i done echo "" fi if [ -n "$makedepends" ]; then echo "%MAKEDEPENDS%" - for i in ${makedepends[@]}; do + for i in "${makedepends[@]}"; do echo $i done echo "" fi if [ -n "$conflicts" ]; then echo "%CONFLICTS%" - for i in ${conflicts[@]}; do echo $i; done + for i in "${conflicts[@]}"; do echo $i; done echo "" fi if [ -n "$provides" ]; then echo "%PROVIDES%" - for i in ${provides[@]}; do echo $i; done + for i in "${provides[@]}"; do echo $i; done echo "" fi } @@ -75,7 +75,7 @@ source_pkgbuild() { ret=0 dir=$1 pkgbuild=$dir/PKGBUILD - for var in ${variables[@]}; do + for var in "${variables[@]}"; do unset ${var} done source $pkgbuild &>/dev/null || ret=$? @@ -88,7 +88,7 @@ source_pkgbuild() { if [ "${#pkgname[@]}" -gt "1" ]; then pkgbase=${pkgbase:-${pkgname[0]}} - for pkg in ${pkgname[@]}; do + for pkg in "${pkgname[@]}"; do if [ "$(type -t package_${pkg})" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" return 1 @@ -98,7 +98,7 @@ source_pkgbuild() { while IFS= read -r line; do var=${line%%=*} var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters - for realvar in ${variables[@]}; do + for realvar in "${variables[@]}"; do if [ "$var" == "$realvar" ]; then eval $line break diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index b290138..6455ed7 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -25,8 +25,8 @@ clean_pkg() { ${CLEANUP_DRYRUN} && warning 'dry run mode is active' -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi @@ -69,7 +69,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then fi # cleanup of legacy $repo/os/any directories -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then continue fi @@ -103,8 +103,8 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_unlock ${repo} ${arch} done done diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index ee4c061..9d351df 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -5,7 +5,7 @@ . "$(dirname "$(readlink -e "$0")")/../db-functions" # Traverse all repos -for _repo in ${PKGREPOS[@]}; do +for _repo in "${PKGREPOS[@]}"; do msg "Cleaning up [${_repo}]" # Find all pkgnames on this repo's abs @@ -19,7 +19,7 @@ for _repo in ${PKGREPOS[@]}; do >/dev/null 2>&1 # also cleanup package functions - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${pkg} >/dev/null 2>&1 done diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index c9aadba..8171980 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -7,8 +7,8 @@ pushd "${WORKDIR}" >/dev/null script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_lock ${repo} ${arch} || exit 1 done done @@ -18,8 +18,8 @@ renice +10 -p $$ > /dev/null # Create a readable file for each repo with the following format # - [ ] -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do # Repo does not exist; skip it if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue @@ -39,8 +39,8 @@ for repo in ${PKGREPOS[@]}; do done | sort -u > "${WORKDIR}/db-${repo}" done -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_unlock ${repo} ${arch} done done @@ -49,15 +49,15 @@ done find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do newpkgs=() failedpkgs=() while read line; do - pkginfo=(${line}) + pkginfo=("${line}") pkgbase=${pkginfo[0]} pkgver=${pkginfo[1]} pkgarch=${pkginfo[2]} - pkglicense=(${pkginfo[@]:3}) + pkglicense=("${pkginfo[@]:3}") # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then @@ -88,7 +88,7 @@ for repo in ${PKGREPOS[@]}; do cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue fi @@ -109,13 +109,13 @@ for repo in ${PKGREPOS[@]}; do if [ ${#newpkgs[@]} -ge 1 ]; then msg "Adding source packages for [${repo}]..." - for new_pkg in ${newpkgs[@]}; do + for new_pkg in "${newpkgs[@]}"; do msg2 "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then msg "Failed to create source packages for [${repo}]..." - for failed_pkg in ${failedpkgs[@]}; do + for failed_pkg in "${failedpkgs[@]}"; do msg2 "${failed_pkg}" done fi @@ -129,7 +129,7 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" @@ -141,7 +141,7 @@ fi old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 1432bdf..2a26e6a 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -21,7 +21,7 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do msg "Sourceballing [${repo}]" pushd $repo >/dev/null @@ -40,7 +40,7 @@ for repo in ${PKGREPOS[@]}; do unset build package url pkgdesc source md5sums depends makedepends \ optdepends license arch options check mksource - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${_pkg} >/dev/null 2>&1 done diff --git a/db-check-nonfree b/db-check-nonfree index 5cb7f6f..6e2dc17 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -9,16 +9,16 @@ if [ $# -ge 1 ]; then fi # TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo} ${pkgarch} || exit 1 done done msg "Check nonfree in repo:" nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) -for repo in ${ARCHREPOS[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${ARCHREPOS[@]}"; do + for pkgarch in "${ARCHES[@]}"; do msg2 "$repo $pkgarch" if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue @@ -27,20 +27,20 @@ for repo in ${ARCHREPOS[@]}; do unset cleanpkgs cleanpkgs=() dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in ${dbpkgs[@]}; do - if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs+=(${pkgname}) + for pkgname in "${dbpkgs[@]}"; do + if in_array "${pkgname}" "${nonfree[@]}"; then + cleanpkgs+=("${pkgname}") fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: ${cleanpkgs[@]}" - arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + msg2 "Nonfree: ${cleanpkgs[*]}" + arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" fi done done -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo} ${pkgarch} done done diff --git a/db-functions b/db-functions index e681504..83e0613 100644 --- a/db-functions +++ b/db-functions @@ -115,7 +115,7 @@ cleanup() { local arch trap - EXIT INT QUIT TERM - for l in ${LOCKS[@]}; do + for l in "${LOCKS[@]}"; do repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then @@ -299,7 +299,7 @@ getpkgfiles() { exit 1 fi - for f in ${@}; do + for f in "${@}"; do if [ ! -f "${f}" ]; then error "Package ${f} not found!" exit 1 @@ -309,7 +309,7 @@ getpkgfiles() { fi done - echo ${@} + echo "${@}" } check_pkgfile() { @@ -322,7 +322,7 @@ check_pkgfile() { local pkgarch="$(getpkgarch ${pkgfile})" [ $? -ge 1 ] && return 1 - in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 + in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 if echo "$(basename ${pkgfile})" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then return 0 @@ -343,7 +343,7 @@ check_pkgsvn() { [ $? -ge 1 ] && return 1 local repo="${2}" - in_array "${repo}" ${PKGREPOS[@]} || return 1 + in_array "${repo}" "${PKGREPOS[@]}" || return 1 if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" @@ -356,7 +356,7 @@ check_pkgsvn() { [ "${svnver}" == "${_pkgver}" ] || return 1 local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) - in_array "${_pkgname}" ${svnnames[@]} || return 1 + in_array "${_pkgname}" "${svnnames[@]}" || return 1 return 0 } @@ -364,7 +364,7 @@ check_pkgsvn() { check_splitpkgs() { local repo="${1}" shift - local pkgfiles=(${@}) + local pkgfiles=("${@}") local pkgfile local pkgdir local svnname @@ -372,7 +372,7 @@ check_splitpkgs() { mkdir -p "${WORKDIR}/check_splitpkgs/" pushd "${WORKDIR}/check_splitpkgs" >/dev/null - for pkgfile in ${pkgfiles[@]}; do + for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" msg2 "Checking $_pkgbase" @@ -395,7 +395,7 @@ check_splitpkgs() { fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) - for svnname in ${svnnames[@]}; do + for svnname in "${svnnames[@]}"; do echo "${svnname}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" done done @@ -430,8 +430,8 @@ check_pkgrepos() { local repo local arch - for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do + for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 [ -f "${FTP_BASE}/${repo}/os/${arch}/$(basename ${pkgfile})" ] && return 1 @@ -493,7 +493,7 @@ set_repo_permission() { arch_repo_add() { local repo=$1 local arch=$2 - local pkgs=(${@:3}) + local pkgs=("${@:3}") # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null @@ -508,7 +508,7 @@ arch_repo_add() { arch_repo_remove() { local repo=$1 local arch=$2 - local pkgs=(${@:3}) + local pkgs=("${@:3}") local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 5105096..f593686 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -31,7 +31,7 @@ fi arch=$1 shift -for repo in ${PKGREPOS[@]} +for repo in "${PKGREPOS[@]}" do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" diff --git a/db-move b/db-move index eed48eb..a6abb8f 100755 --- a/db-move +++ b/db-move @@ -8,7 +8,7 @@ if [ $# -lt 3 ]; then exit 1 fi -args=(${@}) +args=("${@}") repo_from="${args[0]}" repo_to="${args[1]}" ftppath_from="${FTP_BASE}/${repo_from}/os/" @@ -19,14 +19,14 @@ if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then fi # TODO: this might lock too much (architectures) -for pkgarch in ${ARCHES[@]}; do +for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo_to} ${pkgarch} || exit 1 repo_lock ${repo_from} ${pkgarch} || exit 1 done # First loop is to check that all necessary files exist -for pkgbase in ${args[@]:2}; do - for pkgarch in ${ARCHES[@]} 'any'; do +for pkgbase in "${args[@]:2}"; do + for pkgarch in "${ARCHES[@]}" 'any'; do svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) @@ -40,13 +40,13 @@ for pkgbase in ${args[@]:2}; do fi if [ "${pkgarch}" == 'any' ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi - for pkgname in ${pkgnames[@]}; do - for tarch in ${tarches[@]}; do + for pkgname in "${pkgnames[@]}"; do + for tarch in "${tarches[@]}"; do getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null done done @@ -60,8 +60,8 @@ msg "Moving packages from [${repo_from}] to [${repo_to}]..." declare -A add_pkgs declare -A remove_pkgs -for pkgbase in ${args[@]:2}; do - for pkgarch in ${ARCHES[@]} 'any'; do +for pkgbase in "${args[@]:2}"; do + for pkgarch in "${ARCHES[@]}" 'any'; do svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" if [ -f "${svnrepo_from}/PKGBUILD" ]; then @@ -70,12 +70,12 @@ for pkgbase in ${args[@]:2}; do else tarches=("${pkgarch}") fi - msg2 "${pkgbase} ($(echo ${tarches[@]}))" + msg2 "${pkgbase} (${tarches[*]})" pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - for pkgname in ${pkgnames[@]}; do - for tarch in ${tarches[@]}; do + for pkgname in "${pkgnames[@]}"; do + for tarch in "${tarches[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) pkgfile=$(basename "${pkgpath}") @@ -96,14 +96,14 @@ for pkgbase in ${args[@]:2}; do done done -for tarch in ${ARCHES[@]}; do +for tarch in "${ARCHES[@]}"; do if [ -n "${add_pkgs[${tarch}]}" ]; then arch_repo_add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} arch_repo_remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} fi done -for pkgarch in ${ARCHES[@]}; do +for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo_from} ${pkgarch} repo_unlock ${repo_to} ${pkgarch} done diff --git a/db-update b/db-update index e3da232..60cc6cd 100755 --- a/db-update +++ b/db-update @@ -28,7 +28,7 @@ for repo in "${repos[@]}"; do if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then die "You don't have permission to update packages in ${repo}" fi - for pkg in ${pkgs[@]}; do + for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then die "Package ${repo}/$(basename ${pkg}) is a symbolic link" fi @@ -40,7 +40,7 @@ for repo in "${repos[@]}"; do fi done # This is fucking obnoxious - #if ! check_splitpkgs ${repo} ${pkgs[@]}; then + #if ! check_splitpkgs ${repo} "${pkgs[@]}"; then # die "Missing split packages for ${repo}" #fi else @@ -51,10 +51,10 @@ done for repo in "${repos[@]}"; do msg "Updating [${repo}]..." any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) - for pkgarch in ${ARCHES[@]}; do + for pkgarch in "${ARCHES[@]}"; do add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) - for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do + for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="$(basename ${pkg})" msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run diff --git a/get-repos b/get-repos index a477cf2..b98d601 100755 --- a/get-repos +++ b/get-repos @@ -23,8 +23,8 @@ TMPDIR="$(mktemp -dt "${0##*/}.XXXX")" DBLIST=() # Repos -for _repo in ${PKGREPOS[@]}; do - for _arch in ${ARCHES[@]}; do +for _repo in "${PKGREPOS[@]}"; do + for _arch in "${ARCHES[@]}"; do DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${FILESEXT}") done done @@ -35,7 +35,7 @@ wget --directory-prefix=${TMPDIR} \ --no-verbose \ --force-directories \ --no-host-directories \ - ${DBLIST[@]} || true + "${DBLIST[@]}" || true # Always return true, some databases are expect to be missing # Create the arches regexp arch1|arch2 diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 3fe4816..063aacf 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -12,7 +12,7 @@ PKGREPOS=(community) # sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/")) # Traverse all repos -for _repo in ${PKGREPOS[@]}; do +for _repo in "${PKGREPOS[@]}"; do msg "Restoring [${_repo}]" # Find all pkgnames on this repo's abs @@ -27,7 +27,7 @@ for _repo in ${PKGREPOS[@]}; do >/dev/null 2>&1 # also cleanup package functions - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${pkg} >/dev/null 2>&1 # this fills the on_abs array echo ${_pkg}-${pkgver}-${pkgrel} @@ -49,7 +49,7 @@ for _repo in ${PKGREPOS[@]}; do msg2 "Restoring the following packages:" # plain "$(echo ${restore[@]} | tr ' ' "\n")" - for _pkg in ${on_abs[@]}; do + for _pkg in "${on_abs[@]}"; do find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \; done -- cgit v1.2.3 From aefd5a4d961e3eca3326c90ba266d04b325c1f15 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 20:36:20 -0500 Subject: db-sync: use tab indent --- db-sync | 263 ++++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 131 insertions(+), 132 deletions(-) diff --git a/db-sync b/db-sync index c4d495e..0e07757 100755 --- a/db-sync +++ b/db-sync @@ -21,158 +21,157 @@ ${VERBOSE} && extra="-v" # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/${0##*/}.$$.cache -# Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache + mkdir -p ${TMPDIR}/${0##*/}.$$.cache + # Exclude everything but db files + rsync ${extra} -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache } get_repo_content() { -# Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u + # Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u } # Prints blacklisted packages get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" + cut -d ':' -f 1 "${BLACKLIST_FILE}" } # repo # arch get_repo_file() { - echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" + echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" } # Process the databases and get the libre packages init() { -# Get the blacklisted packages - blacklist=($(get_blacklist)) -# Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - -# Sync the repos databases - get_repos - -# Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - -# Remove blacklisted packages and count them -# TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from .db database..." - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" - msg2 "Removing blacklisted packages from .files database..." - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" - -# Get db contents - db=($(get_repo_content ${db_file})) - - msg2 "Process clean db for syncing..." - -# Create a whitelist, add * wildcard to end -# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if -# we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - -# Sync excluding everything but whitelist -# We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Cleanup - unset db - done - done - - - msg "Syncing package pool" -# Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - -# Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - -# Cleanup - unset blacklist whitelists _arch _repo repo_file + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from .db database..." + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" + msg2 "Removing blacklisted packages from .files database..." + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" + + # Get db contents + db=($(get_repo_content ${db_file})) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync ${extra} -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done + + # Cleanup + unset blacklist whitelists _arch _repo repo_file } trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } -- cgit v1.2.3 From d3afe9bb766bbdfaab1474d21fd5e28f0a356039 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 20:49:48 -0500 Subject: Avoid using $(basename $var) , use ${var##*/} instead --- abslibre | 2 +- cron-jobs/ftpdir-cleanup | 2 +- db-functions | 10 +++++----- db-move | 2 +- db-update | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/abslibre b/abslibre index fa777ac..a5733f0 100755 --- a/abslibre +++ b/abslibre @@ -96,7 +96,7 @@ sync_pre_mips64el() { # Create .abs.tar.gz tarballs create_tarballs() { for repo in ${ABSLIBRE}/{i686,x86_64}/*; do - baserepo=$(basename $repo) + baserepo=${repo##*/} arch=$(basename $(dirname $repo)) # Remove the old one diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 6455ed7..cb14382 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -16,7 +16,7 @@ clean_pkg() { if [ -e "$pkg.sig" ]; then mv -f "$pkg.sig" "$CLEANUP_DESTDIR" fi - touch "${CLEANUP_DESTDIR}/$(basename ${pkg})" + touch "${CLEANUP_DESTDIR}/${pkg##*/}" fi done fi diff --git a/db-functions b/db-functions index 83e0613..52ad8bb 100644 --- a/db-functions +++ b/db-functions @@ -324,7 +324,7 @@ check_pkgfile() { in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 - if echo "$(basename ${pkgfile})" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then + if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then return 0 else return 1 @@ -425,8 +425,8 @@ check_pkgrepos() { [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/$(basename ${pkgfile})" ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/$(basename ${pkgfile}).sig" ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 local repo local arch @@ -434,8 +434,8 @@ check_pkgrepos() { for arch in "${ARCHES[@]}"; do [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/$(basename ${pkgfile})" ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/$(basename ${pkgfile}).sig" ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}.sig" ] && return 1 done done diff --git a/db-move b/db-move index a6abb8f..f1c3dea 100755 --- a/db-move +++ b/db-move @@ -77,7 +77,7 @@ for pkgbase in "${args[@]:2}"; do for pkgname in "${pkgnames[@]}"; do for tarch in "${tarches[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) - pkgfile=$(basename "${pkgpath}") + pkgfile="${pkgpath##*/}" # copy package to pool if needed # TODO: can be removed once every package has been moved to the package pool diff --git a/db-update b/db-update index 60cc6cd..99c8be5 100755 --- a/db-update +++ b/db-update @@ -30,13 +30,13 @@ for repo in "${repos[@]}"; do fi for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then - die "Package ${repo}/$(basename ${pkg}) is a symbolic link" + die "Package ${repo}/${pkg##*/} is a symbolic link" fi if ! check_pkgfile "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" + die "Package ${repo}/${pkg##*/} is not consistent with its meta data" fi if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + die "Package ${repo}/${pkg##*/} already exists in another repository" fi done # This is fucking obnoxious @@ -55,7 +55,7 @@ for repo in "${repos[@]}"; do add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do - pkgfile="$(basename ${pkg})" + pkgfile="${pkg##*/}" msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run if [ -f "${pkg}" ]; then -- cgit v1.2.3 From a1fe8b8d0b49b9285189b1e8e2b5858b0fa4e2aa Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:13:03 -0500 Subject: remove unused variables from local_config --- local_config | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/local_config b/local_config index 8f3946a..9b4415e 100644 --- a/local_config +++ b/local_config @@ -1,4 +1,7 @@ -# Mirror options +#/bin/bash # as a hint to text editors +_paraboladir=/srv/http/repo/public + +# db-sync #mirror="mirrors.uk2.net" mirror="mirrors.kernel.org" #mirror="mirror.umd.edu" @@ -7,25 +10,11 @@ mirror="mirrors.kernel.org" #mirror="mirror.de.leaseweb.net" mirrorpath="archlinux" -# Directories: they should end without / -paraboladir=/srv/http/repo/public -tempdir=/tmp -archdb=${tempdir}/db -docs_dir=${paraboladir}/docs -repodir=${paraboladir} -licenses_dir=${docs_dir}/pending_licenses -# End Directories - -# Files -logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log -rsout_file=${tempdir}/rsout -rsync_not_needed=${tempdir}/rsync_not_needed - -rsync_blacklist=${docs_dir}/rsyncBlacklist +# mkrepo +repodir=${_paraboladir} -blacklist=${docs_dir}/blacklist.txt -whitelist=${docs_dir}/whitelist.txt +# yf-update +blacklist=${_paraboladir}/docs/blacklist.txt +whitelist=${_paraboladir}/docs/whitelist.txt -# Rsync commands -rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " -rsync_update_command="rsync -vrptgoL --exclude='*.abs.tar.*' --no-motd " +unset _paraboladir -- cgit v1.2.3 From 93eb77cbad072a25b9aa5542ce7fa59746ca9284 Mon Sep 17 00:00:00 2001 From: aurelien Date: Sat, 11 Jan 2014 12:39:37 +0100 Subject: modification on db-update for mail --- db-update | 1 + 1 file changed, 1 insertion(+) diff --git a/db-update b/db-update index 99c8be5..73d5d3f 100755 --- a/db-update +++ b/db-update @@ -91,5 +91,6 @@ while read -r file; do else mkdir -p -- "${pub%/*}" mv -vn "$file" "$pub" + echo "At `date`, script \"`basename $0`\" mailed to "maintenance@lists.parabolagnulinux.org"." fi done < <(find other sources -type f) -- cgit v1.2.3 From 2d217815fcfe05152272c20e49e4e69927d04a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 15 Jan 2014 19:12:38 -0200 Subject: revert to old db-sync until fix it, rename lastest db-sync to db-sync.orig --- db-sync | 265 +++++++++++++++++++++++++++++------------------------------ db-sync.orig | 191 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+), 135 deletions(-) create mode 100755 db-sync.orig diff --git a/db-sync b/db-sync index 0e07757..81dee24 100755 --- a/db-sync +++ b/db-sync @@ -21,163 +21,158 @@ ${VERBOSE} && extra="-v" # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/${0##*/}.$$.cache - # Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache + mkdir -p ${TMPDIR}/$0.$$.cache +# Exclude everything but db files + rsync ${extra} -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache } get_repo_content() { - # Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u +# Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u } # Prints blacklisted packages get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" + cut -d ':' -f 1 "${BLACKLIST_FILE}" } # repo # arch get_repo_file() { - echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" +# shopt -s nullglob + + echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}" } # Process the databases and get the libre packages init() { - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from .db database..." - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" - msg2 "Removing blacklisted packages from .files database..." - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" - - # Get db contents - db=($(get_repo_content ${db_file})) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - # Cleanup - unset blacklist whitelists _arch _repo repo_file +# Get the blacklisted packages + blacklist=($(get_blacklist)) +# Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" + +# Sync the repos databases + get_repos + +# Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + repo_file=$(get_repo_file ${_repo} ${_arch}) + + if [ ! -f "${repo_file}" ]; then + warning "${repo_file} doesn't exist, skipping this repo-arch" + continue + fi + +# Remove blacklisted packages and count them +# TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages: $( + LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ + grep "\-> Removing" 2>/dev/null| wc -l)" + +# Get db contents + db=($(get_repo_content ${repo_file})) + + msg2 "Process clean db for syncing..." + +# Create a whitelist, add * wildcard to end +# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if +# we aren't syncing them + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + +# Sync excluding everything but whitelist +# We delete here for cleanup + rsync ${extra} -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + +# Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + +# Cleanup + unset db + done + done + + + msg "Syncing package pool" +# Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + +# Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done + + +# Cleanup + unset blacklist whitelists _arch _repo repo_file } trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" -source "$(dirname "$(readlink -e "$0")")/libremessages" +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages # From makepkg set -E @@ -188,4 +183,4 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init -rm -r ${TMPDIR}/${0##*/}.$$.cache +rm -r ${TMPDIR}/$0.$$.cache diff --git a/db-sync.orig b/db-sync.orig new file mode 100755 index 0000000..0e07757 --- /dev/null +++ b/db-sync.orig @@ -0,0 +1,191 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * make a tarball of files used for forensics +# * get files db + +# Run as `V=true db-sync` to get verbose output +VERBOSE=${V} +${VERBOSE} && extra="-v" + +# Returns contents of a repo +get_repos() { + mkdir -p ${TMPDIR}/${0##*/}.$$.cache + # Exclude everything but db files + rsync ${extra} -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache +} + +get_repo_content() { + # Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" +} + +# Process the databases and get the libre packages +init() { + + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from .db database..." + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" + msg2 "Removing blacklisted packages from .files database..." + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" + + # Get db contents + db=($(get_repo_content ${db_file})) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync ${extra} -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done + + # Cleanup + unset blacklist whitelists _arch _repo repo_file +} + +trap_exit() { + echo + error "$@" + exit 1 +} + + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +init + +rm -r ${TMPDIR}/${0##*/}.$$.cache -- cgit v1.2.3 From 0684ad038134d7c9ccbaebc709c4267a904b98b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 15 Jan 2014 19:31:03 -0200 Subject: db-update: add shopt -s nullglob and remove mail script to maintenance@lists.parabolagnulinux.org because it's unstable --- db-update | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index 73d5d3f..186ed55 100755 --- a/db-update +++ b/db-update @@ -3,6 +3,8 @@ . "$(dirname "$(readlink -e "$0")")/config" . "$(dirname "$(readlink -e "$0")")/db-functions" +shopt -s nullglob + if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is no longer supported" exit 1 @@ -91,6 +93,5 @@ while read -r file; do else mkdir -p -- "${pub%/*}" mv -vn "$file" "$pub" - echo "At `date`, script \"`basename $0`\" mailed to "maintenance@lists.parabolagnulinux.org"." fi done < <(find other sources -type f) -- cgit v1.2.3 From 0f9c53d616116cac705b01bfabb2186506aac52a Mon Sep 17 00:00:00 2001 From: Parabola Date: Thu, 16 Jan 2014 04:02:30 +0000 Subject: fix new db-sync --- db-sync | 281 ++++++++++++++++++++++++++++++----------------------------- db-sync.orig | 191 ---------------------------------------- 2 files changed, 144 insertions(+), 328 deletions(-) delete mode 100755 db-sync.orig diff --git a/db-sync b/db-sync index 81dee24..c90f89b 100755 --- a/db-sync +++ b/db-sync @@ -13,174 +13,181 @@ # TODO # * make a tarball of files used for forensics -# * get files db # Run as `V=true db-sync` to get verbose output VERBOSE=${V} ${VERBOSE} && extra="-v" +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/$0.$$.cache -# Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache + # Exclude everything but db files + rsync ${extra} --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ "$WORKDIR" } get_repo_content() { -# Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u + # Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u } # Prints blacklisted packages get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" + cut -d ':' -f 1 "${BLACKLIST_FILE}" } # repo # arch get_repo_file() { -# shopt -s nullglob - - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}" + echo "${WORKDIR}/${1}/os/${2}/${1}" } # Process the databases and get the libre packages init() { -# Get the blacklisted packages - blacklist=($(get_blacklist)) -# Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - -# Sync the repos databases - get_repos - -# Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - repo_file=$(get_repo_file ${_repo} ${_arch}) - - if [ ! -f "${repo_file}" ]; then - warning "${repo_file} doesn't exist, skipping this repo-arch" - continue - fi - -# Remove blacklisted packages and count them -# TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages: $( - LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ - grep "\-> Removing" 2>/dev/null| wc -l)" - -# Get db contents - db=($(get_repo_content ${repo_file})) - - msg2 "Process clean db for syncing..." - -# Create a whitelist, add * wildcard to end -# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if -# we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - -# Sync excluding everything but whitelist -# We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Cleanup - unset db - done - done - - - msg "Syncing package pool" -# Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - -# Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - -# Cleanup - unset blacklist whitelists _arch _repo repo_file + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content ${db_file})) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + printf '%s\n' "${db[@]}" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync ${extra} --no-motd -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + ${WORKDIR}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done + + # Cleanup + unset blacklist whitelists _arch _repo repo_file } trap_exit() { - echo - error "$@" - exit 1 + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" } - -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR init - -rm -r ${TMPDIR}/$0.$$.cache diff --git a/db-sync.orig b/db-sync.orig deleted file mode 100755 index 0e07757..0000000 --- a/db-sync.orig +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics -# * get files db - -# Run as `V=true db-sync` to get verbose output -VERBOSE=${V} -${VERBOSE} && extra="-v" - -# Returns contents of a repo -get_repos() { - mkdir -p ${TMPDIR}/${0##*/}.$$.cache - # Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache -} - -get_repo_content() { - # Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from .db database..." - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" - msg2 "Removing blacklisted packages from .files database..." - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" - - # Get db contents - db=($(get_repo_content ${db_file})) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - # Cleanup - unset blacklist whitelists _arch _repo repo_file -} - -trap_exit() { - echo - error "$@" - exit 1 -} - - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" -source "$(dirname "$(readlink -e "$0")")/libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -init - -rm -r ${TMPDIR}/${0##*/}.$$.cache -- cgit v1.2.3 From 0b52c6e030fc60377a4dfc0a4e142bee50b0344e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 29 Jan 2014 19:01:46 -0200 Subject: add [multilib-testing], [libre-multilib] and [libre-multilib-testing] repos --- config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config b/config index c3f9a55..29c42fe 100644 --- a/config +++ b/config @@ -3,9 +3,9 @@ FTP_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') +ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') # Official Parabola repos -OURREPOS=('libre' 'libre-testing') +OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') # User repos USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos -- cgit v1.2.3 From 5bbd4077065e9a10bab0c709c9e3edb0b7d0fc05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 12 Mar 2014 10:46:56 -0300 Subject: Hopefully fix a horrible bug that caused leaked packages Also added a few checks --- db-sync | 24 ++++++++++++++++++------ libremessages | 6 ++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/db-sync b/db-sync index c90f89b..4a28aa1 100755 --- a/db-sync +++ b/db-sync @@ -62,6 +62,8 @@ init() { msg "%d packages in blacklist" ${#blacklist[@]} + test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" + # Sync the repos databases get_repos @@ -86,10 +88,10 @@ init() { # TODO capture all removed packages for printing on debug mode msg2 "Removing blacklisted packages from %s database..." .db LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' + |& sed -n 's/-> Removing/ &/p' msg2 "Removing blacklisted packages from %s database..." .files LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' + |& sed -n 's/-> Removing/ &/p' # Get db contents db=($(get_repo_content ${db_file})) @@ -98,7 +100,10 @@ init() { # Create a whitelist, add * wildcard to end # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if # we aren't syncing them - printf '%s\n' "${db[@]}" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > /tmp/${_repo}-${_arch}.whitelist msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" @@ -130,8 +135,8 @@ init() { msg "Syncing package pool" - # Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + # Concatenate all whitelists, check for single *s just in case + cat ${whitelists[@]} | grep -v "^\*$" | sort -u > /tmp/any.whitelist msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" @@ -182,10 +187,17 @@ source "$(dirname "$(readlink -e "$0")")/config" source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" +# Check variables presence +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE + FTP_BASE SRCPOOLS PKGPOOLS; do + + test -z "${!var}" && fatal_error "Empty ${var}" +done + # From makepkg set -E for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal done trap 'trap_exit INT "Aborted by user! Exiting..."' INT trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR diff --git a/libremessages b/libremessages index 9fbbc2b..0c6ded1 100755 --- a/libremessages +++ b/libremessages @@ -75,3 +75,9 @@ error() { printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 } +fatal_error() { + local mesg=$1; shift + error "$mesg" "$@" + + exit 1 +} -- cgit v1.2.3 From dc7260243263b8e49bbc308eb67945c43f3bf122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 12 Mar 2014 10:48:26 -0300 Subject: oops --- db-sync | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/db-sync b/db-sync index 4a28aa1..58bb465 100755 --- a/db-sync +++ b/db-sync @@ -188,8 +188,7 @@ source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE - FTP_BASE SRCPOOLS PKGPOOLS; do +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE SRCPOOLS PKGPOOLS; do test -z "${!var}" && fatal_error "Empty ${var}" done -- cgit v1.2.3 From c9ad9623c80ddf2db2a734b4a7914ab9eebd84fb Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 1 Jun 2014 16:04:55 +0100 Subject: Added the script `make_repo_torrents' which makes torrents for all the packages. --- cron-jobs/make_repo_torrents | 81 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 cron-jobs/make_repo_torrents diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents new file mode 100644 index 0000000..d3c3df6 --- /dev/null +++ b/cron-jobs/make_repo_torrents @@ -0,0 +1,81 @@ +#! /bin/bash +# Copyright (C) 2014 Joseph Graham +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script depends on `mktorrent' + +username=$( id -un ) + +case "${username}" in + repo | root ) + true + ;; + * ) + echo "This script must be run as repo user or root user." + echo "ByeBye!" + exit 1 + ;; +esac + +# pacman doesn't support multiple different packages of the same name, +# so it's OK to just stuff all the torrents into a single directory. +torrent_location='/srv/http/repo/public/torrents/' +public_location='/srv/http/repo/public/' + +# Tracker announce URL +tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon + +# All mirrors go here +declare -a url_prefixes=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/') + +cd "${torrent_location}" + +find "${public_location}" -name 'os' -type 'd' | +while read dir +do + find "${dir}" -name '*\.pkg\.tar\.xz' | + while read pkg + do + pkg_name="${pkg##*/}" + + if [[ -h "${pkg}" ]] # check if it's a symbolic link + then + # We get the target of the symlink + pkg=$( readlink -f "${pkg}" ) + fi + + if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] + then + # We need to make a comma seperated list of webseeds (this is passed + # as a single argument to mktorrent) + webseeds='' + + for prefix in "${url_prefixes[@]}" + do + webseeds+="${prefix}${pkg#${public_location}}," + done + + # There should not be a random comma at the end of the webseeds + webseeds="${webseeds%,}" + + mktorrent -a "${tracker}" "${pkg}" -w "${webseeds}" # "${torrent_location}" + fi + done +done + +if [[ "${username}" == root ]] +then + chown repo * +fi -- cgit v1.2.3 From ee8da885dc5062001861e4355602a39ae972c32c Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Mon, 2 Jun 2014 12:48:47 +0100 Subject: Made it randomize the order of the webseed list for every torrent. --- cron-jobs/make_repo_torrents | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index d3c3df6..dffd0dc 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -38,7 +38,25 @@ public_location='/srv/http/repo/public/' tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon # All mirrors go here -declare -a url_prefixes=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/') +declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/') + +# I got this function from http://mywiki.wooledge.org/BashFAQ/026 . It +# shuffles an array. Uses a global array variable. Must be compact +# (not a sparse array). The array must be called `array'. +shuffle() { + local i tmp size max rand + + # $RANDOM % (i+1) is biased because of the limited range of $RANDOM + # Compensate by using a range which is a multiple of the array size. + size=${#array[*]} + max=$(( 32768 / size * size )) + + for ((i=size-1; i>0; i--)); do + while (( (rand=$RANDOM) >= max )); do :; done + rand=$(( rand % (i+1) )) + tmp=${array[i]} array[i]=${array[rand]} array[rand]=$tmp + done +} cd "${torrent_location}" @@ -62,7 +80,12 @@ do # as a single argument to mktorrent) webseeds='' - for prefix in "${url_prefixes[@]}" + # Randomize the order of the list of webseeds because I + # don't know if transmission might always use the one at + # the top otherwize. + shuffle + + for prefix in "${array[@]}" do webseeds+="${prefix}${pkg#${public_location}}," done -- cgit v1.2.3 From 6979cf5d1e4a599fbabaed4fe643e9db0f0f871c Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Mon, 2 Jun 2014 12:54:47 +0100 Subject: Added encyclomundi's mirror to the webseeds array. --- cron-jobs/make_repo_torrents | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index dffd0dc..246264d 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -38,7 +38,7 @@ public_location='/srv/http/repo/public/' tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon # All mirrors go here -declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/') +declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') # I got this function from http://mywiki.wooledge.org/BashFAQ/026 . It # shuffles an array. Uses a global array variable. Must be compact -- cgit v1.2.3 From ad79659b3057fd5afac3629e9dd53ea3b93fb58e Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Fri, 6 Jun 2014 12:29:21 +0100 Subject: Made it only give output on an error. --- cron-jobs/make_repo_torrents | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index 246264d..797142e 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -93,7 +93,8 @@ do # There should not be a random comma at the end of the webseeds webseeds="${webseeds%,}" - mktorrent -a "${tracker}" "${pkg}" -w "${webseeds}" # "${torrent_location}" + mktorrent -a "${tracker}" "${pkg}" -w "${webseeds}" >/dev/null || + echo "Error making torrent for \"${pkg}\"" fi done done -- cgit v1.2.3 From d902025a8a000c1383c7612e9f442c6734290169 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 15 Jun 2014 22:20:18 +0000 Subject: put our packages in a separate pool --- config | 12 ++++++++---- db-sync | 16 +++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/config b/config index 29c42fe..c6efac6 100644 --- a/config +++ b/config @@ -12,14 +12,18 @@ USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jor PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' +PKGPOOL='pool/parabola' +SRCPOOL='sources/parabola' # Directories where packages are shared between repos # *relative to FTP_BASE* -PKGPOOLS=('pool/packages' 'pool/community') +ARCHPKGPOOLS=(pool/{packages,community}) +OURPKGPOOLS=(pool/parabola) +PKGPOOLS=(${OURPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) # Directories where sources are stored -SRCPOOLS=('sources/packages' 'sources/community') +ARCHSRCPOOLS=(sources/{packages,community}) +OURPKGPOOLS=(sources/parabola) +SRCPOOLS=(${OURSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false diff --git a/db-sync b/db-sync index 58bb465..4e692e9 100755 --- a/db-sync +++ b/db-sync @@ -143,14 +143,15 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - for PKGPOOL in ${PKGPOOLS[@]}; do + local pkgpool + for pkgpool in ${ARCHPKGPOOLS[@]}; do rsync ${extra} --no-motd -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ + rsync://${mirror}/${mirrorpath}/${pkgpool}/ \ + ${FTP_BASE}/${pkgpool}/ done # Sync sources @@ -161,14 +162,15 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - for SRCPOOL in ${SRCPOOLS[@]}; do + local srcpool + for srcpool in ${ARCHSRCPOOLS[@]}; do rsync ${extra} --no-motd -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ + rsync://${mirror}/${mirrorpath}/${srcpool}/ \ + ${FTP_BASE}/${srcpool}/ done # Cleanup @@ -188,7 +190,7 @@ source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE SRCPOOLS PKGPOOLS; do +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do test -z "${!var}" && fatal_error "Empty ${var}" done -- cgit v1.2.3 From d8e9a3ea7c9d8924eef16e6bd9725cf151151936 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 15 Jun 2014 23:02:17 -0400 Subject: config: don't clobber the existing STAGING scheme --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index baf5da7..63698b9 100644 --- a/config +++ b/config @@ -41,7 +41,7 @@ REQUIRE_SIGNATURE=true LOCK_DELAY=10 LOCK_TIMEOUT=300 -[ -n "${STAGING:-}" ] || STAGING="$HOME/staging" +[ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) OURARCHES=(mips64el) -- cgit v1.2.3 From c121ead116f26ed2b36247247fdf4a0d19f063f2 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Mon, 16 Jun 2014 14:09:50 +0100 Subject: removed all mirrors but one since pacman2pacman now re-writes the webseeds list to just point to the user's chosen mirror. --- cron-jobs/make_repo_torrents | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index 797142e..0798a5e 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -38,7 +38,11 @@ public_location='/srv/http/repo/public/' tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon # All mirrors go here -declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') +#declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') + +# I'm removing all mirrors but one since pacman2pacman now re-writes +# the webseeds list to just point to the user's chosen mirror +declare -a array=('http://repo.parabolagnulinux.org/') # I got this function from http://mywiki.wooledge.org/BashFAQ/026 . It # shuffles an array. Uses a global array variable. Must be compact -- cgit v1.2.3 From fae18d372c7d5fad545526b554e9cde657192226 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 16 Jun 2014 10:59:34 -0400 Subject: config:remove java-ugly --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index c6efac6..c7ea7c2 100644 --- a/config +++ b/config @@ -9,7 +9,7 @@ OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') # User repos USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos -PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') +PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'nonprism') # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") PKGPOOL='pool/parabola' -- cgit v1.2.3 From 7a94b2db24e707606d9e620ba1b8a9462dce5e0f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 16 Jun 2014 11:01:06 -0400 Subject: config: don't have default STAGING be public --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index c7ea7c2..75e88cb 100644 --- a/config +++ b/config @@ -40,7 +40,7 @@ REQUIRE_SIGNATURE=true LOCK_DELAY=10 LOCK_TIMEOUT=300 -[ -n "${STAGING:-}" ] || STAGING="$FTP_BASE/staging" +[ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) OURARCHES=(mips64el) -- cgit v1.2.3 From 73dcd17cf3201295f2ee29f63b4353a5e3c95779 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 16 Jun 2014 12:16:56 -0400 Subject: abslibre: also run git gc --- abslibre | 1 + 1 file changed, 1 insertion(+) diff --git a/abslibre b/abslibre index a5733f0..09105c1 100755 --- a/abslibre +++ b/abslibre @@ -90,6 +90,7 @@ sync_pre_mips64el() { git add . && \ git commit -m \"$(date)\" -a git push origin master + git gc " } -- cgit v1.2.3 From 64a9e2fcdeaf99ae982270743c6cca4dd5314b5c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 16 Jun 2014 12:20:45 -0400 Subject: abslibre: use tab indent --- abslibre | 111 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/abslibre b/abslibre index 09105c1..0b3e371 100755 --- a/abslibre +++ b/abslibre @@ -23,44 +23,44 @@ REPOS=(core extra community testing community-testing !staging !community-stagin # * Create repo.abs.tar.gz tarballs function sync_abs() { - for ARCH in any i686 x86_64; do - rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? - done + for ARCH in any i686 x86_64; do + rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? + done } function get_blacklist() { printf ":: Updating blacklist...\t" wget -q -O - "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ - sed "s/^/**\//" > ${BLFILE} || { + sed "s/^/**\//" > ${BLFILE} || { printf "[FAILED]\n" return 1 } -# Prevent using an empty blacklist - [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 + # Prevent using an empty blacklist + [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 printf "[OK]\n" } function sync_abs_libre() { -# Clone ABSLibre git repo - if [ -d /var/tmp/abslibre/.git ]; then - pushd /var/tmp/abslibre >/dev/null 2>&1 - git pull - popd >/dev/null 2>&1 - else - git clone /srv/git/abslibre.git /var/tmp/abslibre - fi + # Clone ABSLibre git repo + if [ -d /var/tmp/abslibre/.git ]; then + pushd /var/tmp/abslibre >/dev/null 2>&1 + git pull + popd >/dev/null 2>&1 + else + git clone /srv/git/abslibre.git /var/tmp/abslibre + fi -# Sync from ABS and then sync from ABSLibre + # Sync from ABS and then sync from ABSLibre printf ":: Syncing ABSLibre...\t" (rsync ${SYNCARGS} --delete-excluded \ - --exclude-from=${BLFILE} \ - ${ABSROOT} \ - ${ABSLIBRE} \ - && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + --exclude-from=${BLFILE} \ + ${ABSROOT} \ + ${ABSLIBRE} \ + && + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { printf "[FAILED]\n" return 1 } @@ -70,46 +70,45 @@ function sync_abs_libre() { # This part is very hacky and particular to the current setup :P sync_pre_mips64el() { - pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null - - sudo -u fauno sh -c " - - rsync ${SYNCARGS} \ - --exclude=.git* \ - --exclude=community-staging \ - --exclude=community-testing \ - --exclude=gnome-unstable \ - --exclude=kde-unstable \ - --exclude=multilib \ - --exclude=multilib-testing \ - --exclude=multilib-staging \ - --exclude=staging \ - --exclude=testing \ - ${ABSLIBRE}/x86_64/ \ - /home/fauno/Repos/abslibre-pre-mips64el/ && \ - git add . && \ - git commit -m \"$(date)\" -a - git push origin master - git gc - " + pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null + + sudo -u fauno sh -c " + rsync ${SYNCARGS} \ + --exclude=.git* \ + --exclude=community-staging \ + --exclude=community-testing \ + --exclude=gnome-unstable \ + --exclude=kde-unstable \ + --exclude=multilib \ + --exclude=multilib-testing \ + --exclude=multilib-staging \ + --exclude=staging \ + --exclude=testing \ + ${ABSLIBRE}/x86_64/ \ + /home/fauno/Repos/abslibre-pre-mips64el/ && + git add . && + git commit -m \"$(date)\" -a + git push origin master + git gc + " } # Create .abs.tar.gz tarballs create_tarballs() { - for repo in ${ABSLIBRE}/{i686,x86_64}/*; do - baserepo=${repo##*/} - arch=$(basename $(dirname $repo)) - -# Remove the old one - mkdir -p /srv/http/web/media/abs/$baserepo/os/$arch/ - rm /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz -# Create a new one joining arch and any -# Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czvf /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz \ - -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ - $repo/* ${ABSLIBRE}/any/${baserepo}/* - - done + for repo in ${ABSLIBRE}/{i686,x86_64}/*; do + baserepo=${repo##*/} + arch=$(basename $(dirname $repo)) + + # Remove the old one + mkdir -p /srv/http/web/media/abs/$baserepo/os/$arch/ + rm /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz + # Create a new one joining arch and any + # Remove the first part of the path (it could be $repo but any isn't hit) + bsdtar -czvf /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ + $repo/* ${ABSLIBRE}/any/${baserepo}/* + + done } sync_abs || exit 1 -- cgit v1.2.3 From db388e32cb1f6707c37a19df6b68b3d7e67d9ab5 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 21:18:57 -0400 Subject: db-import.conf: add all staging repos --- db-import.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db-import.conf b/db-import.conf index 2728077..4330fa9 100644 --- a/db-import.conf +++ b/db-import.conf @@ -3,10 +3,10 @@ IMPORTDIR=/srv/repo/import _archrepos=( - {core,extra,testing}-{i686,x86_64} + {core,extra,testing,staging}-{i686,x86_64} {gnome,kde}-unstable-{i686,x86_64} - community{,-testing}-{i686,x86_64} - multilib{,-testing}-x86_64 + community{,-testing,-staging}-{i686,x86_64} + multilib{,-testing,-staging}-x86_64 ) _archpkgmirror=$(db-pick-mirror rsync https://www.archlinux.org/mirrors/status/tier/1/json/) -- cgit v1.2.3 From 88472049c1e8ccd64f355eed2ce624954539944d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 21:19:10 -0400 Subject: db-pick-mirror: remove obsolete comment --- db-pick-mirror | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-pick-mirror b/db-pick-mirror index 7cbc032..9474ed7 100755 --- a/db-pick-mirror +++ b/db-pick-mirror @@ -18,7 +18,7 @@ rsync_urls = urls.select{|a| a["protocol"]==protocol} # By score ( (delay+speed)/completion ) #best = rsync_urls.sort{|a,b| (a["score"] || Float::INFINITY) <=> (b["score"] || Float::INFINITY) }.first -# By delay/completion ; hopefully this gives us a tier 1 mirror +# By delay/completion best = rsync_urls.sort{|a,b| a["delay"]/a["completion_pct"] <=> b["delay"]/b["completion_pct"] }.first puts best["url"] -- cgit v1.2.3 From e3d6871deb5c448b5856ffc25e2c4e55c77f719e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 21:28:35 -0400 Subject: rm get-repos (obsolete) and git-pbs (never used/testing) --- get-repos | 58 ---------------------------------------------------------- git-pbs | 44 -------------------------------------------- 2 files changed, 102 deletions(-) delete mode 100755 get-repos delete mode 100755 git-pbs diff --git a/get-repos b/get-repos deleted file mode 100755 index b98d601..0000000 --- a/get-repos +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Gets repo databases and updates parabolaweb -# Note: It works remotely because our parabolaweb server and repo server are -# two different hosts - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -TMPDIR="$(mktemp -dt "${0##*/}.XXXX")" -DBLIST=() - -# Repos -for _repo in "${PKGREPOS[@]}"; do - for _arch in "${ARCHES[@]}"; do - DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${FILESEXT}") - done -done - -# Get them all -msg "Retrieving ${#DBLIST[@]} databases" -wget --directory-prefix=${TMPDIR} \ - --no-verbose \ - --force-directories \ - --no-host-directories \ - "${DBLIST[@]}" || true -# Always return true, some databases are expect to be missing - -# Create the arches regexp arch1|arch2 -arch_re="$(echo "(${ARCHES[@]} i586)" | tr ' ' '|')" - -msg "Adding to parabolaweb" -find "${TMPDIR}" -iname "*${FILESEXT}" | while read _db; do - _arch=$(echo "${_db}" | egrep -o "${arch_re}") - - if [ -z "${_arch}" ]; then - error "Can't find database architecture: ${_db}" - continue - fi - - "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" || true -done - -rm -r ${TMPDIR} - -exit $? diff --git a/git-pbs b/git-pbs deleted file mode 100755 index b815863..0000000 --- a/git-pbs +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -_pkg=$1 - -mkdir -p $_pkg -pushd $_pkg - - -if [ ! -d .git ]; then -# Start a git repo for the package -# Add the remote origin -# Pull the package branch onto an unmodified branch - git init - git remote add arch git://projects.archlinux.org/svntogit/packages.git - -# Export the repository - touch .git/git-daemon-export-ok - -# Pass the -b flag to checkout to create the branches - extra="-b" -fi - -git checkout ${extra} upstream -git pull arch packages/$_pkg - -# Move PKGBUILD and files to the basedir -# Remove everything else from the repo -git checkout ${extra} master - -# This produces a lot of merging conflicts -git merge upstream - -# This apparently solves them -git mv trunk/* . -git rm -rf repos - -# Remove the actual files -rm -rf trunk repos - -# Commit everything -git commit -a -m "Converted to PBS" - -# Return to the repo -popd >/dev/null -- cgit v1.2.3 From 22f7ae3e5c791694edf4c4b6e5f8623f9a9dddf4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 21:37:44 -0400 Subject: mkrepo: cleanup - Use config:FTP_BASE instead of local_config:repodir - Don't mention get_repos script anymore - Remove redundant lines of code. - Better quoting. --- local_config | 3 --- mkrepo | 17 ++++------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/local_config b/local_config index 9b4415e..57725fe 100644 --- a/local_config +++ b/local_config @@ -10,9 +10,6 @@ mirror="mirrors.kernel.org" #mirror="mirror.de.leaseweb.net" mirrorpath="archlinux" -# mkrepo -repodir=${_paraboladir} - # yf-update blacklist=${_paraboladir}/docs/blacklist.txt whitelist=${_paraboladir}/docs/whitelist.txt diff --git a/mkrepo b/mkrepo index 10d014b..f30ad00 100755 --- a/mkrepo +++ b/mkrepo @@ -4,21 +4,12 @@ # Description: A script to quickly create new [repos] source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" - -# TODO it would be simpler to expand arrays to {element1,element2,etc} -for repo in $@; do +for repo in "$@"; do echo ":: Creating [$repo]" - mkdir -pv ${repodir}/{staging/,}${repo} - - for arch in ${ARCHES[@]}; do - mkdir -pv ${repodir}/${repo}/os/${arch} + for arch in "${ARCHES[@]}"; do + mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" done - done -echo ":: All done. Add the repo to the parabolaweb admin page" -echo " and the get_repos script on the same server." - -exit $? +echo ":: All done. Add the repo to the ParabolaWeb admin page." -- cgit v1.2.3 From ea871cfb29b7e9d438cb50c5a75642deee86f652 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 21:43:02 -0400 Subject: rm -r yf/ yf-update # they aren't used anymore --- local_config | 10 ---------- yf-update | 18 ------------------ yf/PKGBUILD | 28 ---------------------------- yf/your-freedom.install | 32 -------------------------------- 4 files changed, 88 deletions(-) delete mode 100755 yf-update delete mode 100644 yf/PKGBUILD delete mode 100644 yf/your-freedom.install diff --git a/local_config b/local_config index 57725fe..3d6c709 100644 --- a/local_config +++ b/local_config @@ -1,7 +1,3 @@ -#/bin/bash # as a hint to text editors -_paraboladir=/srv/http/repo/public - -# db-sync #mirror="mirrors.uk2.net" mirror="mirrors.kernel.org" #mirror="mirror.umd.edu" @@ -9,9 +5,3 @@ mirror="mirrors.kernel.org" #mirror="mirror.us.leaseweb.net" #mirror="mirror.de.leaseweb.net" mirrorpath="archlinux" - -# yf-update -blacklist=${_paraboladir}/docs/blacklist.txt -whitelist=${_paraboladir}/docs/whitelist.txt - -unset _paraboladir diff --git a/yf-update b/yf-update deleted file mode 100755 index b6dff14..0000000 --- a/yf-update +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" -source "$(dirname "$(readlink -e "$0")")/libremessages" - -blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) -last_bl_mtime=$(cat "$(dirname "$(readlink -e "$0")")/yftime") - -if [ $blacklist_mtime -gt $last_bl_mtime ]; then - pushd "$(dirname "$(readlink -e "$0")")/yf" - makepkg -f - find . -name "*${PKGEXT}" -exec mv {} ${STAGING}/libre \; - popd - echo ${blacklist_mtime} > "$(dirname "$(readlink -e "$0")")/yftime" - msg2 "built and staged" -else - msg2 "nothing to do" -fi diff --git a/yf/PKGBUILD b/yf/PKGBUILD deleted file mode 100644 index e737482..0000000 --- a/yf/PKGBUILD +++ /dev/null @@ -1,28 +0,0 @@ -# Maintainer: Parabola Project -pkgname=your-freedom -pkgver=$(LC_ALL=C date -u +%Y%m%d) -pkgrel=1 -pkgdesc="This package conflicts with every nonfree package known to date." -arch=('any') -url="https://parabolagnulinux.org" -license=('GPL') -groups=('base') -install=${pkgname}.install -source=() -md5sums=() -noextract=() - -build() { - cd ${srcdir} - source ~/repm/local_config - install -d ${pkgdir}/usr/share/doc/${pkgname} - install -m644 $blacklist $whitelist ${pkgdir}/usr/share/doc/${pkgname}/ -} - -package() { - conflicts=($(cut -d: -f1,2 ${pkgdir}/usr/share/doc/${pkgname}/blacklist.txt | \ - sed "s/:$//" | \ - grep -v ":" | \ - sort -u - )) -} diff --git a/yf/your-freedom.install b/yf/your-freedom.install deleted file mode 100644 index 731a575..0000000 --- a/yf/your-freedom.install +++ /dev/null @@ -1,32 +0,0 @@ - -pre_install() { - cat < Date: Tue, 17 Jun 2014 21:44:14 -0400 Subject: mv local_config db-sync.conf # now that its other uses are gone --- db-sync | 2 +- db-sync.conf | 7 +++++++ local_config | 7 ------- 3 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 db-sync.conf delete mode 100644 local_config diff --git a/db-sync b/db-sync index 4e692e9..138328b 100755 --- a/db-sync +++ b/db-sync @@ -186,7 +186,7 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/db-sync.conf" source "$(dirname "$(readlink -e "$0")")/libremessages" # Check variables presence diff --git a/db-sync.conf b/db-sync.conf new file mode 100644 index 0000000..3d6c709 --- /dev/null +++ b/db-sync.conf @@ -0,0 +1,7 @@ +#mirror="mirrors.uk2.net" +mirror="mirrors.kernel.org" +#mirror="mirror.umd.edu" +#mirror="archlinux.c3sl.ufpr.br" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.de.leaseweb.net" +mirrorpath="archlinux" diff --git a/local_config b/local_config deleted file mode 100644 index 3d6c709..0000000 --- a/local_config +++ /dev/null @@ -1,7 +0,0 @@ -#mirror="mirrors.uk2.net" -mirror="mirrors.kernel.org" -#mirror="mirror.umd.edu" -#mirror="archlinux.c3sl.ufpr.br" -#mirror="mirror.us.leaseweb.net" -#mirror="mirror.de.leaseweb.net" -mirrorpath="archlinux" -- cgit v1.2.3 From ad922cd1ea67945d538036b021d8ccf8f8b475b6 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 22:11:19 -0400 Subject: cron-jobs/ftpdir-cleanup: remove old /any/ cleanup code --- cron-jobs/ftpdir-cleanup | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index cb14382..f2d8b33 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -68,29 +68,6 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -# cleanup of legacy $repo/os/any directories -for repo in "${PKGREPOS[@]}"; do - if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then - continue - fi - if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then - msg "Removing empty legacy directory ${repo}/os/any" - ${CLEANUP_DRYRUN} || rmdir "${FTP_BASE}/${repo}/os/any" - continue - fi - find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}" - cat "${WORKDIR}/db-${repo}-"* | sort -u > "${WORKDIR}/all-${repo}" - - old_pkgs=($(comm -23 "${WORKDIR}/any-${repo}" "${WORKDIR}/all-${repo}")) - if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old packages from [${repo}] (any)..." - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" - clean_pkg "${FTP_BASE}/${repo}/os/any/${old_pkg}" - done - fi -done - old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from the cleanup directory..." -- cgit v1.2.3 From c52b1fa175b0bcc1bd2ec86ca7f4a926905dce0b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 22:11:32 -0400 Subject: db-functions: add mv_acl --- db-functions | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/db-functions b/db-functions index 52ad8bb..6b49e6a 100644 --- a/db-functions +++ b/db-functions @@ -15,6 +15,15 @@ restore_umask () { umask $UMASK >/dev/null } +# just like mv -f, but we touch the file and then copy the content so +# default ACLs in the target dir will be applied +mv_acl() { + rm -f "$2" + touch "$2" + cat "$1" >"$2" || return 1 + rm -f "$1" +} + # set up general environment WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") LOCKS=() -- cgit v1.2.3 From fb93f14b724c82427ab2531502ed3b79e6ccfe7f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 22:52:21 -0400 Subject: s/libremessages/db-functions/ --- any-to-ours | 2 +- db-cleanup | 2 +- db-sync | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/any-to-ours b/any-to-ours index a1697c7..020f623 100755 --- a/any-to-ours +++ b/any-to-ours @@ -8,7 +8,7 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/libremessages" +source "$(dirname "$(readlink -e "$0")")/db-functions" # From makepkg set -E diff --git a/db-cleanup b/db-cleanup index 57ef36e..f6580b6 100755 --- a/db-cleanup +++ b/db-cleanup @@ -16,7 +16,7 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/libremessages" +source "$(dirname "$(readlink -e "$0")")/db-functions" # From makepkg set -E diff --git a/db-sync b/db-sync index b5484d1..2a0e8e1 100755 --- a/db-sync +++ b/db-sync @@ -193,7 +193,7 @@ trap_exit() { source "$(dirname "$(readlink -e "$0")")/config" source "$(dirname "$(readlink -e "$0")")/db-sync.conf" -source "$(dirname "$(readlink -e "$0")")/libremessages" +source "$(dirname "$(readlink -e "$0")")/db-functions" # Check variables presence for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do -- cgit v1.2.3 From 56e6bf0c9a2dac9e34b4ce12af1e614308a38319 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 22:53:26 -0400 Subject: db-cleanup: cleanup --- db-cleanup | 60 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/db-cleanup b/db-cleanup index f6580b6..2051130 100755 --- a/db-cleanup +++ b/db-cleanup @@ -10,9 +10,9 @@ # * Instant cleanup! trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } source "$(dirname "$(readlink -e "$0")")/config" @@ -25,42 +25,42 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -${CLEANUP_DRYRUN} && EXTRAFLAGS+=" --dry-run" +EXTRAFLAGS=() +"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) -for _repo in ${PKGREPOS[@]}; do - for _arch in ${ARCHES[@]}; do - msg "Getting ${_repo}-${_arch} database" +for _repo in "${PKGREPOS[@]}"; do + for _arch in "${ARCHES[@]}"; do + msg "Getting ${_repo}-${_arch} database" - dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" + dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" - if [ ! -r "${dbfile}" ]; then - warning "Not found" - continue - fi + if [ ! -r "${dbfile}" ]; then + warning "Not found" + continue + fi -# Echo the contents into a filter file - bsdtar tf "${dbfile}" | \ - cut -d'/' -f1 | \ - sort -u | \ - sed "s|$|*|" >> /tmp/${0##*/}.$$.filter + # Echo the contents into a filter file + bsdtar tf "${dbfile}" | \ + cut -d'/' -f1 | \ + sort -u | \ + sed "s|$|*|" >> "/tmp/${0##*/}.$$.filter" - done + done done msg "Removing old files:" -for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do - msg2 "${POOL}" +for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do + msg2 "${POOL}" - rsync ${EXTRAFLAGS} -va --delete-excluded \ - --include-from="/tmp/${0##*/}.$$.filter" \ - --exclude="*" \ - ${FTP_BASE}/${POOL}/ \ - ${FTP_BASE}/${POOL}/ + rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ + --include-from="/tmp/${0##*/}.$$.filter" \ + --exclude="*" \ + "${FTP_BASE}/${POOL}/" \ + "${FTP_BASE}/${POOL}/" done -msg "Removing symlinks:" -find -L ${FTP_BASE}/ -type l -${CLEANUP_DRYRUN} || find -L ${FTP_BASE}/ -type l -delete - -exit $? +msg "Removing dead symlinks:" +actions=(-print) +"${CLEANUP_DRYRUN}" || actions+=(-delete) +find -L "${FTP_BASE}/" -type l -print -delete -- cgit v1.2.3 From 4143e0fe34e549e34e1bfb0efe3111430017046a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 22:53:54 -0400 Subject: sourceballs2, db-check-nonfree: quote and tab-indent --- cron-jobs/sourceballs2 | 50 +++++++++++++++++++++++++------------------------- db-check-nonfree | 44 ++++++++++++++++++++++---------------------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 2a26e6a..64bae4a 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -13,7 +13,7 @@ pushd "${WORKDIR}" >/dev/null script_lock -#adjust the nice level to run at a lower priority +# Adjust the nice level to run at a lower priority renice +10 -p $$ > /dev/null # Create a list of all available source package file names @@ -22,41 +22,41 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null for repo in "${PKGREPOS[@]}"; do - msg "Sourceballing [${repo}]" + msg "Sourceballing [${repo}]" - pushd $repo >/dev/null - find -maxdepth 1 -type d | while read pkg; do - pushd "${SVNREPO}/$repo/$pkg" >/dev/null + pushd "$repo" >/dev/null + find -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null - [[ ! -e PKGBUILD ]] && { - warning "$repo/$pkg is not a package" - continue - } + [[ ! -e ./PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } -# Unset the previous data - unset pkgbase pkgname pkgver pkgrel - source PKGBUILD + # Unset the previous data + unset pkgbase pkgname pkgver pkgrel + source PKGBUILD - unset build package url pkgdesc source md5sums depends makedepends \ - optdepends license arch options check mksource + unset build package url pkgdesc source md5sums depends makedepends \ + optdepends license arch options check mksource - for _pkg in "${pkgname[@]}"; do - unset package_${_pkg} >/dev/null 2>&1 - done + for _pkg in "${pkgname[@]}"; do + unset "package_${_pkg}" >/dev/null 2>&1 + done - pkgbase=${pkgbase:-$pkgname} - srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + pkgbase=${pkgbase:-$pkgname} + srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs" - # Skip already sourceballed - [ -e "${SRCPKGDEST}/${srcfile}" ] && continue + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${srcfile}" ]] && continue - makepkg --allsource --ignorearch -c >/dev/null 2>&1 + makepkg --allsource --ignorearch -c >/dev/null 2>&1 - [ $? -ne 0 ] && plain ${srcfile} + [[ $? -ne 0 ]] && plain "${srcfile}" - done # end find pkgs - popd >/dev/null + done # end find pkgs + popd >/dev/null done # end repos diff --git a/db-check-nonfree b/db-check-nonfree index 6e2dc17..253490b 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -3,7 +3,7 @@ . "$(dirname "$(readlink -e "$0")")/config" . "$(dirname "$(readlink -e "$0")")/db-functions" -if [ $# -ge 1 ]; then +if [[ $# -ge 1 ]]; then warning "Calling ${0##*/} with a specific repository is not supported" exit 1 fi @@ -11,36 +11,36 @@ fi # TODO: this might lock too much (architectures) for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do - repo_lock ${repo} ${pkgarch} || exit 1 + repo_lock "${repo}" "${pkgarch}" || exit 1 done done msg "Check nonfree in repo:" -nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) +nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u)) for repo in "${ARCHREPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - msg2 "$repo $pkgarch" - if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then - continue - fi - unset dbpkgs - unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in "${dbpkgs[@]}"; do - if in_array "${pkgname}" "${nonfree[@]}"; then - cleanpkgs+=("${pkgname}") - fi + for pkgarch in "${ARCHES[@]}"; do + msg2 "$repo $pkgarch" + if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then + continue + fi + unset dbpkgs + unset cleanpkgs + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in "${dbpkgs[@]}"; do + if in_array "${pkgname}" "${nonfree[@]}"; then + cleanpkgs+=("${pkgname}") + fi + done + if [ ${#cleanpkgs[@]} -ge 1 ]; then + msg2 "Nonfree: ${cleanpkgs[*]}" + arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" + fi done - if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: ${cleanpkgs[*]}" - arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" - fi - done done for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do - repo_unlock ${repo} ${pkgarch} + repo_unlock "${repo}" "${pkgarch}" done done -- cgit v1.2.3 From 6ce6d9f136122c8b8d6cf152216996de083328d4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 22:54:15 -0400 Subject: touch up a bunch of quoting --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 36 ++++++++++++++-------------- cron-jobs/devlist-mailer | 2 +- cron-jobs/ftpdir-cleanup | 18 +++++++------- cron-jobs/integrity-check | 4 ++-- cron-jobs/sourceballs | 14 +++++------ cron-jobs/update-web-db | 16 ++++++------- 6 files changed, 45 insertions(+), 45 deletions(-) diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index c8d8618..38af179 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -12,7 +12,7 @@ readonly -a variables splitpkg_overrides backup_package_variables() { for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" - eval "${indirect}=(\${$var[@]})" + eval "${indirect}=(\"\${$var[@]}\")" done } @@ -20,9 +20,9 @@ restore_package_variables() { for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then - eval "${var}=(\${$indirect[@]})" + eval "${var}=(\"\${$indirect[@]}\")" else - unset ${var} + unset "${var}" fi done } @@ -42,31 +42,31 @@ print_info() { if [ -n "$arch" ]; then echo "%ARCH%" - for i in "${arch[@]}"; do echo $i; done + for i in "${arch[@]}"; do echo "$i"; done echo "" fi if [ -n "$depends" ]; then echo "%DEPENDS%" for i in "${depends[@]}"; do - echo $i + echo "$i" done echo "" fi if [ -n "$makedepends" ]; then echo "%MAKEDEPENDS%" for i in "${makedepends[@]}"; do - echo $i + echo "$i" done echo "" fi if [ -n "$conflicts" ]; then echo "%CONFLICTS%" - for i in "${conflicts[@]}"; do echo $i; done + for i in "${conflicts[@]}"; do echo "$i"; done echo "" fi if [ -n "$provides" ]; then echo "%PROVIDES%" - for i in "${provides[@]}"; do echo $i; done + for i in "${provides[@]}"; do echo "$i"; done echo "" fi } @@ -76,9 +76,9 @@ source_pkgbuild() { dir=$1 pkgbuild=$dir/PKGBUILD for var in "${variables[@]}"; do - unset ${var} + unset "${var}" done - source $pkgbuild &>/dev/null || ret=$? + source "$pkgbuild" &>/dev/null || ret=$? # ensure $pkgname and $pkgver variables were found if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then @@ -89,7 +89,7 @@ source_pkgbuild() { if [ "${#pkgname[@]}" -gt "1" ]; then pkgbase=${pkgbase:-${pkgname[0]}} for pkg in "${pkgname[@]}"; do - if [ "$(type -t package_${pkg})" != "function" ]; then + if [ "$(type -t "package_${pkg}")" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" return 1 else @@ -104,7 +104,7 @@ source_pkgbuild() { break fi done - done < <(type package_${pkg}) + done < <(type "package_${pkg}") print_info restore_package_variables fi @@ -124,14 +124,14 @@ find_pkgbuilds() { return fi - if [ -f $1/PKGBUILD ]; then - source_pkgbuild $1 + if [ -f "$1/PKGBUILD" ]; then + source_pkgbuild "$1" return fi empty=1 - for dir in $1/*; do - if [ -d $dir ]; then - find_pkgbuilds $dir + for dir in "$1"/*; do + if [ -d "$dir" ]; then + find_pkgbuilds "$dir" unset empty fi done @@ -147,7 +147,7 @@ fi CARCH=$1 shift for dir in "$@"; do - find_pkgbuilds $dir + find_pkgbuilds "$dir" done exit 0 diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index ca2e46b..61cd32c 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -23,6 +23,6 @@ echo "Subject: $SUBJECT To: $LIST From: $FROM -$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" +$stdin" | /usr/sbin/sendmail -F"$FROM" "$LIST" fi diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index ad2e7f9..292952b 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -7,7 +7,7 @@ clean_pkg() { local pkg local target - if ! ${CLEANUP_DRYRUN}; then + if ! "${CLEANUP_DRYRUN}"; then for pkg in "$@"; do if [ -h "$pkg" ]; then rm -f "$pkg" "$pkg.sig" @@ -26,11 +26,11 @@ script_lock for repo in "${PKGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do - repo_lock ${repo} ${arch} || exit 1 + repo_lock "${repo}" "${arch}" || exit 1 done done -${CLEANUP_DRYRUN} && warning 'dry run mode is active' +"${CLEANUP_DRYRUN}" && warning 'dry run mode is active' for repo in "${PKGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do @@ -45,7 +45,7 @@ for repo in "${PKGREPOS[@]}"; do missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#missing_pkgs[@]} -ge 1 ]; then error "Missing packages in [${repo}] (${arch})..." - for missing_pkg in ${missing_pkgs[@]}; do + for missing_pkg in "${missing_pkgs[@]}"; do msg2 "${missing_pkg}" done fi @@ -69,18 +69,18 @@ cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from package pool..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" done fi -old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find "${CLEANUP_DESTDIR}" -type f -name "*${PKGEXT}" -mtime +"${CLEANUP_KEEP}" -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" - if ! ${CLEANUP_DRYRUN}; then + if ! "${CLEANUP_DRYRUN}"; then rm -f "${CLEANUP_DESTDIR}/${old_pkg}" rm -f "${CLEANUP_DESTDIR}/${old_pkg}.sig" fi @@ -89,7 +89,7 @@ fi for repo in "${PKGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do - repo_unlock ${repo} ${arch} + repo_unlock "${repo}" "${arch}" done done diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 86a8f1d..33a4eb6 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -13,12 +13,12 @@ fi mailto=$1 check() { - ${dirname}/check_archlinux/check_packages.py \ + "${dirname}"/check_archlinux/check_packages.py \ --repos="${repos}" \ --abs-tree="/srv/abs/rsync/${arch},/srv/abs/rsync/any" \ --repo-dir="${FTP_BASE}" \ --arch="${arch}" \ - 2>&1 | ${dirname}/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" + 2>&1 | "${dirname}"/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" } repos='core,extra,community' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index d5bf54b..2c11de2 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -9,7 +9,7 @@ script_lock for repo in "${PKGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do - repo_lock ${repo} ${arch} || exit 1 + repo_lock "${repo}" "${arch}" || exit 1 done done @@ -41,7 +41,7 @@ done for repo in "${PKGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do - repo_unlock ${repo} ${arch} + repo_unlock "${repo}" "${arch}" done done @@ -74,7 +74,7 @@ for repo in "${PKGREPOS[@]}"; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then # Check if we had failed before - if in_array "${pkgbase}-${pkgver}${SRCEXT}" ${failedpkgs[@]}; then + if in_array "${pkgbase}-${pkgver}${SRCEXT}" "${failedpkgs[@]}"; then continue fi @@ -124,22 +124,22 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." - ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + "${SOURCE_CLEANUP_DRYRUN}" && warning 'dry run mode is active' for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" - if ! ${SOURCE_CLEANUP_DRYRUN}; then + if ! "${SOURCE_CLEANUP_DRYRUN}"; then mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done fi -old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find "${SOURCE_CLEANUP_DESTDIR}" -type f -name "*${SRCEXT}" -mtime +"${SOURCE_CLEANUP_KEEP}" -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" - ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + "${SOURCE_CLEANUP_DRYRUN}" || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done fi diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 713e75e..ec9d255 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -43,26 +43,26 @@ case "$cmd" in esac # Lock the repos and get a copy of the db files to work on -for repo in ${REPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 +for repo in "${REPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" if [ -f "${dbfile}" ]; then mkdir -p "${WORKDIR}/${repo}/${arch}" cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" fi - repo_unlock ${repo} ${arch} + repo_unlock "${repo}" "${arch}" done done # Run reporead on our db copy -pushd $SPATH >/dev/null -for repo in ${REPOS[@]}; do - for arch in ${ARCHES[@]}; do +pushd "$SPATH" >/dev/null +for repo in "${REPOS[@]}"; do + for arch in "${ARCHES[@]}"; do dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" if [ -f "${dbcopy}" ]; then echo "Updating ${repo}-${arch}" >> "${LOGOUT}" - ./manage.py reporead ${flags} ${arch} "${dbcopy}" >> "${LOGOUT}" 2>&1 + ./manage.py reporead "${flags}" "${arch}" "${dbcopy}" >> "${LOGOUT}" 2>&1 echo "" >> "${LOGOUT}" fi done -- cgit v1.2.3 From b1ac233691ac28645cb729d7d5e7c7b51fc899bd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 23:11:44 -0400 Subject: cron-jobs/repo-sanity-check: clean up --- cron-jobs/repo-sanity-check | 95 ++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 8b0758f..73ec171 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,57 +1,54 @@ #!/bin/bash -# Solves issue165 +# Solves issue165... on the old flyspray install. I have no idea what issue that was. . "$(dirname "$(readlink -e "$0")")/../config" . "$(dirname "$(readlink -e "$0")")/../db-functions" # Traverse all repos for _repo in "${PKGREPOS[@]}"; do - msg "Cleaning up [${_repo}]" - -# Find all pkgnames on this repo's abs - on_abs=($( - find ${SVNREPO}/${_repo} -name PKGBUILD | \ - while read pkgbuild; do - source ${pkgbuild} >/dev/null 2>&1 -# cleanup to save memory - unset build package source md5sums pkgdesc pkgver pkgrel epoch \ - url license arch depends makedepends optdepends options \ - >/dev/null 2>&1 - -# also cleanup package functions - for _pkg in "${pkgname[@]}"; do - unset package_${pkg} >/dev/null 2>&1 - done - -# this fills the on_abs array - echo ${pkgname[@]} - done - )) - -# quit if abs is empty - if [ ${#on_abs[*]} -eq 0 ]; then - warning "[${_repo}]'s ABS tree is empty, skipping" - break - fi - -# Find all pkgnames on repos - on_repo=($( - find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ - sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" - )) - -# Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 \ - <(printf '%s\n' "${on_abs[@]}" | sort -u) \ - <(printf '%s\n' "${on_repo[@]}" | sort -u) )) - -# Remove them from databases, ftpdir-cleanup will take care of the rest - find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ - repo-remove {} ${remove[@]} >/dev/null 2>&1 \; - - msg2 "Removed the following packages:" - plain "$(echo ${remove[@]} | tr ' ' "\n")" - + msg "Cleaning up [${_repo}]" + + # Find all pkgnames on this repo's abs + on_abs=($( + find "${SVNREPO}/${_repo}" -name PKGBUILD | \ + while read pkgbuild; do + source "${pkgbuild}" >/dev/null 2>&1 + # cleanup to save memory + unset build package source md5sums pkgdesc pkgver pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + + # also cleanup package functions + for _pkg in "${pkgname[@]}"; do + unset "package_${pkg}" >/dev/null 2>&1 + done + + # this fills the on_abs array + printf '%q ' "${pkgname[@]}" + done + )) + + # quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[%s]'s ABS tree is empty, skipping" "${_repo}" + break + fi + + # Find all pkgnames on repos + on_repo=($( + find "${FTP_BASE}/${_repo}" -name "*.pkg.tar.?z" \ + -printf "%f\n" | sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + )) + + # Compares them, whatever is on repos but not on abs should be removed + remove=($(comm -13 \ + <(printf '%s\n' "${on_abs[@]}" | sort -u) \ + <(printf '%s\n' "${on_repo[@]}" | sort -u) )) + + # Remove them from databases, ftpdir-cleanup will take care of the rest + find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \ + -exec repo-remove {} "${remove[@]}" \; >/dev/null 2>&1 + + msg2 "Removed the following packages:" + plain '%s' "${remove[@]}" done - -exit $? -- cgit v1.2.3 From c54e53593927e5469cfe13bacd29d25168235606 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 23:23:31 -0400 Subject: more quoting fixes --- cron-jobs/ftpdir-cleanup | 2 +- db-functions | 30 +++++++++++++++--------------- db-move | 4 ++-- db-remove | 2 +- test/lib/common.inc | 12 ++++++------ test/test.d/create-filelists.sh | 34 +++++++++++++++++----------------- 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 292952b..77d90d2 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -53,7 +53,7 @@ for repo in "${PKGREPOS[@]}"; do old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from [${repo}] (${arch})..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" done diff --git a/db-functions b/db-functions index 5b10e05..86a2b1e 100644 --- a/db-functions +++ b/db-functions @@ -12,7 +12,7 @@ set_umask () { } restore_umask () { - umask $UMASK >/dev/null + umask "$UMASK" >/dev/null } # just like mv -f, but we touch the file and then copy the content so @@ -39,7 +39,7 @@ REPO_MODIFIED=0 script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" error "Script ${0##*/} is already locked by $_owner." exit 1 else @@ -72,7 +72,7 @@ cleanup() { arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then msg "Removing left over lock from [${repo}] (${arch})" - repo_unlock $repo $arch + repo_unlock "$repo" "$arch" fi done if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then @@ -85,7 +85,7 @@ cleanup() { date +%s > "${FTP_BASE}/lastupdate" fi - [ "$1" ] && exit $1 + [ "$1" ] && exit "$1" } abort() { @@ -133,9 +133,9 @@ repo_lock () { fi _count=0 - while [ $_count -le $_trial ] || $_lockblock ; do + while [ "$_count" -le "$_trial" ] || "$_lockblock" ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -143,7 +143,7 @@ repo_lock () { set_umask return 0 fi - sleep $LOCK_DELAY + sleep "$LOCK_DELAY" let _count=$_count+1 done @@ -241,12 +241,12 @@ getpkgfile() { elif [ ! -f "${1}" ]; then error "Package ${1} not found!" exit 1 - elif ${REQUIRE_SIGNATURE} && [ ! -f "${1}.sig" ]; then + elif "${REQUIRE_SIGNATURE}" && [ ! -f "${1}.sig" ]; then error "Package signature ${1}.sig not found!" exit 1 fi - echo ${1} + echo "${1}" } getpkgfiles() { @@ -260,7 +260,7 @@ getpkgfiles() { if [ ! -f "${f}" ]; then error "Package ${f} not found!" exit 1 - elif ${REQUIRE_SIGNATURE} && [ ! -f "${f}.sig" ]; then + elif "${REQUIRE_SIGNATURE}" && [ ! -f "${f}.sig" ]; then error "Package signature ${f}.sig not found!" exit 1 fi @@ -272,11 +272,11 @@ getpkgfiles() { check_pkgfile() { local pkgfile=$1 - local pkgname="$(getpkgname ${pkgfile})" + local pkgname="$(getpkgname "${pkgfile}")" [ $? -ge 1 ] && return 1 - local pkgver="$(getpkgver ${pkgfile})" + local pkgver="$(getpkgver "${pkgfile}")" [ $? -ge 1 ] && return 1 - local pkgarch="$(getpkgarch ${pkgfile})" + local pkgarch="$(getpkgarch "${pkgfile}")" [ $? -ge 1 ] && return 1 in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 @@ -305,7 +305,7 @@ check_pkgxbs() { local xbsver="$(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; get_full_version "${_pkgname}")" [ "${xbsver}" == "${_pkgver}" ] || return 1 - local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo ${pkgname[@]})) + eval "local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" in_array "${_pkgname}" "${xbsnames[@]}" || return 1 return 0 @@ -331,7 +331,7 @@ check_splitpkgs() { mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" - local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo ${pkgname[@]})) + eval "local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" for xbsname in "${xbsnames[@]}"; do echo "${xbsname}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" done diff --git a/db-move b/db-move index 5fe9691..fe94883 100755 --- a/db-move +++ b/db-move @@ -29,7 +29,7 @@ for pkgbase in "${args[@]:2}"; do for pkgarch in "${ARCHES[@]}" 'any'; do xbsrepo_from="$(xbs releasepath ${pkgbase} ${repo_from} ${pkgarch})" if [ -r "${xbsrepo_from}/PKGBUILD" ]; then - pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; echo ${pkgname[@]})) + eval "pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" if [ ${#pkgnames[@]} -lt 1 ]; then die "Could not read pkgname" fi @@ -71,7 +71,7 @@ for pkgbase in "${args[@]:2}"; do else tarches=("${pkgarch}") fi - pkgnames=($(. "${dir_to}/PKGBUILD"; echo ${pkgname[@]})) + eval "pkgnames=($(. "${dir_to}/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" for pkgname in "${pkgnames[@]}"; do pkgver=$(. "${dir_to}/PKGBUILD"; get_full_version ${pkgname}) diff --git a/db-remove b/db-remove index 33d0933..73919be 100755 --- a/db-remove +++ b/db-remove @@ -31,7 +31,7 @@ for pkgbase in "${pkgbases[@]}"; do msg "Removing $pkgbase from [$repo]..." path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" if [ -d "$path" ]; then - remove_pkgs+=($(. "$path/PKGBUILD"; echo ${pkgname[@]})) + eval "remove_pkgs+=($(. "$path/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" xbs unrelease "$pkgbase" "$repo" "$arch" else warning "$pkgbase not found in XBS $repo-$arch" diff --git a/test/lib/common.inc b/test/lib/common.inc index 2f308dd..a241615 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -52,8 +52,8 @@ oneTimeSetUp() { msg 'Building packages...' for d in "${pkgdir}"/*; do pushd $d >/dev/null - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - pkgarch=($(. PKGBUILD; echo ${arch[@]})) + eval "pkgname=($(. PKGBUILD; printf '%q ' "${pkgname[@]}"))" + eval "pkgarch=($(. PKGBUILD; printf '%q ' "${arch[@]}"))" pkgversion=$(. PKGBUILD; get_full_version) build=true @@ -170,13 +170,13 @@ releasePackage() { pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null xbs release ${repo} ${arch} >/dev/null 2>&1 pkgver=$(. PKGBUILD; get_full_version) - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) + eval "pkgname=($(. PKGBUILD; printf '%q ' "${pkgname[@]}"))" popd >/dev/null cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ - if ${REQUIRE_SIGNATURE}; then - for a in ${arch[@]}; do - for p in ${pkgname[@]}; do + if "${REQUIRE_SIGNATURE}"; then + for a in "${arch[@]}"; do + for p in "${pkgname[@]}"; do signpkg "${STAGING}"/${repo}/${p}-${pkgver}-${a}${PKGEXT} done done diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index e78bde8..1126972 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -1,6 +1,6 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) +curdir="$(readlink -e "$(dirname "$0")")" . "${curdir}/../lib/common.inc" testCreateSimpleFileLists() { @@ -9,15 +9,15 @@ testCreateSimpleFileLists() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${arches[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do + for pkgbase in "${pkgs[@]}"; do + for arch in "${arches[@]}"; do if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/${pkgbase}" &>/dev/null; then fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}" fi @@ -31,13 +31,13 @@ testCreateAnyFileLists() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" any done ../db-update - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do + for pkgbase in "${pkgs[@]}"; do + for arch in "${arches[@]}"; do if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/share/${pkgbase}/test" &>/dev/null; then fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}" fi @@ -54,17 +54,17 @@ testCreateSplitFileLists() { local pkgnames local arch - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} + for pkgbase in "${pkgs[@]}"; do + for arch in "${arches[@]}"; do + releasePackage extra "${pkgbase}" "${arch}" done done ../db-update - for pkgbase in ${pkgs[@]}; do - pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) - for pkgname in ${pkgnames[@]}; do - for arch in ${arches[@]}; do + for pkgbase in "${pkgs[@]}"; do + eval "pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" + for pkgname in "${pkgnames[@]}"; do + for arch in "${arches[@]}"; do if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/${pkgname}" &>/dev/null; then fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}" fi -- cgit v1.2.3 From 63381988ae8a0f4e0c8eaede88b43a5f685ec80f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 23:25:11 -0400 Subject: remove more scripts that I know to be obsolete --- cron-jobs/update-web-db | 78 ------------------------------------------- cron-jobs/update-web-files-db | 1 - migrate-repo | 29 ---------------- repo-restore-to-normal | 58 -------------------------------- 4 files changed, 166 deletions(-) delete mode 100755 cron-jobs/update-web-db delete mode 120000 cron-jobs/update-web-files-db delete mode 100755 migrate-repo delete mode 100755 repo-restore-to-normal diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db deleted file mode 100755 index 713e75e..0000000 --- a/cron-jobs/update-web-db +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/../config" -. "$(dirname "$(readlink -e "$0")")/../db-functions" - -# setup paths -SPATH="/srv/http/archweb" -ENVPATH="/srv/http/archweb-env/bin/activate" - -# having "more important repos" last should make [core] trickle to the top of -# the updates list each hour rather than being overwhelmed by big [extra] and -# [community] updates -REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'testing' 'core') -LOGOUT="/tmp/archweb_update.log" - -# figure out what operation to perform -cmd="${0##*/}" -if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - die "Invalid command name '$cmd' specified!" -fi - -script_lock - -# run at nice 5. it can churn quite a bit of cpu after all. -renice +5 -p $$ > /dev/null - -echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" - -# source our virtualenv if it exists -if [ -f "$ENVPATH" ]; then - . "$ENVPATH" -fi - -case "$cmd" in - update-web-db) - dbfileext="${DBEXT}" - flags="" - ;; - update-web-files-db) - dbfileext="${FILESEXT}" - flags="--filesonly" - ;; -esac - -# Lock the repos and get a copy of the db files to work on -for repo in ${REPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" - if [ -f "${dbfile}" ]; then - mkdir -p "${WORKDIR}/${repo}/${arch}" - cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" - fi - repo_unlock ${repo} ${arch} - done -done - -# Run reporead on our db copy -pushd $SPATH >/dev/null -for repo in ${REPOS[@]}; do - for arch in ${ARCHES[@]}; do - dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" - if [ -f "${dbcopy}" ]; then - echo "Updating ${repo}-${arch}" >> "${LOGOUT}" - ./manage.py reporead ${flags} ${arch} "${dbcopy}" >> "${LOGOUT}" 2>&1 - echo "" >> "${LOGOUT}" - fi - done -done -popd >/dev/null -echo "" >> "${LOGOUT}" - -# rotate the file if it is getting big (> 10M), overwriting any old backup -if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then - mv "${LOGOUT}" "${LOGOUT}.old" -fi - -script_unlock diff --git a/cron-jobs/update-web-files-db b/cron-jobs/update-web-files-db deleted file mode 120000 index 0c2c4fa..0000000 --- a/cron-jobs/update-web-files-db +++ /dev/null @@ -1 +0,0 @@ -update-web-db \ No newline at end of file diff --git a/migrate-repo b/migrate-repo deleted file mode 100755 index 751d5bd..0000000 --- a/migrate-repo +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -source "$(dirname "$(readlink -e "$0")")/config" - -#dryrun="--dry-run" - -# Sync our repos -#for parabola_repo in ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}; do -for parabola_repo in ${USERREPOS[@]} ${PROJREPOS[@]}; do - echo ":: Syncing ${parabola_repo}/ => ${FTP_BASE}/${parabola_repo}/" - rsync -avL ${dryrun} --progress -e ssh parabolavnx@69.163.153.218:repo/${parabola_repo}/ ${FTP_BASE}/${parabola_repo}/ -done - -# Sync our arches -#for parabola_arch in ${OURARCHES[@]}; do -#for arch_repo in ${ARCHREPOS[@]}; do - #echo ":: Syncing ${arch_repo}/os/${parabola_arch}/ => ${FTP_BASE}/${arch_repo}/os/${parabola_arch}/" -# rsync -avL ${dryrun} --progress -e ssh parabolavnx@69.163.153.218:repo/${arch_repo}/os/${parabola_arch}/ ${FTP_BASE}/${arch_repo}/os/${parabola_arch}/ -#done -#done -# -# Sync other dirs last -#for other in screenshots isos; do -for other in other; do - echo ":: Syncing ${other}/ => ${FTP_BASE}/${other}/" - rsync -avL ${dryrun} --progress -e ssh parabolavnx@69.163.153.218:repo/${other}/ ${FTP_BASE}/${other} -done - -exit $? diff --git a/repo-restore-to-normal b/repo-restore-to-normal deleted file mode 100755 index 063aacf..0000000 --- a/repo-restore-to-normal +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Solves issue165 - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore -PKGREPOS=(community) - -# Find all pkgnames on old with pkgver-pkgrels -#on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \ -# sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/")) - -# Traverse all repos -for _repo in "${PKGREPOS[@]}"; do - msg "Restoring [${_repo}]" - -# Find all pkgnames on this repo's abs - on_abs=($( - find ${SVNREPO}/${_repo} -name PKGBUILD | \ - while read pkgbuild; do - unset pkgname pkgver pkgrel - source ${pkgbuild} >/dev/null 2>&1 -# cleanup to save memory - unset build package source md5sums pkgdesc epoch \ - url license arch depends makedepends optdepends options \ - >/dev/null 2>&1 - -# also cleanup package functions - for _pkg in "${pkgname[@]}"; do - unset package_${pkg} >/dev/null 2>&1 -# this fills the on_abs array - echo ${_pkg}-${pkgver}-${pkgrel} - done - - done - )) - -# quit if abs is empty - if [ ${#on_abs[*]} -eq 0 ]; then - warning "[${_repo}]'s ABS tree is empty, skipping" - continue - fi - -# Compares them, whatever is on abs should be restored -# restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ -# <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) - - msg2 "Restoring the following packages:" -# plain "$(echo ${restore[@]} | tr ' ' "\n")" - - for _pkg in "${on_abs[@]}"; do - find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \; - done - -done - -exit $? -- cgit v1.2.3 From 5b6036ac62755ebc7414e0583abeeab33bb75644 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 00:08:51 -0400 Subject: db-cleanup: fix dryrun support, and obey TMPDIR --- db-cleanup | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/db-cleanup b/db-cleanup index 2051130..67fb051 100755 --- a/db-cleanup +++ b/db-cleanup @@ -28,9 +28,12 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR EXTRAFLAGS=() "${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) +filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") +trap "rm $(printf %q "$filter")" EXIT + for _repo in "${PKGREPOS[@]}"; do for _arch in "${ARCHES[@]}"; do - msg "Getting ${_repo}-${_arch} database" + msg "Getting %s database" "${_repo}-${_arch}" dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" @@ -43,7 +46,7 @@ for _repo in "${PKGREPOS[@]}"; do bsdtar tf "${dbfile}" | \ cut -d'/' -f1 | \ sort -u | \ - sed "s|$|*|" >> "/tmp/${0##*/}.$$.filter" + sed "s|$|*|" >> "$filter" done done @@ -54,7 +57,7 @@ for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do msg2 "${POOL}" rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ - --include-from="/tmp/${0##*/}.$$.filter" \ + --include-from="$filter" \ --exclude="*" \ "${FTP_BASE}/${POOL}/" \ "${FTP_BASE}/${POOL}/" @@ -63,4 +66,4 @@ done msg "Removing dead symlinks:" actions=(-print) "${CLEANUP_DRYRUN}" || actions+=(-delete) -find -L "${FTP_BASE}/" -type l -print -delete +find -L "${FTP_BASE}/" -type l "${actions[@]}" -- cgit v1.2.3 From 878a9afd8fe40fa4263c5af04491ff91c8f0c09a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 00:10:08 -0400 Subject: Use printf-formatters when possible (incomplete) --- cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/integrity-check | 2 +- cron-jobs/sourceballs | 8 ++++---- cron-jobs/sourceballs2 | 4 ++-- db-check-nonfree | 6 +++--- db-functions | 26 +++++++++++++------------- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 77d90d2..af56aac 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -44,7 +44,7 @@ for repo in "${PKGREPOS[@]}"; do missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#missing_pkgs[@]} -ge 1 ]; then - error "Missing packages in [${repo}] (${arch})..." + error "Missing packages in [%s] (%s)..." "${repo}" "${arch}" for missing_pkg in "${missing_pkgs[@]}"; do msg2 "${missing_pkg}" done diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 33a4eb6..7459380 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -8,7 +8,7 @@ dirname="$(dirname "$(readlink -e "$0")")" script_lock if [ $# -ne 1 ]; then - die "usage: ${0##*/} " + die "usage: %s " "${0##*/}" fi mailto=$1 diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 2c11de2..2fc09d5 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -104,15 +104,15 @@ for repo in "${PKGREPOS[@]}"; do done < "${WORKDIR}/db-${repo}" if [ ${#newpkgs[@]} -ge 1 ]; then - msg "Adding source packages for [${repo}]..." + msg "Adding source packages for [%s]..." "${repo}" for new_pkg in "${newpkgs[@]}"; do - msg2 "${new_pkg}" + msg2 '%s' "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then - msg "Failed to create source packages for [${repo}]..." + msg "Failed to create source packages for [%s]..." "${repo}" for failed_pkg in "${failedpkgs[@]}"; do - msg2 "${failed_pkg}" + msg2 '%s' "${failed_pkg}" done fi done diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 64bae4a..02ed04c 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -22,14 +22,14 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null for repo in "${PKGREPOS[@]}"; do - msg "Sourceballing [${repo}]" + msg "Sourceballing [%s]" "${repo}" pushd "$repo" >/dev/null find -maxdepth 1 -type d | while read pkg; do pushd "${SVNREPO}/$repo/$pkg" >/dev/null [[ ! -e ./PKGBUILD ]] && { - warning "$repo/$pkg is not a package" + warning "%s is not a package" "$repo/$pkg" continue } diff --git a/db-check-nonfree b/db-check-nonfree index 253490b..bea0fa8 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [[ $# -ge 1 ]]; then - warning "Calling ${0##*/} with a specific repository is not supported" + error "Calling %s with a specific repository is not supported" "${0##*/}" exit 1 fi @@ -19,7 +19,7 @@ msg "Check nonfree in repo:" nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u)) for repo in "${ARCHREPOS[@]}"; do for pkgarch in "${ARCHES[@]}"; do - msg2 "$repo $pkgarch" + msg2 "%s %s" "$repo" "$pkgarch" if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue fi @@ -33,7 +33,7 @@ for repo in "${ARCHREPOS[@]}"; do fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: ${cleanpkgs[*]}" + msg2 "Nonfree: %s" "${cleanpkgs[*]}" arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" fi done diff --git a/db-functions b/db-functions index 86a2b1e..a2f638e 100644 --- a/db-functions +++ b/db-functions @@ -40,7 +40,7 @@ script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then local _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" - error "Script ${0##*/} is already locked by $_owner." + error "Script %s is already locked by $_owner." "${0##*/}" exit 1 else set_umask @@ -51,7 +51,7 @@ script_lock() { script_unlock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if [ ! -d "$LOCKDIR" ]; then - warning "Script ${0##*/} was not locked!" + warning "Script %s was not locked!" "${0##*/}" restore_umask return 1 else @@ -71,12 +71,12 @@ cleanup() { repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then - msg "Removing left over lock from [${repo}] (${arch})" + msg "Removing left over lock from [%s] (%s)" "${repo}" "${arch}" repo_unlock "$repo" "$arch" fi done if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then - msg "Removing left over lock from ${0##*/}" + msg "Removing left over lock from %s" "${0##*/}" script_unlock fi rm -rf "$WORKDIR" @@ -115,11 +115,11 @@ repo_lock () { # This is the lock file used by repo-add and repo-remove if [ -f "${DBLOCKFILE}" ]; then - error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $DBLOCKFILE)" + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$DBLOCKFILE")" return 1 fi if [ -f "${FILESLOCKFILE}" ]; then - error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat ${FILESLOCKFILE})" + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$FILESLOCKFILE")" return 1 fi @@ -136,8 +136,8 @@ repo_lock () { while [ "$_count" -le "$_trial" ] || "$_lockblock" ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" - warning "Repo [${1}] (${2}) is already locked by $_owner. " - msg2 "Retrying in $LOCK_DELAY seconds..." + warning "Repo [%s] (%s) is already locked by %s. " "${1}" "${2}" "$_owner" + msg2 "Retrying in %d seconds..." "$LOCK_DELAY" else LOCKS[${#LOCKS[*]}]="$1.$2" set_umask @@ -147,14 +147,14 @@ repo_lock () { let _count=$_count+1 done - error "Repo [${1}] (${2}) is already locked by $_owner. Giving up!" + error "Repo [%s] (%s) is already locked by %s. Giving up!" "${1}" "${2}" "$_owner" return 1 } repo_unlock () { #repo_unlock local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [ ! -d "$LOCKDIR" ]; then - warning "Repo lock [${1}] (${2}) was not locked!" + warning "Repo lock [%s] (%s) was not locked!" "${1}" "${2}" restore_umask return 1 else @@ -202,7 +202,7 @@ getpkgname() { _name="$(_grep_pkginfo "$1" "pkgname")" if [ -z "$_name" ]; then - error "Package '$1' has no pkgname in the PKGINFO. Fail!" + error "Package '%s' has no pkgname in the PKGINFO. Fail!" "$1" exit 1 fi @@ -215,7 +215,7 @@ getpkgver() { _ver="$(_grep_pkginfo "$1" "pkgver")" if [ -z "$_ver" ]; then - error "Package '$1' has no pkgver in the PKGINFO. Fail!" + error "Package '%s' has no pkgver in the PKGINFO. Fail!" "$1" exit 1 fi @@ -227,7 +227,7 @@ getpkgarch() { _ver="$(_grep_pkginfo "$1" "arch")" if [ -z "$_ver" ]; then - error "Package '$1' has no arch in the PKGINFO. Fail!" + error "Package '%s' has no arch in the PKGINFO. Fail!" "$1" exit 1 fi -- cgit v1.2.3 From 5e84c1596d3ae2a66bd859f1fac2490025946fbb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 00:13:55 -0400 Subject: use += when possible --- cron-jobs/sourceballs | 2 +- db-functions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 2fc09d5..44492ef 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -94,7 +94,7 @@ for repo in "${PKGREPOS[@]}"; do mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" - newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + newpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") else failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") cat "${WORKDIR}/${pkgbase}.log" >> "${WORKDIR}/makepkg-fail.log" diff --git a/db-functions b/db-functions index a2f638e..006dde9 100644 --- a/db-functions +++ b/db-functions @@ -139,7 +139,7 @@ repo_lock () { warning "Repo [%s] (%s) is already locked by %s. " "${1}" "${2}" "$_owner" msg2 "Retrying in %d seconds..." "$LOCK_DELAY" else - LOCKS[${#LOCKS[*]}]="$1.$2" + LOCKS+=("$1.$2") set_umask return 0 fi -- cgit v1.2.3 From 426c14d9a8364193dec631a389470fb4465ef9ce Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 00:12:25 -0400 Subject: cron-jobs/update-abs-tarbals: quote, remove unnescessary exit --- cron-jobs/update-abs-tarballs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs index 901cc4b..e710f7c 100755 --- a/cron-jobs/update-abs-tarballs +++ b/cron-jobs/update-abs-tarballs @@ -2,6 +2,4 @@ . "$(dirname "$(readlink -e "$0")")/../config" -rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ - -exit $? +rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ "${FTP_BASE}/" -- cgit v1.2.3 From 43f8af33f08924092826e2094d15be704e842f3a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 00:32:24 -0400 Subject: more quoting and printf fixes --- db-functions | 52 ++++++++++++++++++++++++++-------------------------- db-move | 34 +++++++++++++++++----------------- db-remove | 16 ++++++++-------- db-repo-add | 26 +++++++++++++------------- db-repo-remove | 24 ++++++++++++------------ 5 files changed, 76 insertions(+), 76 deletions(-) diff --git a/db-functions b/db-functions index 006dde9..6e27148 100644 --- a/db-functions +++ b/db-functions @@ -40,7 +40,7 @@ script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then local _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" - error "Script %s is already locked by $_owner." "${0##*/}" + error "Script %s is already locked by %s." "${0##*/}" "$_owner" exit 1 else set_umask @@ -136,7 +136,7 @@ repo_lock () { while [ "$_count" -le "$_trial" ] || "$_lockblock" ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" - warning "Repo [%s] (%s) is already locked by %s. " "${1}" "${2}" "$_owner" + warning "Repo [%s] (%s) is already locked by %s." "${1}" "${2}" "$_owner" msg2 "Retrying in %d seconds..." "$LOCK_DELAY" else LOCKS+=("$1.$2") @@ -239,10 +239,10 @@ getpkgfile() { error 'No canonical package found!' exit 1 elif [ ! -f "${1}" ]; then - error "Package ${1} not found!" + error "Package %s not found!" "${1}" exit 1 elif "${REQUIRE_SIGNATURE}" && [ ! -f "${1}.sig" ]; then - error "Package signature ${1}.sig not found!" + error "Package signature %s not found!" "${1}.sig" exit 1 fi @@ -258,10 +258,10 @@ getpkgfiles() { for f in "${@}"; do if [ ! -f "${f}" ]; then - error "Package ${f} not found!" + error "Package %s not found!" "${f}" exit 1 elif "${REQUIRE_SIGNATURE}" && [ ! -f "${f}.sig" ]; then - error "Package signature ${f}.sig not found!" + error "Package signature %s not found!" "${f}.sig" exit 1 fi done @@ -290,13 +290,13 @@ check_pkgfile() { check_pkgxbs() { local pkgfile="${1}" - local _pkgbase="$(getpkgbase ${pkgfile})" + local _pkgbase="$(getpkgbase "${pkgfile}")" [ $? -ge 1 ] && return 1 - local _pkgname="$(getpkgname ${pkgfile})" + local _pkgname="$(getpkgname "${pkgfile}")" [ $? -ge 1 ] && return 1 - local _pkgver="$(getpkgver ${pkgfile})" + local _pkgver="$(getpkgver "${pkgfile}")" [ $? -ge 1 ] && return 1 - local _pkgarch="$(getpkgarch ${pkgfile})" + local _pkgarch="$(getpkgarch "${pkgfile}")" [ $? -ge 1 ] && return 1 local repo="${2}" @@ -324,10 +324,10 @@ check_splitpkgs() { for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue - local _pkgbase="$(getpkgbase ${pkgfile})" + local _pkgbase="$(getpkgbase "${pkgfile}")" msg2 "Checking %s" "$_pkgbase" - local _pkgname="$(getpkgname ${pkgfile})" - local _pkgarch="$(getpkgarch ${pkgfile})" + local _pkgname="$(getpkgname "${pkgfile}")" + local _pkgarch="$(getpkgarch "${pkgfile}")" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" @@ -353,11 +353,11 @@ check_splitpkgs() { check_pkgrepos() { local pkgfile=$1 - local pkgname="$(getpkgname ${pkgfile})" + local pkgname="$(getpkgname "${pkgfile}")" [ $? -ge 1 ] && return 1 - local pkgver="$(getpkgver ${pkgfile})" + local pkgver="$(getpkgver "${pkgfile}")" [ $? -ge 1 ] && return 1 - local pkgarch="$(getpkgarch ${pkgfile})" + local pkgarch="$(getpkgarch "${pkgfile}")" [ $? -ge 1 ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 @@ -389,11 +389,11 @@ check_repo_permission() { [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 local arch - for arch in ${ARCHES}; do + for arch in "${ARCHES[@]}"; do local dir="${FTP_BASE}/${repo}/os/${arch}/" [ -w "${dir}" ] || return 1 - [ -f "${dir}"${repo}${DBEXT} -a ! -w "${dir}"${repo}${DBEXT} ] && return 1 - [ -f "${dir}"${repo}${FILESEXT} -a ! -w "${dir}"${repo}${FILESEXT} ] && return 1 + [ -f "${dir}${repo}"${DBEXT} -a ! -w "${dir}${repo}"${DBEXT} ] && return 1 + [ -f "${dir}${repo}"${FILESEXT} -a ! -w "${dir}${repo}"${FILESEXT} ] && return 1 done return 0 @@ -407,12 +407,12 @@ set_repo_permission() { if [ -w "${dbfile}" ]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") - chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" - chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group" - chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" - chmod g+w "${filesfile}" || error "Could not set write permission for group $group to ${filesfile}" + chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "${dbfile}" "$group" + chgrp "$group" "${filesfile}" || error "Could not change group of %s to %s" "${filesfile}" "$group" + chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "${dbfile}" + chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "${filesfile}" else - error "You don't have permission to change ${dbfile}" + error "You don't have permission to change %s" "${dbfile}" fi } @@ -424,9 +424,9 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ - || error "repo-add ${repo}${DBEXT} ${pkgs[*]}" + || error '%s' "repo-add ${repo}${DBEXT} ${pkgs[*]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ - || error "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" + || error '%s' "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" diff --git a/db-move b/db-move index fe94883..f6588c3 100755 --- a/db-move +++ b/db-move @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi @@ -14,14 +14,14 @@ repo_to="${args[1]}" ftppath_from="${FTP_BASE}/${repo_from}/os/" ftppath_to="${FTP_BASE}/${repo_to}/os/" -if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then +if ! check_repo_permission "$repo_to" || ! check_repo_permission "$repo_from"; then die "You don't have permission to move packages from ${repo_from} to ${repo_to}" fi # TODO: this might lock too much (architectures) for pkgarch in "${ARCHES[@]}"; do - repo_lock ${repo_to} ${pkgarch} || exit 1 - repo_lock ${repo_from} ${pkgarch} || exit 1 + repo_lock "${repo_to}" "${pkgarch}" || exit 1 + repo_lock "${repo_from}" "${pkgarch}" || exit 1 done # First loop is to check that all necessary files exist @@ -41,30 +41,30 @@ for pkgbase in "${args[@]:2}"; do fi for pkgname in "${pkgnames[@]}"; do - pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version ${pkgname}) + pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version "${pkgname}") if [ -z "${pkgver}" ]; then die "Could not read pkgver" fi for tarch in "${tarches[@]}"; do - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null + getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} >/dev/null done done continue 2 fi done - die "${pkgbase} not found in ${repo_from}" + die "%s not found in %s" "${pkgbase}" "${repo_from}" done -msg "Moving packages from [${repo_from}] to [${repo_to}]..." +msg "Moving packages from [%s] to [%s]..." "${repo_from}" "${repo_to}" declare -A add_pkgs declare -A remove_pkgs for pkgbase in "${args[@]:2}"; do # move the package in xbs - arches=($(xbs move ${repo_from} ${repo_to} ${pkgbase})) + arches=($(xbs move "${repo_from}" "${repo_to}" "${pkgbase}")) # move the package in ftp for pkgarch in "${arches[@]}"; do - dir_to="$(xbs releasepath $pkgbase $repo_to $pkgarch)" + dir_to="$(xbs releasepath "$pkgbase" "$repo_to" "$pkgarch")" if true; then # to add in indent level to make merging easier if [ "${pkgarch}" == 'any' ]; then tarches=("${ARCHES[@]}") @@ -74,14 +74,14 @@ for pkgbase in "${args[@]:2}"; do eval "pkgnames=($(. "${dir_to}/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" for pkgname in "${pkgnames[@]}"; do - pkgver=$(. "${dir_to}/PKGBUILD"; get_full_version ${pkgname}) + pkgver=$(. "${dir_to}/PKGBUILD"; get_full_version "${pkgname}") for tarch in "${tarches[@]}"; do - pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) + pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}) pkgfile="${pkgpath##*/}" - ln -s "../../../${PKGPOOL}/${pkgfile}" ${ftppath_to}/${tarch}/ - if [ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" ${ftppath_to}/${tarch}/ + ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" + if [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig" ]; then + ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" fi add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " remove_pkgs[${tarch}]+="${pkgname} " @@ -99,6 +99,6 @@ for tarch in "${ARCHES[@]}"; do done for pkgarch in "${ARCHES[@]}"; do - repo_unlock ${repo_from} ${pkgarch} - repo_unlock ${repo_to} ${pkgarch} + repo_unlock "${repo_from}" "${pkgarch}" + repo_unlock "${repo_to}" "${pkgarch}" done diff --git a/db-remove b/db-remove index 73919be..970d252 100755 --- a/db-remove +++ b/db-remove @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi @@ -12,8 +12,8 @@ repo="$1" arch="$2" pkgbases=("${@:3}") -if ! check_repo_permission $repo; then - die "You don't have permission to remove packages from ${repo}" +if ! check_repo_permission "$repo"; then + die "You don't have permission to remove packages from %s" "${repo}" fi if [ "$arch" == "any" ]; then @@ -23,19 +23,19 @@ else fi for tarch in "${tarches[@]}"; do - repo_lock $repo $tarch || exit 1 + repo_lock "$repo" "$tarch" || exit 1 done remove_pkgs=() for pkgbase in "${pkgbases[@]}"; do - msg "Removing $pkgbase from [$repo]..." + msg "Removing %s from [%s]..." "$pkgbase" "$repo" path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" if [ -d "$path" ]; then eval "remove_pkgs+=($(. "$path/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" xbs unrelease "$pkgbase" "$repo" "$arch" else - warning "$pkgbase not found in XBS $repo-$arch" - warning "Removing only $pkgbase from the repo" + warning "%s not found in XBS %s" "$pkgbase" "$repo-$arch" + warning "Removing only %s from the repo" "$pkgbase" warning "If it was a split package you have to remove the others yourself!" remove_pkgs+=("$pkgbase") fi @@ -43,5 +43,5 @@ done for tarch in "${tarches[@]}"; do arch_repo_remove "${repo}" "${tarch}" "${remove_pkgs[@]}" - repo_unlock $repo $tarch + repo_unlock "$repo" "$tarch" done diff --git a/db-repo-add b/db-repo-add index a6355a1..4611bdf 100755 --- a/db-repo-add +++ b/db-repo-add @@ -4,38 +4,38 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi repo="$1" arch="$2" -pkgfiles=(${@:3}) +pkgfiles=("${@:3}") ftppath="$FTP_BASE/$repo/os" -if ! check_repo_permission $repo; then - die "You don't have permission to add packages to ${repo}" +if ! check_repo_permission "$repo"; then + die "You don't have permission to add packages to %s" "${repo}" fi if [ "$arch" == "any" ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do - repo_lock $repo $tarch || exit 1 +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 done -for tarch in ${tarches[@]}; do - for pkgfile in ${pkgfiles[@]}; do +for tarch in "${tarches[@]}"; do + for pkgfile in "${pkgfiles[@]}"; do if [[ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ]]; then - die "Package file ${pkgfile##*/} not found in ${FTP_BASE}/${repo}/os/${arch}/" + die "Package file %s not found in %s" "${pkgfile##*/}" "${FTP_BASE}/${repo}/os/${arch}/" else - msg "Adding $pkgfile to [$repo]..." + msg "Adding %s to [%s]..." "$pkgfile" "$repo" fi done - arch_repo_add "${repo}" "${tarch}" ${pkgfiles[@]} - repo_unlock $repo $tarch + arch_repo_add "${repo}" "${tarch}" "${pkgfiles[@]}" + repo_unlock "$repo" "$tarch" done diff --git a/db-repo-remove b/db-repo-remove index 7077d62..aadc4ce 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -4,34 +4,34 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi repo="$1" arch="$2" -pkgnames=(${@:3}) +pkgnames=("${@:3}") ftppath="$FTP_BASE/$repo/os" -if ! check_repo_permission $repo; then - die "You don't have permission to remove packages from ${repo}" +if ! check_repo_permission "$repo"; then + die "You don't have permission to remove packages from %s" "${repo}" fi if [ "$arch" == "any" ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do - repo_lock $repo $tarch || exit 1 +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 done -for tarch in ${tarches[@]}; do - for pkgname in ${pkgnames[@]}; do - msg "Removing $pkgname from [$repo]..." +for tarch in "${tarches[@]}"; do + for pkgname in "${pkgnames[@]}"; do + msg "Removing %s from [%s]..." "$pkgname" "$repo" done - arch_repo_remove "${repo}" "${tarch}" ${pkgnames[@]} - repo_unlock $repo $tarch + arch_repo_remove "${repo}" "${tarch}" "${pkgnames[@]}" + repo_unlock "$repo" "$tarch" done -- cgit v1.2.3 From 202a83d99581019d8bf215bfa04f6d30cf794578 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 11:03:30 -0400 Subject: Remember commit fb93f14? Use librelib instead. --- any-to-ours | 2 +- db-cleanup | 2 +- db-sync | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/any-to-ours b/any-to-ours index 020f623..e2458ca 100755 --- a/any-to-ours +++ b/any-to-ours @@ -8,7 +8,7 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-functions" +source "$(librelib messages)" # From makepkg set -E diff --git a/db-cleanup b/db-cleanup index 67fb051..cc0a5b4 100755 --- a/db-cleanup +++ b/db-cleanup @@ -16,7 +16,7 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-functions" +source "$(librelib messages)" # From makepkg set -E diff --git a/db-sync b/db-sync index 2a0e8e1..8b74244 100755 --- a/db-sync +++ b/db-sync @@ -193,7 +193,7 @@ trap_exit() { source "$(dirname "$(readlink -e "$0")")/config" source "$(dirname "$(readlink -e "$0")")/db-sync.conf" -source "$(dirname "$(readlink -e "$0")")/db-functions" +source "$(librelib messages)" # Check variables presence for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do -- cgit v1.2.3 From bcaa90a2e6f26191ff4aa9e0279080d1efeec3e7 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 11:04:32 -0400 Subject: db-functions: line repetitive things up --- db-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-functions b/db-functions index 6e27148..ebf1d47 100644 --- a/db-functions +++ b/db-functions @@ -407,9 +407,9 @@ set_repo_permission() { if [ -w "${dbfile}" ]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") - chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "${dbfile}" "$group" + chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "${dbfile}" "$group" chgrp "$group" "${filesfile}" || error "Could not change group of %s to %s" "${filesfile}" "$group" - chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "${dbfile}" + chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "${dbfile}" chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "${filesfile}" else error "You don't have permission to change %s" "${dbfile}" -- cgit v1.2.3 From e8f411803648f64b386dd2970b024b9ba15ba682 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 11:12:18 -0400 Subject: The eval+printf %q thing wasn't worth the hard-to-read code --- cron-jobs/repo-sanity-check | 2 +- db-functions | 4 ++-- db-move | 4 ++-- db-remove | 2 +- test/lib/common.inc | 6 +++--- test/test.d/create-filelists.sh | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 73ec171..fadb248 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -24,7 +24,7 @@ for _repo in "${PKGREPOS[@]}"; do done # this fills the on_abs array - printf '%q ' "${pkgname[@]}" + echo "${pkgname[@]}" done )) diff --git a/db-functions b/db-functions index ebf1d47..f703386 100644 --- a/db-functions +++ b/db-functions @@ -305,7 +305,7 @@ check_pkgxbs() { local xbsver="$(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; get_full_version "${_pkgname}")" [ "${xbsver}" == "${_pkgver}" ] || return 1 - eval "local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" + local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) in_array "${_pkgname}" "${xbsnames[@]}" || return 1 return 0 @@ -331,7 +331,7 @@ check_splitpkgs() { mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" - eval "local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" + local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) for xbsname in "${xbsnames[@]}"; do echo "${xbsname}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" done diff --git a/db-move b/db-move index f6588c3..4624d75 100755 --- a/db-move +++ b/db-move @@ -29,7 +29,7 @@ for pkgbase in "${args[@]:2}"; do for pkgarch in "${ARCHES[@]}" 'any'; do xbsrepo_from="$(xbs releasepath ${pkgbase} ${repo_from} ${pkgarch})" if [ -r "${xbsrepo_from}/PKGBUILD" ]; then - eval "pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" + pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) if [ ${#pkgnames[@]} -lt 1 ]; then die "Could not read pkgname" fi @@ -71,7 +71,7 @@ for pkgbase in "${args[@]:2}"; do else tarches=("${pkgarch}") fi - eval "pkgnames=($(. "${dir_to}/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" + pkgnames=($(. "${dir_to}/PKGBUILD"; echo "${pkgname[@]}")) for pkgname in "${pkgnames[@]}"; do pkgver=$(. "${dir_to}/PKGBUILD"; get_full_version "${pkgname}") diff --git a/db-remove b/db-remove index 970d252..c4439a6 100755 --- a/db-remove +++ b/db-remove @@ -31,7 +31,7 @@ for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" if [ -d "$path" ]; then - eval "remove_pkgs+=($(. "$path/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" + remove_pkgs+=($(. "$path/PKGBUILD"; echo "${pkgname[@]}")) xbs unrelease "$pkgbase" "$repo" "$arch" else warning "%s not found in XBS %s" "$pkgbase" "$repo-$arch" diff --git a/test/lib/common.inc b/test/lib/common.inc index a241615..1feb1bb 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -52,8 +52,8 @@ oneTimeSetUp() { msg 'Building packages...' for d in "${pkgdir}"/*; do pushd $d >/dev/null - eval "pkgname=($(. PKGBUILD; printf '%q ' "${pkgname[@]}"))" - eval "pkgarch=($(. PKGBUILD; printf '%q ' "${arch[@]}"))" + pkgname=($(. PKGBUILD; echo "${pkgname[@]}")) + pkgarch=($(. PKGBUILD; echo "${arch[@]}")) pkgversion=$(. PKGBUILD; get_full_version) build=true @@ -170,7 +170,7 @@ releasePackage() { pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null xbs release ${repo} ${arch} >/dev/null 2>&1 pkgver=$(. PKGBUILD; get_full_version) - eval "pkgname=($(. PKGBUILD; printf '%q ' "${pkgname[@]}"))" + pkgname=($(. PKGBUILD; echo "${pkgname[@]}")) popd >/dev/null cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 1126972..b5ec5c8 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -62,7 +62,7 @@ testCreateSplitFileLists() { ../db-update for pkgbase in "${pkgs[@]}"; do - eval "pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; printf '%q ' "${pkgname[@]}"))" + pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo "${pkgname[@]}")) for pkgname in "${pkgnames[@]}"; do for arch in "${arches[@]}"; do if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep "usr/bin/${pkgname}" &>/dev/null; then -- cgit v1.2.3 From 7d061a7e0faa365ae2448154c010eed26409713d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 11:58:49 -0400 Subject: more quoting an printf fixes --- db-cleanup | 2 +- db-functions | 15 ++++++------- db-move | 2 +- db-update | 36 +++++++++++++++---------------- test/lib/common.inc | 62 ++++++++++++++++++++++++++--------------------------- 5 files changed, 58 insertions(+), 59 deletions(-) diff --git a/db-cleanup b/db-cleanup index cc0a5b4..413ce06 100755 --- a/db-cleanup +++ b/db-cleanup @@ -29,7 +29,7 @@ EXTRAFLAGS=() "${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") -trap "rm $(printf %q "$filter")" EXIT +trap "rm -f -- $(printf %q "$filter")" EXIT for _repo in "${PKGREPOS[@]}"; do for _arch in "${ARCHES[@]}"; do diff --git a/db-functions b/db-functions index f703386..a1e86d9 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,7 @@ #!/bin/bash # Some PKGBUILDs need CARCH to be set -CARCH=$(. $(librelib conf.sh); load_files makepkg; echo $CARCH) +CARCH=$(. "$(librelib conf.sh)"; load_files makepkg; echo "$CARCH") # Useful functions UMASK="" @@ -251,7 +251,7 @@ getpkgfile() { getpkgfiles() { local f - if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then + if [ ! -z "$(printf '%s\n' "${@%\.*}" | sort | uniq -D)" ]; then error 'Duplicate packages found!' exit 1 fi @@ -302,7 +302,7 @@ check_pkgxbs() { in_array "${repo}" "${PKGREPOS[@]}" || return 1 - local xbsver="$(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; get_full_version "${_pkgname}")" + local xbsver="$(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; get_full_version "${_pkgname}")" [ "${xbsver}" == "${_pkgver}" ] || return 1 local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) @@ -332,8 +332,7 @@ check_splitpkgs() { echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) - for xbsname in "${xbsnames[@]}"; do - echo "${xbsname}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" + printf '%s\n' "${xbsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" done done popd >/dev/null @@ -441,13 +440,13 @@ arch_repo_remove() { local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ ! -f "${dbfile}" ]; then - error "No database found at '${dbfile}'" + error "No database found at '%s'" "${dbfile}" return 1 fi /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ - || error "repo-remove ${dbfile} ${pkgs[*]}" + || error '%s' "repo-remove ${dbfile} ${pkgs[*]}" /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ - || error "repo-remove ${filesfile} ${pkgs[*]}" + || error '%s' "repo-remove ${filesfile} ${pkgs[*]}" set_repo_permission "${repo}" "${arch}" REPO_MODIFIED=1 diff --git a/db-move b/db-move index 4624d75..2c7c19e 100755 --- a/db-move +++ b/db-move @@ -15,7 +15,7 @@ ftppath_from="${FTP_BASE}/${repo_from}/os/" ftppath_to="${FTP_BASE}/${repo_to}/os/" if ! check_repo_permission "$repo_to" || ! check_repo_permission "$repo_from"; then - die "You don't have permission to move packages from ${repo_from} to ${repo_to}" + die "You don't have permission to move packages from %s to %s" "${repo_from}" "${repo_to}" fi # TODO: this might lock too much (architectures) diff --git a/db-update b/db-update index f830e11..9a52e01 100755 --- a/db-update +++ b/db-update @@ -6,18 +6,18 @@ shopt -s nullglob if [ $# -ge 1 ]; then - warning "Calling ${0##*/} with a specific repository is no longer supported" + warning "Calling %s with a specific repository is no longer supported" "${0##*/}" exit 1 fi # Find repos with packages to release staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)) if [ $? -ge 1 ]; then - die "Could not read ${STAGING}" + die "Could not read %s" "${STAGING}" fi repos=() -for staging_repo in ${staging_repos[@]##*/}; do +for staging_repo in "${staging_repos[@]##*/}"; do if in_array "${staging_repo}" "${PKGREPOS[@]}"; then repos+=("${staging_repo}") fi @@ -26,7 +26,7 @@ done # TODO: this might lock too much (architectures) for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do - repo_lock ${repo} ${pkgarch} || exit 1 + repo_lock "${repo}" "${pkgarch}" || exit 1 done done @@ -35,42 +35,42 @@ for repo in "${repos[@]}"; do pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then - die "You don't have permission to update packages in ${repo}" + die "You don't have permission to update packages in %s" "${repo}" fi for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then - die "Package ${repo}/${pkg##*/} is a symbolic link" + die "Package %s is a symbolic link" "${repo}/${pkg##*/}" fi if ! check_pkgfile "${pkg}"; then - die "Package ${repo}/${pkg##*/} is not consistent with its meta data" + die "Package %s is not consistent with its meta data" "${repo}/${pkg##*/}" fi - if ${REQUIRE_SIGNATURE} && ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then - die "Package ${repo}/${pkg##*/} does not have a valid signature" + if "${REQUIRE_SIGNATURE}" && ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then + die "Package %s does not have a valid signature" "${repo}/${pkg##*/}" fi if ! check_pkgxbs "${pkg}" "${repo}"; then - die "Package ${repo}/${pkg##*/} is not consistent with xbs" + die "Package %s is not consistent with XBS" "${repo}/${pkg##*/}" fi if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/${pkg##*/} already exists in another repository" + die "Package %s already exists in another repository" "${repo}/${pkg##*/}" fi done - if ! check_splitpkgs ${repo} ${pkgs[@]}; then - die "Missing split packages for ${repo}" + if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then + die "Missing split packages for %s" "${repo}" fi else - die "Could not read ${STAGING}" + die "Could not read %s" "${STAGING}" fi done for repo in "${repos[@]}"; do - msg "Updating [${repo}]..." + msg "Updating [%s]..." "${repo}" any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do add_pkgs=() - arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) + arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-"${pkgarch}"${PKGEXT} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" - msg2 "${pkgfile} (${pkgarch})" + msg2 "%s (%s)" "${pkgfile}" "${pkgarch}" # any packages might have been moved by the previous run if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" @@ -93,7 +93,7 @@ done for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do - repo_unlock ${repo} ${pkgarch} + repo_unlock "${repo}" "${pkgarch}" done done diff --git a/test/lib/common.inc b/test/lib/common.inc index 1feb1bb..b9a2f9d 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -33,9 +33,9 @@ signpkg() { . ~/.makepkg.conf fi if [[ -n $GPGKEY ]]; then - SIGNWITHKEY="-u ${GPGKEY}" + SIGNWITHKEY=(-u "${GPGKEY}") fi - gpg --detach-sign --use-agent ${SIGNWITHKEY} ${@} || die + gpg --detach-sign --use-agent "${SIGNWITHKEY[@]}" "${@}" || die } oneTimeSetUp() { @@ -47,8 +47,8 @@ oneTimeSetUp() { local pkgarch local pkgversion local build - pkgdir="$(mktemp -dqt ${0##*/}.XXXXXXXXXX)" - cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" + pkgdir="$(mktemp -dt "${0##*/}.XXXXXXXXXX")" + cp -Lr "$(dirname "${BASH_SOURCE[0]}")"/../packages/* "${pkgdir}" msg 'Building packages...' for d in "${pkgdir}"/*; do pushd $d >/dev/null @@ -57,21 +57,21 @@ oneTimeSetUp() { pkgversion=$(. PKGBUILD; get_full_version) build=true - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - [ ! -f ${p}-${pkgversion}-${a}${PKGEXT} ] && build=false + for a in "${pkgarch[@]}"; do + for p in "${pkgname[@]}"; do + [ ! -f "${p}-${pkgversion}-${a}"${PKGEXT} ] && build=false done done - if ! ${build}; then + if ! "${build}"; then if [ "${pkgarch[0]}" == 'any' ]; then - sudo libremakepkg || die 'libremakepkg failed' + sudo libremakepkg || die 'libremakepkg failed' else - for a in ${pkgarch[@]}; do - if in_array $a ${arches[@]}; then - sudo setarch $a libremakepkg -n $a || die "setarch ${a} libremakepkg -n ${a} failed" - for p in ${pkgname[@]}; do - cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/${d##*/} + for a in "${pkgarch[@]}"; do + if in_array "$a" "${arches[@]}"; then + sudo setarch "$a" libremakepkg -n "$a" || die "setarch ${a} libremakepkg -n ${a} failed" + for p in "${pkgname[@]}"; do + cp "${p}-${pkgversion}-${a}"${PKGEXT} "$(dirname "${BASH_SOURCE[0]})/../packages/${d##*/}")" done else warning "skipping arch %s" "$a" @@ -93,9 +93,9 @@ setUp() { local r local a - [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" + [ -f "$(dirname "${BASH_SOURCE[0]}")/../../config.local" ] && die "$(dirname "${BASH_SOURCE[0]}")/../../config.local exists" init_tmpdir - TMP="$(mktemp -dqt ${0##*/}.XXXXXXXXXX)" + TMP="$(mktemp -dt "${0##*/}.XXXXXXXXXX")" #msg "Using ${TMP}" PKGREPOS=('core' 'extra' 'testing') @@ -103,9 +103,9 @@ setUp() { SRCPOOL='pool/sources' mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} - for r in ${PKGREPOS[@]}; do + for r in "${PKGREPOS[@]}"; do mkdir -p "${TMP}/staging/${r}" - for a in ${ARCHES[@]}; do + for a in "${ARCHES[@]}"; do mkdir -p "${TMP}/ftp/${r}/os/${a}" done done @@ -119,8 +119,8 @@ setUp() { for p in "${pkgdir}"/*; do pkg=${p##*/} mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} - cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ - arch_svn add -q "${TMP}/svn-packages-copy"/${pkg} + cp "${p}"/* "${TMP}/svn-packages-copy/${pkg}/trunk/" + arch_svn add -q "${TMP}/svn-packages-copy/${pkg}" arch_svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done @@ -130,13 +130,13 @@ setUp() { 'SVNURL=foo' \ "SVNREPO=\"${TMP}/svn-packages-copy\"" \ "ARCHES=($(arches))" \ - > $XDG_CONFIG_HOME/libretools/xbs-abs.conf - printf '%s\n' 'BUILDSYSTEM=abs' > $XDG_CONFIG_HOME/xbs.conf + > "$XDG_CONFIG_HOME/libretools/xbs-abs.conf" + printf '%s\n' 'BUILDSYSTEM=abs' > "$XDG_CONFIG_HOME/xbs.conf" - cat < "$(dirname ${BASH_SOURCE[0]})/../../config.local" + cat < "$(dirname "${BASH_SOURCE[0]}")/../../config.local" FTP_BASE="${TMP}/ftp" SVNREPO="${TMP}/svn-packages-copy" - PKGREPOS=(${PKGREPOS[@]}) + PKGREPOS=("${PKGREPOS[@]}") PKGPOOL="${PKGPOOL}" SRCPOOL="${SRCPOOL}" TESTING_REPO='testing' @@ -149,12 +149,12 @@ setUp() { SOURCE_CLEANUP_DRYRUN=false REQUIRE_SIGNATURE=true eot - . "$(dirname ${BASH_SOURCE[0]})/../../config" + . "$(dirname "${BASH_SOURCE[0]}")/../../config" } tearDown() { rm -rf "${TMP}" - rm -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" + rm -f "$(dirname "${BASH_SOURCE[0]}")/../../config.local" echo } @@ -167,17 +167,17 @@ releasePackage() { local pkgver local pkgname - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null - xbs release ${repo} ${arch} >/dev/null 2>&1 + pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" >/dev/null + xbs release "${repo}" "${arch}" >/dev/null 2>&1 pkgver=$(. PKGBUILD; get_full_version) pkgname=($(. PKGBUILD; echo "${pkgname[@]}")) popd >/dev/null - cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ + cp "${pkgdir}/${pkgbase}"/*-"${pkgver}-${arch}"${PKGEXT} "${STAGING}/${repo}/" if "${REQUIRE_SIGNATURE}"; then for a in "${arch[@]}"; do for p in "${pkgname[@]}"; do - signpkg "${STAGING}"/${repo}/${p}-${pkgver}-${a}${PKGEXT} + signpkg "${STAGING}/${repo}/${p}-${pkgver}-${a}"${PKGEXT} done done fi @@ -190,7 +190,7 @@ checkAnyPackageDB() { local db [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - if ${REQUIRE_SIGNATURE}; then + if "${REQUIRE_SIGNATURE}"; then [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" fi -- cgit v1.2.3 From 457e2b0ab0870e19bcf9bdcdd88a2c338cc3bfd3 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 12:27:35 -0400 Subject: use tab indent --- any-to-ours | 78 ++++++++++++++-------------- create-repo | 18 +++---- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 10 ++-- cron-jobs/devlist-mailer | 6 +-- db-list-unsigned-packages | 4 +- mkrepo | 8 +-- 6 files changed, 62 insertions(+), 62 deletions(-) diff --git a/any-to-ours b/any-to-ours index e2458ca..4ba01dc 100755 --- a/any-to-ours +++ b/any-to-ours @@ -2,9 +2,9 @@ # Releases 'any' packages from Arch arches to ours trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } source "$(dirname "$(readlink -e "$0")")/config" @@ -22,51 +22,51 @@ BASEARCH='x86_64' # Traverse all Arch repos for _repo in ${ARCHREPOS[@]}; do - msg "Processing ${_repo}..." + msg "Processing ${_repo}..." -# Find 'any' packages -# This is hardcoded but it could release other arches... - PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - -iname '*-any.pkg.tar.?z' \ - -printf "%f ")) + # Find 'any' packages + # This is hardcoded but it could release other arches... + PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + -iname '*-any.pkg.tar.?z' \ + -printf "%f ")) - if [ ${#PKGS[@]} -eq 0 ]; then - msg2 "No 'any' packages here" - continue - fi + if [ ${#PKGS[@]} -eq 0 ]; then + msg2 "No 'any' packages here" + continue + fi - for _arch in ${OURARCHES[@]}; do - msg2 "Syncing ${_arch}..." + for _arch in ${OURARCHES[@]}; do + msg2 "Syncing ${_arch}..." -# Sync 'any' only and extract the synced packages - SYNCED=($( - rsync -av \ - --include='*-any.pkg.tar.?z' \ - --include='*-any.pkg.tar.?z.sig' \ - --exclude='*' \ - ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ - grep 'any\.pkg\.tar\..z$' | \ - cut -d ' ' -f 1 )) + # Sync 'any' only and extract the synced packages + SYNCED=($( + rsync -av \ + --include='*-any.pkg.tar.?z' \ + --include='*-any.pkg.tar.?z.sig' \ + --exclude='*' \ + ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ + grep 'any\.pkg\.tar\..z$' | \ + cut -d ' ' -f 1 )) - if [ ${#SYNCED[@]} -eq 0 ]; then - msg2 "Already synced (or error happened)" - continue - fi + if [ ${#SYNCED[@]} -eq 0 ]; then + msg2 "Already synced (or error happened)" + continue + fi - msg2 "Synced ${#SYNCED[@]} packages: ${SYNCED[@]}" + msg2 "Synced ${#SYNCED[@]} packages: ${SYNCED[@]}" - msg2 "Adding to db..." + msg2 "Adding to db..." - pushd ${FTP_BASE}/${_repo}/os/${_arch}/ >/dev/null + pushd ${FTP_BASE}/${_repo}/os/${_arch}/ >/dev/null -# Add the packages to the db - repo-add ${_repo}${DBEXT} \ - ${SYNCED[@]} + # Add the packages to the db + repo-add ${_repo}${DBEXT} \ + ${SYNCED[@]} - popd >/dev/null + popd >/dev/null -# Avoid mixups - unset SYNCED PKGS - done + # Avoid mixups + unset SYNCED PKGS + done done diff --git a/create-repo b/create-repo index 1ec9798..cfdbc23 100755 --- a/create-repo +++ b/create-repo @@ -5,20 +5,20 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -eq 0 ]; then - msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" - exit 1 + msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" + exit 1 fi msg "Creating repos..." for _repo in "$@"; do - msg2 "Creating [${_repo}]" - mkdir -p "${FTP_BASE}/staging/${_repo}" || \ - error "Failed creating staging dir" + msg2 "Creating [${_repo}]" + mkdir -p "${FTP_BASE}/staging/${_repo}" || \ + error "Failed creating staging dir" - for _arch in "${ARCHES[@]}"; do - mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ - error "Failed creating ${_arch} dir" - done + for _arch in "${ARCHES[@]}"; do + mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ + error "Failed creating ${_arch} dir" + done done msg "Don't forget to add them to the PKGREPOS array on %s/config" "$(dirname "$(readlink -e "$0")")" diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 38af179..b857ac8 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -118,11 +118,11 @@ source_pkgbuild() { } find_pkgbuilds() { - #Skip over some dirs - local d="${1##*/}" - if [ "$d" = "CVS" -o "$d" = ".svn" ]; then - return - fi + #Skip over some dirs + local d="${1##*/}" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi if [ -f "$1/PKGBUILD" ]; then source_pkgbuild "$1" diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 61cd32c..1a05521 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -8,18 +8,18 @@ FROM="repomaint@archlinux.org" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if [ $# -ge 1 ]; then - SUBJECT="$1 $(date +"%d-%m-%Y")" + SUBJECT="$1 $(date +"%d-%m-%Y")" fi if [ $# -ge 2 ]; then - LIST="$2" + LIST="$2" fi stdin="$(cat)" #echo used to strip whitespace for checking for actual data if [ -n "$(echo $stdin)" ]; then -echo "Subject: $SUBJECT + echo "Subject: $SUBJECT To: $LIST From: $FROM diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index f593686..c88203b 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -33,6 +33,6 @@ shift for repo in "${PKGREPOS[@]}" do - db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" + db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" done diff --git a/mkrepo b/mkrepo index f30ad00..b11dc0b 100755 --- a/mkrepo +++ b/mkrepo @@ -6,10 +6,10 @@ source "$(dirname "$(readlink -e "$0")")/config" for repo in "$@"; do - echo ":: Creating [$repo]" - for arch in "${ARCHES[@]}"; do - mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" - done + echo ":: Creating [$repo]" + for arch in "${ARCHES[@]}"; do + mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" + done done echo ":: All done. Add the repo to the ParabolaWeb admin page." -- cgit v1.2.3 From df49c9b89e56b6c8d2d20cdcc9e03dc80996fcdd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 12:36:33 -0400 Subject: use tab indent --- any-to-ours | 78 ++++++++++++------------ create-repo | 18 +++--- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 10 ++-- cron-jobs/devlist-mailer | 6 +- cron-jobs/repo-sanity-check | 88 ++++++++++++++-------------- cron-jobs/sourceballs2 | 48 +++++++-------- db-check-nonfree | 36 ++++++------ db-cleanup | 44 +++++++------- db-list-unsigned-packages | 4 +- libremessages | 4 +- mkrepo | 8 +-- 11 files changed, 172 insertions(+), 172 deletions(-) diff --git a/any-to-ours b/any-to-ours index a1697c7..6afc7b0 100755 --- a/any-to-ours +++ b/any-to-ours @@ -2,9 +2,9 @@ # Releases 'any' packages from Arch arches to ours trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } source "$(dirname "$(readlink -e "$0")")/config" @@ -22,51 +22,51 @@ BASEARCH='x86_64' # Traverse all Arch repos for _repo in ${ARCHREPOS[@]}; do - msg "Processing ${_repo}..." + msg "Processing ${_repo}..." -# Find 'any' packages -# This is hardcoded but it could release other arches... - PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - -iname '*-any.pkg.tar.?z' \ - -printf "%f ")) + # Find 'any' packages + # This is hardcoded but it could release other arches... + PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + -iname '*-any.pkg.tar.?z' \ + -printf "%f ")) - if [ ${#PKGS[@]} -eq 0 ]; then - msg2 "No 'any' packages here" - continue - fi + if [ ${#PKGS[@]} -eq 0 ]; then + msg2 "No 'any' packages here" + continue + fi - for _arch in ${OURARCHES[@]}; do - msg2 "Syncing ${_arch}..." + for _arch in ${OURARCHES[@]}; do + msg2 "Syncing ${_arch}..." -# Sync 'any' only and extract the synced packages - SYNCED=($( - rsync -av \ - --include='*-any.pkg.tar.?z' \ - --include='*-any.pkg.tar.?z.sig' \ - --exclude='*' \ - ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ - grep 'any\.pkg\.tar\..z$' | \ - cut -d ' ' -f 1 )) + # Sync 'any' only and extract the synced packages + SYNCED=($( + rsync -av \ + --include='*-any.pkg.tar.?z' \ + --include='*-any.pkg.tar.?z.sig' \ + --exclude='*' \ + ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ + grep 'any\.pkg\.tar\..z$' | \ + cut -d ' ' -f 1 )) - if [ ${#SYNCED[@]} -eq 0 ]; then - msg2 "Already synced (or error happened)" - continue - fi + if [ ${#SYNCED[@]} -eq 0 ]; then + msg2 "Already synced (or error happened)" + continue + fi - msg2 "Synced ${#SYNCED[@]} packages: ${SYNCED[@]}" + msg2 "Synced ${#SYNCED[@]} packages: ${SYNCED[@]}" - msg2 "Adding to db..." + msg2 "Adding to db..." - pushd ${FTP_BASE}/${_repo}/os/${_arch}/ >/dev/null + pushd ${FTP_BASE}/${_repo}/os/${_arch}/ >/dev/null -# Add the packages to the db - repo-add ${_repo}${DBEXT} \ - ${SYNCED[@]} + # Add the packages to the db + repo-add ${_repo}${DBEXT} \ + ${SYNCED[@]} - popd >/dev/null + popd >/dev/null -# Avoid mixups - unset SYNCED PKGS - done + # Avoid mixups + unset SYNCED PKGS + done done diff --git a/create-repo b/create-repo index 1ec9798..cfdbc23 100755 --- a/create-repo +++ b/create-repo @@ -5,20 +5,20 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -eq 0 ]; then - msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" - exit 1 + msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" + exit 1 fi msg "Creating repos..." for _repo in "$@"; do - msg2 "Creating [${_repo}]" - mkdir -p "${FTP_BASE}/staging/${_repo}" || \ - error "Failed creating staging dir" + msg2 "Creating [${_repo}]" + mkdir -p "${FTP_BASE}/staging/${_repo}" || \ + error "Failed creating staging dir" - for _arch in "${ARCHES[@]}"; do - mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ - error "Failed creating ${_arch} dir" - done + for _arch in "${ARCHES[@]}"; do + mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ + error "Failed creating ${_arch} dir" + done done msg "Don't forget to add them to the PKGREPOS array on %s/config" "$(dirname "$(readlink -e "$0")")" diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index c8d8618..9b26f1e 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -118,11 +118,11 @@ source_pkgbuild() { } find_pkgbuilds() { - #Skip over some dirs - local d="${1##*/}" - if [ "$d" = "CVS" -o "$d" = ".svn" ]; then - return - fi + #Skip over some dirs + local d="${1##*/}" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi if [ -f $1/PKGBUILD ]; then source_pkgbuild $1 diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index ca2e46b..32896f7 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -8,18 +8,18 @@ FROM="repomaint@archlinux.org" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if [ $# -ge 1 ]; then - SUBJECT="$1 $(date +"%d-%m-%Y")" + SUBJECT="$1 $(date +"%d-%m-%Y")" fi if [ $# -ge 2 ]; then - LIST="$2" + LIST="$2" fi stdin="$(cat)" #echo used to strip whitespace for checking for actual data if [ -n "$(echo $stdin)" ]; then -echo "Subject: $SUBJECT + echo "Subject: $SUBJECT To: $LIST From: $FROM diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 9d351df..f5f80c8 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -6,50 +6,50 @@ # Traverse all repos for _repo in "${PKGREPOS[@]}"; do - msg "Cleaning up [${_repo}]" - -# Find all pkgnames on this repo's abs - on_abs=($( - find ${SVNREPO}/${_repo} -name PKGBUILD | \ - while read pkgbuild; do - source ${pkgbuild} >/dev/null 2>&1 -# cleanup to save memory - unset build package source md5sums pkgdesc pkgver pkgrel epoch \ - url license arch depends makedepends optdepends options \ - >/dev/null 2>&1 - -# also cleanup package functions - for _pkg in "${pkgname[@]}"; do - unset package_${pkg} >/dev/null 2>&1 - done - -# this fills the on_abs array - echo ${pkgname[@]} - done - )) - -# quit if abs is empty - if [ ${#on_abs[*]} -eq 0 ]; then - warning "[${_repo}]'s ABS tree is empty, skipping" - break - fi - -# Find all pkgnames on repos - on_repo=($( - find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ - sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" - )) - -# Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ - <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) - -# Remove them from databases, ftpdir-cleanup will take care of the rest - find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ - repo-remove {} ${remove[@]} >/dev/null 2>&1 \; - - msg2 "Removed the following packages:" - plain "$(echo ${remove[@]} | tr ' ' "\n")" + msg "Cleaning up [${_repo}]" + + # Find all pkgnames on this repo's abs + on_abs=($( + find ${SVNREPO}/${_repo} -name PKGBUILD | \ + while read pkgbuild; do + source ${pkgbuild} >/dev/null 2>&1 + # cleanup to save memory + unset build package source md5sums pkgdesc pkgver pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + + # also cleanup package functions + for _pkg in "${pkgname[@]}"; do + unset package_${pkg} >/dev/null 2>&1 + done + + # this fills the on_abs array + echo ${pkgname[@]} + done + )) + + # quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[${_repo}]'s ABS tree is empty, skipping" + break + fi + + # Find all pkgnames on repos + on_repo=($( + find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ + sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + )) + + # Compares them, whatever is on repos but not on abs should be removed + remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ + <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) + + # Remove them from databases, ftpdir-cleanup will take care of the rest + find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ + repo-remove {} ${remove[@]} >/dev/null 2>&1 \; + + msg2 "Removed the following packages:" + plain "$(echo ${remove[@]} | tr ' ' "\n")" done diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 2a26e6a..49a2dac 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -22,41 +22,41 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null for repo in "${PKGREPOS[@]}"; do - msg "Sourceballing [${repo}]" + msg "Sourceballing [${repo}]" - pushd $repo >/dev/null - find -maxdepth 1 -type d | while read pkg; do - pushd "${SVNREPO}/$repo/$pkg" >/dev/null + pushd $repo >/dev/null + find -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null - [[ ! -e PKGBUILD ]] && { - warning "$repo/$pkg is not a package" - continue - } + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } -# Unset the previous data - unset pkgbase pkgname pkgver pkgrel - source PKGBUILD + # Unset the previous data + unset pkgbase pkgname pkgver pkgrel + source PKGBUILD - unset build package url pkgdesc source md5sums depends makedepends \ - optdepends license arch options check mksource + unset build package url pkgdesc source md5sums depends makedepends \ + optdepends license arch options check mksource - for _pkg in "${pkgname[@]}"; do - unset package_${_pkg} >/dev/null 2>&1 - done + for _pkg in "${pkgname[@]}"; do + unset package_${_pkg} >/dev/null 2>&1 + done - pkgbase=${pkgbase:-$pkgname} - srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + pkgbase=${pkgbase:-$pkgname} + srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs" - # Skip already sourceballed - [ -e "${SRCPKGDEST}/${srcfile}" ] && continue + # Skip already sourceballed + [ -e "${SRCPKGDEST}/${srcfile}" ] && continue - makepkg --allsource --ignorearch -c >/dev/null 2>&1 + makepkg --allsource --ignorearch -c >/dev/null 2>&1 - [ $? -ne 0 ] && plain ${srcfile} + [ $? -ne 0 ] && plain ${srcfile} - done # end find pkgs - popd >/dev/null + done # end find pkgs + popd >/dev/null done # end repos diff --git a/db-check-nonfree b/db-check-nonfree index 6e2dc17..b08b7b1 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -18,25 +18,25 @@ done msg "Check nonfree in repo:" nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) for repo in "${ARCHREPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - msg2 "$repo $pkgarch" - if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then - continue - fi - unset dbpkgs - unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in "${dbpkgs[@]}"; do - if in_array "${pkgname}" "${nonfree[@]}"; then - cleanpkgs+=("${pkgname}") - fi + for pkgarch in "${ARCHES[@]}"; do + msg2 "$repo $pkgarch" + if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then + continue + fi + unset dbpkgs + unset cleanpkgs + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in "${dbpkgs[@]}"; do + if in_array "${pkgname}" "${nonfree[@]}"; then + cleanpkgs+=("${pkgname}") + fi + done + if [ ${#cleanpkgs[@]} -ge 1 ]; then + msg2 "Nonfree: ${cleanpkgs[*]}" + arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" + fi done - if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: ${cleanpkgs[*]}" - arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" - fi - done done for repo in "${repos[@]}"; do diff --git a/db-cleanup b/db-cleanup index 57ef36e..a35bdf2 100755 --- a/db-cleanup +++ b/db-cleanup @@ -10,9 +10,9 @@ # * Instant cleanup! trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } source "$(dirname "$(readlink -e "$0")")/config" @@ -28,35 +28,35 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR ${CLEANUP_DRYRUN} && EXTRAFLAGS+=" --dry-run" for _repo in ${PKGREPOS[@]}; do - for _arch in ${ARCHES[@]}; do - msg "Getting ${_repo}-${_arch} database" + for _arch in ${ARCHES[@]}; do + msg "Getting ${_repo}-${_arch} database" - dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" + dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" - if [ ! -r "${dbfile}" ]; then - warning "Not found" - continue - fi + if [ ! -r "${dbfile}" ]; then + warning "Not found" + continue + fi -# Echo the contents into a filter file - bsdtar tf "${dbfile}" | \ - cut -d'/' -f1 | \ - sort -u | \ - sed "s|$|*|" >> /tmp/${0##*/}.$$.filter + # Echo the contents into a filter file + bsdtar tf "${dbfile}" | \ + cut -d'/' -f1 | \ + sort -u | \ + sed "s|$|*|" >> /tmp/${0##*/}.$$.filter - done + done done msg "Removing old files:" for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do - msg2 "${POOL}" + msg2 "${POOL}" - rsync ${EXTRAFLAGS} -va --delete-excluded \ - --include-from="/tmp/${0##*/}.$$.filter" \ - --exclude="*" \ - ${FTP_BASE}/${POOL}/ \ - ${FTP_BASE}/${POOL}/ + rsync ${EXTRAFLAGS} -va --delete-excluded \ + --include-from="/tmp/${0##*/}.$$.filter" \ + --exclude="*" \ + ${FTP_BASE}/${POOL}/ \ + ${FTP_BASE}/${POOL}/ done msg "Removing symlinks:" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index f593686..c88203b 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -33,6 +33,6 @@ shift for repo in "${PKGREPOS[@]}" do - db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" + db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" done diff --git a/libremessages b/libremessages index 0c6ded1..37df149 100755 --- a/libremessages +++ b/libremessages @@ -77,7 +77,7 @@ error() { fatal_error() { local mesg=$1; shift - error "$mesg" "$@" + error "$mesg" "$@" - exit 1 + exit 1 } diff --git a/mkrepo b/mkrepo index f30ad00..b11dc0b 100755 --- a/mkrepo +++ b/mkrepo @@ -6,10 +6,10 @@ source "$(dirname "$(readlink -e "$0")")/config" for repo in "$@"; do - echo ":: Creating [$repo]" - for arch in "${ARCHES[@]}"; do - mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" - done + echo ":: Creating [$repo]" + for arch in "${ARCHES[@]}"; do + mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" + done done echo ":: All done. Add the repo to the ParabolaWeb admin page." -- cgit v1.2.3 From 46510e1fc48f37ce76c2bf5f19f885bba8d5d098 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 13:05:12 -0400 Subject: Clean up quoting. --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 36 +++++++------- cron-jobs/devlist-mailer | 2 +- cron-jobs/ftpdir-cleanup | 19 ++++--- cron-jobs/integrity-check | 4 +- cron-jobs/repo-sanity-check | 23 ++++----- cron-jobs/sourceballs | 18 +++---- cron-jobs/sourceballs2 | 8 +-- cron-jobs/update-abs-tarballs | 4 +- db-check-nonfree | 6 +-- db-cleanup | 21 ++++---- db-functions | 74 ++++++++++++++-------------- db-move | 32 ++++++------ db-remove | 8 +-- db-repo-add | 18 +++---- db-repo-remove | 18 +++---- db-sync | 56 ++++++++++----------- db-update | 6 +-- testing2x | 22 ++++----- 18 files changed, 187 insertions(+), 188 deletions(-) diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 9b26f1e..b857ac8 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -12,7 +12,7 @@ readonly -a variables splitpkg_overrides backup_package_variables() { for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" - eval "${indirect}=(\${$var[@]})" + eval "${indirect}=(\"\${$var[@]}\")" done } @@ -20,9 +20,9 @@ restore_package_variables() { for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then - eval "${var}=(\${$indirect[@]})" + eval "${var}=(\"\${$indirect[@]}\")" else - unset ${var} + unset "${var}" fi done } @@ -42,31 +42,31 @@ print_info() { if [ -n "$arch" ]; then echo "%ARCH%" - for i in "${arch[@]}"; do echo $i; done + for i in "${arch[@]}"; do echo "$i"; done echo "" fi if [ -n "$depends" ]; then echo "%DEPENDS%" for i in "${depends[@]}"; do - echo $i + echo "$i" done echo "" fi if [ -n "$makedepends" ]; then echo "%MAKEDEPENDS%" for i in "${makedepends[@]}"; do - echo $i + echo "$i" done echo "" fi if [ -n "$conflicts" ]; then echo "%CONFLICTS%" - for i in "${conflicts[@]}"; do echo $i; done + for i in "${conflicts[@]}"; do echo "$i"; done echo "" fi if [ -n "$provides" ]; then echo "%PROVIDES%" - for i in "${provides[@]}"; do echo $i; done + for i in "${provides[@]}"; do echo "$i"; done echo "" fi } @@ -76,9 +76,9 @@ source_pkgbuild() { dir=$1 pkgbuild=$dir/PKGBUILD for var in "${variables[@]}"; do - unset ${var} + unset "${var}" done - source $pkgbuild &>/dev/null || ret=$? + source "$pkgbuild" &>/dev/null || ret=$? # ensure $pkgname and $pkgver variables were found if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then @@ -89,7 +89,7 @@ source_pkgbuild() { if [ "${#pkgname[@]}" -gt "1" ]; then pkgbase=${pkgbase:-${pkgname[0]}} for pkg in "${pkgname[@]}"; do - if [ "$(type -t package_${pkg})" != "function" ]; then + if [ "$(type -t "package_${pkg}")" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" return 1 else @@ -104,7 +104,7 @@ source_pkgbuild() { break fi done - done < <(type package_${pkg}) + done < <(type "package_${pkg}") print_info restore_package_variables fi @@ -124,14 +124,14 @@ find_pkgbuilds() { return fi - if [ -f $1/PKGBUILD ]; then - source_pkgbuild $1 + if [ -f "$1/PKGBUILD" ]; then + source_pkgbuild "$1" return fi empty=1 - for dir in $1/*; do - if [ -d $dir ]; then - find_pkgbuilds $dir + for dir in "$1"/*; do + if [ -d "$dir" ]; then + find_pkgbuilds "$dir" unset empty fi done @@ -147,7 +147,7 @@ fi CARCH=$1 shift for dir in "$@"; do - find_pkgbuilds $dir + find_pkgbuilds "$dir" done exit 0 diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 32896f7..1a05521 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -23,6 +23,6 @@ if [ -n "$(echo $stdin)" ]; then To: $LIST From: $FROM -$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" +$stdin" | /usr/sbin/sendmail -F"$FROM" "$LIST" fi diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index f2d8b33..d5a277b 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -7,7 +7,7 @@ clean_pkg() { local pkg local target - if ! ${CLEANUP_DRYRUN}; then + if ! "${CLEANUP_DRYRUN}"; then for pkg in "$@"; do if [ -h "$pkg" ]; then rm -f "$pkg" "$pkg.sig" @@ -22,8 +22,7 @@ clean_pkg() { fi } - -${CLEANUP_DRYRUN} && warning 'dry run mode is active' +"${CLEANUP_DRYRUN}" && warning 'dry run mode is active' for repo in "${PKGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do @@ -38,7 +37,7 @@ for repo in "${PKGREPOS[@]}"; do missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#missing_pkgs[@]} -ge 1 ]; then error "Missing packages in [${repo}] (${arch})..." - for missing_pkg in ${missing_pkgs[@]}; do + for missing_pkg in "${missing_pkgs[@]}"; do msg2 "${missing_pkg}" done fi @@ -46,7 +45,7 @@ for repo in "${PKGREPOS[@]}"; do old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from [${repo}] (${arch})..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" done @@ -62,18 +61,18 @@ cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from package pool..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" done fi -old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find "${CLEANUP_DESTDIR}" -type f -name "*${PKGEXT}" -mtime +"${CLEANUP_KEEP}" -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" - if ! ${CLEANUP_DRYRUN}; then + if ! "${CLEANUP_DRYRUN}"; then rm -f "${CLEANUP_DESTDIR}/${old_pkg}" rm -f "${CLEANUP_DESTDIR}/${old_pkg}.sig" fi @@ -82,7 +81,7 @@ fi for repo in "${PKGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do - repo_unlock ${repo} ${arch} + repo_unlock "${repo}" "${arch}" done done diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 86a8f1d..33a4eb6 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -13,12 +13,12 @@ fi mailto=$1 check() { - ${dirname}/check_archlinux/check_packages.py \ + "${dirname}"/check_archlinux/check_packages.py \ --repos="${repos}" \ --abs-tree="/srv/abs/rsync/${arch},/srv/abs/rsync/any" \ --repo-dir="${FTP_BASE}" \ --arch="${arch}" \ - 2>&1 | ${dirname}/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" + 2>&1 | "${dirname}"/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" } repos='core,extra,community' diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index f5f80c8..105bac4 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -10,9 +10,9 @@ for _repo in "${PKGREPOS[@]}"; do # Find all pkgnames on this repo's abs on_abs=($( - find ${SVNREPO}/${_repo} -name PKGBUILD | \ + find "${SVNREPO}/${_repo}" -name PKGBUILD | \ while read pkgbuild; do - source ${pkgbuild} >/dev/null 2>&1 + source "${pkgbuild}" >/dev/null 2>&1 # cleanup to save memory unset build package source md5sums pkgdesc pkgver pkgrel epoch \ url license arch depends makedepends optdepends options \ @@ -20,11 +20,11 @@ for _repo in "${PKGREPOS[@]}"; do # also cleanup package functions for _pkg in "${pkgname[@]}"; do - unset package_${pkg} >/dev/null 2>&1 + unset "package_${pkg}" >/dev/null 2>&1 done # this fills the on_abs array - echo ${pkgname[@]} + echo "${pkgname[@]}" done )) @@ -36,20 +36,21 @@ for _repo in "${PKGREPOS[@]}"; do # Find all pkgnames on repos on_repo=($( - find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ - sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + find "${FTP_BASE}/${_repo}" -name "*.pkg.tar.?z" \ + -printf "%f\n" | sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" )) # Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ - <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) + remove=($(comm -13 \ + <(printf '%s\n' "${on_abs[@]}" | sort -u) \ + <(printf '%s\n' "${on_repo[@]}" | sort -u) )) # Remove them from databases, ftpdir-cleanup will take care of the rest - find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ - repo-remove {} ${remove[@]} >/dev/null 2>&1 \; + find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" -exec \ + repo-remove {} "${remove[@]}" >/dev/null 2>&1 \; msg2 "Removed the following packages:" - plain "$(echo ${remove[@]} | tr ' ' "\n")" + plain '%s' "${remove[@]}" done diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 8171980..9c07c22 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -9,7 +9,7 @@ script_lock for repo in "${PKGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do - repo_lock ${repo} ${arch} || exit 1 + repo_lock "${repo}" "${arch}" || exit 1 done done @@ -41,7 +41,7 @@ done for repo in "${PKGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do - repo_unlock ${repo} ${arch} + repo_unlock "${repo}" "${arch}" done done @@ -74,7 +74,7 @@ for repo in "${PKGREPOS[@]}"; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then # Check if we had failed before - if in_array "${pkgbase}-${pkgver}${SRCEXT}" ${failedpkgs[@]}; then + if in_array "${pkgbase}-${pkgver}${SRCEXT}" "${failedpkgs[@]}"; then continue fi @@ -99,9 +99,9 @@ for repo in "${PKGREPOS[@]}"; do mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" - newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + newpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") else - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") fi popd >/dev/null fi @@ -128,22 +128,22 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." - ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + "${SOURCE_CLEANUP_DRYRUN}" && warning 'dry run mode is active' for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" - if ! ${SOURCE_CLEANUP_DRYRUN}; then + if ! "${SOURCE_CLEANUP_DRYRUN}"; then mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done fi -old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find "${SOURCE_CLEANUP_DESTDIR}" -type f -name "*${SRCEXT}" -mtime +"${SOURCE_CLEANUP_KEEP}" -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" - ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + "${SOURCE_CLEANUP_DRYRUN}" || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done fi diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 49a2dac..f7b3779 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -24,11 +24,11 @@ pushd "${SVNREPO}" >/dev/null for repo in "${PKGREPOS[@]}"; do msg "Sourceballing [${repo}]" - pushd $repo >/dev/null + pushd "$repo" >/dev/null find -maxdepth 1 -type d | while read pkg; do pushd "${SVNREPO}/$repo/$pkg" >/dev/null - [[ ! -e PKGBUILD ]] && { + [[ ! -e ./PKGBUILD ]] && { warning "$repo/$pkg is not a package" continue } @@ -41,7 +41,7 @@ for repo in "${PKGREPOS[@]}"; do optdepends license arch options check mksource for _pkg in "${pkgname[@]}"; do - unset package_${_pkg} >/dev/null 2>&1 + unset "package_${_pkg}" >/dev/null 2>&1 done pkgbase=${pkgbase:-$pkgname} @@ -54,7 +54,7 @@ for repo in "${PKGREPOS[@]}"; do makepkg --allsource --ignorearch -c >/dev/null 2>&1 - [ $? -ne 0 ] && plain ${srcfile} + [ $? -ne 0 ] && plain '%s' "${srcfile}" done # end find pkgs popd >/dev/null diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs index 901cc4b..e710f7c 100755 --- a/cron-jobs/update-abs-tarballs +++ b/cron-jobs/update-abs-tarballs @@ -2,6 +2,4 @@ . "$(dirname "$(readlink -e "$0")")/../config" -rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ - -exit $? +rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ "${FTP_BASE}/" diff --git a/db-check-nonfree b/db-check-nonfree index b08b7b1..cae4a14 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -11,12 +11,12 @@ fi # TODO: this might lock too much (architectures) for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do - repo_lock ${repo} ${pkgarch} || exit 1 + repo_lock "${repo}" "${pkgarch}" || exit 1 done done msg "Check nonfree in repo:" -nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) +nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u)) for repo in "${ARCHREPOS[@]}"; do for pkgarch in "${ARCHES[@]}"; do msg2 "$repo $pkgarch" @@ -41,6 +41,6 @@ done for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do - repo_unlock ${repo} ${pkgarch} + repo_unlock "${repo}" "${pkgarch}" done done diff --git a/db-cleanup b/db-cleanup index a35bdf2..b2f2e08 100755 --- a/db-cleanup +++ b/db-cleanup @@ -25,10 +25,11 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -${CLEANUP_DRYRUN} && EXTRAFLAGS+=" --dry-run" +EXTRAFLAGS=() +"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) -for _repo in ${PKGREPOS[@]}; do - for _arch in ${ARCHES[@]}; do +for _repo in "${PKGREPOS[@]}"; do + for _arch in "${ARCHES[@]}"; do msg "Getting ${_repo}-${_arch} database" dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" @@ -42,25 +43,25 @@ for _repo in ${PKGREPOS[@]}; do bsdtar tf "${dbfile}" | \ cut -d'/' -f1 | \ sort -u | \ - sed "s|$|*|" >> /tmp/${0##*/}.$$.filter + sed "s|$|*|" >> "/tmp/${0##*/}.$$.filter" done done msg "Removing old files:" -for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do +for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do msg2 "${POOL}" - rsync ${EXTRAFLAGS} -va --delete-excluded \ + rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ --include-from="/tmp/${0##*/}.$$.filter" \ --exclude="*" \ - ${FTP_BASE}/${POOL}/ \ - ${FTP_BASE}/${POOL}/ + "${FTP_BASE}/${POOL}/" \ + "${FTP_BASE}/${POOL}/" done msg "Removing symlinks:" -find -L ${FTP_BASE}/ -type l -${CLEANUP_DRYRUN} || find -L ${FTP_BASE}/ -type l -delete +find -L "${FTP_BASE}/" -type l +"${CLEANUP_DRYRUN}" || find -L "${FTP_BASE}/" -type l -delete exit $? diff --git a/db-functions b/db-functions index 6b49e6a..1384080 100644 --- a/db-functions +++ b/db-functions @@ -12,7 +12,7 @@ set_umask () { } restore_umask () { - umask $UMASK >/dev/null + umask "$UMASK" >/dev/null } # just like mv -f, but we touch the file and then copy the content so @@ -75,7 +75,7 @@ in_array() { [[ -z $1 ]] && return 1 # Not Found local item for item in "$@"; do - [[ $item = $needle ]] && return 0 # Found + [[ $item = "$needle" ]] && return 0 # Found done return 1 # Not Found } @@ -87,16 +87,16 @@ in_array() { get_full_version() { if [[ $1 -eq 0 ]]; then # zero epoch case, don't include it in version - echo $2-$3 + echo "$2-$3" else - echo $1:$2-$3 + echo "$1:$2-$3" fi } script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" error "Script ${0##*/} is already locked by $_owner." exit 1 else @@ -129,7 +129,7 @@ cleanup() { arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then msg "Removing left over lock from [${repo}] (${arch})" - repo_unlock $repo $arch + repo_unlock "$repo" "$arch" fi done if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then @@ -137,7 +137,7 @@ cleanup() { script_unlock fi rm -rf "$WORKDIR" - [ "$1" ] && exit $1 + [ "$1" ] && exit "$1" } abort() { @@ -167,11 +167,11 @@ repo_lock () { # This is the lock file used by repo-add and repo-remove if [ -f "${DBLOCKFILE}" ]; then - error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $DBLOCKFILE)" + error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat "$DBLOCKFILE")" return 1 fi if [ -f "${FILESLOCKFILE}" ]; then - error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat ${FILESLOCKFILE})" + error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat "$FILESLOCKFILE")" return 1 fi @@ -185,17 +185,17 @@ repo_lock () { fi _count=0 - while [ $_count -le $_trial ] || $_lockblock ; do + while [ "$_count" -le "$_trial" ] || "$_lockblock" ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else - LOCKS[${#LOCKS[*]}]="$1.$2" + LOCKS+=("$1.$2") set_umask return 0 fi - sleep $LOCK_DELAY + sleep "$LOCK_DELAY" let _count=$_count+1 done @@ -293,7 +293,7 @@ getpkgfile() { elif [ ! -f "${1}" ]; then error "Package ${1} not found!" exit 1 - elif ${REQUIRE_SIGNATURE} && [ ! -f "${1}.sig" ]; then + elif "${REQUIRE_SIGNATURE}" && [ ! -f "${1}.sig" ]; then error "Package signature ${1}.sig not found!" exit 1 fi @@ -303,7 +303,7 @@ getpkgfile() { getpkgfiles() { local f - if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then + if [ ! -z "$(printf '%s\n' "${@%\.*}" | sort | uniq -D)" ]; then error 'Duplicate packages found!' exit 1 fi @@ -312,7 +312,7 @@ getpkgfiles() { if [ ! -f "${f}" ]; then error "Package ${f} not found!" exit 1 - elif ${REQUIRE_SIGNATURE} && [ ! -f "${f}.sig" ]; then + elif "${REQUIRE_SIGNATURE}" && [ ! -f "${f}.sig" ]; then error "Package signature ${f}.sig not found!" exit 1 fi @@ -324,11 +324,11 @@ getpkgfiles() { check_pkgfile() { local pkgfile=$1 - local pkgname="$(getpkgname ${pkgfile})" + local pkgname="$(getpkgname "${pkgfile}")" [ $? -ge 1 ] && return 1 - local pkgver="$(getpkgver ${pkgfile})" + local pkgver="$(getpkgver "${pkgfile}")" [ $? -ge 1 ] && return 1 - local pkgarch="$(getpkgarch ${pkgfile})" + local pkgarch="$(getpkgarch "${pkgfile}")" [ $? -ge 1 ] && return 1 in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 @@ -342,13 +342,13 @@ check_pkgfile() { check_pkgsvn() { local pkgfile="${1}" - local _pkgbase="$(getpkgbase ${pkgfile})" + local _pkgbase="$(getpkgbase "${pkgfile}")" [ $? -ge 1 ] && return 1 - local _pkgname="$(getpkgname ${pkgfile})" + local _pkgname="$(getpkgname "${pkgfile}")" [ $? -ge 1 ] && return 1 - local _pkgver="$(getpkgver ${pkgfile})" + local _pkgver="$(getpkgver "${pkgfile}")" [ $? -ge 1 ] && return 1 - local _pkgarch="$(getpkgarch ${pkgfile})" + local _pkgarch="$(getpkgarch "${pkgfile}")" [ $? -ge 1 ] && return 1 local repo="${2}" @@ -361,10 +361,10 @@ check_pkgsvn() { [ $? -ge 1 ] && return 1 fi - local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel}) )" + local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}")" [ "${svnver}" == "${_pkgver}" ] || return 1 - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) + local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) in_array "${_pkgname}" "${svnnames[@]}" || return 1 return 0 @@ -383,19 +383,19 @@ check_splitpkgs() { for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue - local _pkgbase="$(getpkgbase ${pkgfile})" + local _pkgbase="$(getpkgbase "${pkgfile}")" msg2 "Checking $_pkgbase" - local _pkgname="$(getpkgname ${pkgfile})" - local _pkgarch="$(getpkgarch ${pkgfile})" + local _pkgname="$(getpkgname "${pkgfile}")" + local _pkgarch="$(getpkgarch "${pkgfile}")" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r ${SVNREPO}/libre/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r ${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + cp -r "${SVNREPO}/$repo/$_pkgbase/PKGBUILD" "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre/$_pkgbase/PKGBUILD" "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD" "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 [[ $? -ge 1 ]] && { echo "Failed $_pkgbase-$_pkgver-$_pkgarch" @@ -403,7 +403,7 @@ check_splitpkgs() { } fi - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) + local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) for svnname in "${svnnames[@]}"; do echo "${svnname}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" done @@ -475,8 +475,8 @@ check_repo_permission() { for arch in ${ARCHES}; do local dir="${FTP_BASE}/${repo}/os/${arch}/" [ -w "${dir}" ] || return 1 - [ -f "${dir}"${repo}${DBEXT} -a ! -w "${dir}"${repo}${DBEXT} ] && return 1 - [ -f "${dir}"${repo}${FILESEXT} -a ! -w "${dir}"${repo}${FILESEXT} ] && return 1 + [ -f "${dir}${repo}"${DBEXT} -a ! -w "${dir}${repo}"${DBEXT} ] && return 1 + [ -f "${dir}${repo}"${FILESEXT} -a ! -w "${dir}${repo}"${FILESEXT} ] && return 1 done return 0 @@ -490,9 +490,9 @@ set_repo_permission() { if [ -w "${dbfile}" ]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") - chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" - chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group" - chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" + chgrp "$group" "${dbfile}" || error "Could not change group of ${dbfile} to $group" + chgrp "$group" "${filesfile}" || error "Could not change group of ${filesfile} to $group" + chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" chmod g+w "${filesfile}" || error "Could not set write permission for group $group to ${filesfile}" else error "You don't have permission to change ${dbfile}" diff --git a/db-move b/db-move index f1c3dea..53543bc 100755 --- a/db-move +++ b/db-move @@ -14,14 +14,14 @@ repo_to="${args[1]}" ftppath_from="${FTP_BASE}/${repo_from}/os/" ftppath_to="${FTP_BASE}/${repo_to}/os/" -if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then +if ! check_repo_permission "$repo_to" || ! check_repo_permission "$repo_from"; then die "You don't have permission to move packages from ${repo_from} to ${repo_to}" fi # TODO: this might lock too much (architectures) for pkgarch in "${ARCHES[@]}"; do - repo_lock ${repo_to} ${pkgarch} || exit 1 - repo_lock ${repo_from} ${pkgarch} || exit 1 + repo_lock "${repo_to}" "${pkgarch}" || exit 1 + repo_lock "${repo_from}" "${pkgarch}" || exit 1 done # First loop is to check that all necessary files exist @@ -29,12 +29,12 @@ for pkgbase in "${args[@]:2}"; do for pkgarch in "${ARCHES[@]}" 'any'; do svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) + pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) if [ ${#pkgnames[@]} -lt 1 ]; then die "Could not read pkgname" fi - pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") if [ -z "${pkgver}" ]; then die "Could not read pkgver" fi @@ -47,7 +47,7 @@ for pkgbase in "${args[@]:2}"; do for pkgname in "${pkgnames[@]}"; do for tarch in "${tarches[@]}"; do - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null + getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} >/dev/null done done continue 2 @@ -71,22 +71,22 @@ for pkgbase in "${args[@]:2}"; do tarches=("${pkgarch}") fi msg2 "${pkgbase} (${tarches[*]})" - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) - pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) + pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") for pkgname in "${pkgnames[@]}"; do for tarch in "${tarches[@]}"; do - pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) + pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}) pkgfile="${pkgpath##*/}" # copy package to pool if needed # TODO: can be removed once every package has been moved to the package pool - if [ ! -f ${FTP_BASE}/${PKGPOOL}/${pkgfile} ]; then - cp ${pkgpath} ${FTP_BASE}/${PKGPOOL} + if [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkgfile}" ]; then + cp "${pkgpath}" "${FTP_BASE}/${PKGPOOL}" fi - ln -s "../../../${PKGPOOL}/${pkgfile}" ${ftppath_to}/${tarch}/ - if [ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" ${ftppath_to}/${tarch}/ + ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" + if [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig" ]; then + ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" fi add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " remove_pkgs[${tarch}]+="${pkgname} " @@ -104,6 +104,6 @@ for tarch in "${ARCHES[@]}"; do done for pkgarch in "${ARCHES[@]}"; do - repo_unlock ${repo_from} ${pkgarch} - repo_unlock ${repo_to} ${pkgarch} + repo_unlock "${repo_from}" "${pkgarch}" + repo_unlock "${repo_to}" "${pkgarch}" done diff --git a/db-remove b/db-remove index 46585ad..b0ed9bd 100755 --- a/db-remove +++ b/db-remove @@ -12,7 +12,7 @@ repo="$1" arch="$2" pkgbases=("${@:3}") -if ! check_repo_permission $repo; then +if ! check_repo_permission "$repo"; then die "You don't have permission to remove packages from ${repo}" fi @@ -23,7 +23,7 @@ else fi for tarch in "${tarches[@]}"; do - repo_lock $repo $tarch || exit 1 + repo_lock "$repo" "$tarch" || exit 1 done remove_pkgs=() @@ -31,7 +31,7 @@ for pkgbase in "${pkgbases[@]}"; do msg "Removing $pkgbase from [$repo]..." if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then - remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) + remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo "${pkgname[@]}")) else warning "$pkgbase not found in ABS(libre)" warning "Removing only $pkgbase from the repo" @@ -42,5 +42,5 @@ done for tarch in "${tarches[@]}"; do arch_repo_remove "${repo}" "${tarch}" "${remove_pkgs[@]}" - repo_unlock $repo $tarch + repo_unlock "$repo" "$tarch" done diff --git a/db-repo-add b/db-repo-add index a6355a1..92be22e 100755 --- a/db-repo-add +++ b/db-repo-add @@ -10,32 +10,32 @@ fi repo="$1" arch="$2" -pkgfiles=(${@:3}) +pkgfiles=("${@:3}") ftppath="$FTP_BASE/$repo/os" -if ! check_repo_permission $repo; then +if ! check_repo_permission "$repo"; then die "You don't have permission to add packages to ${repo}" fi if [ "$arch" == "any" ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do - repo_lock $repo $tarch || exit 1 +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 done -for tarch in ${tarches[@]}; do - for pkgfile in ${pkgfiles[@]}; do +for tarch in "${tarches[@]}"; do + for pkgfile in "${pkgfiles[@]}"; do if [[ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ]]; then die "Package file ${pkgfile##*/} not found in ${FTP_BASE}/${repo}/os/${arch}/" else msg "Adding $pkgfile to [$repo]..." fi done - arch_repo_add "${repo}" "${tarch}" ${pkgfiles[@]} - repo_unlock $repo $tarch + arch_repo_add "${repo}" "${tarch}" "${pkgfiles[@]}" + repo_unlock "$repo" "$tarch" done diff --git a/db-repo-remove b/db-repo-remove index 7077d62..487abd8 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -10,28 +10,28 @@ fi repo="$1" arch="$2" -pkgnames=(${@:3}) +pkgnames=("${@:3}") ftppath="$FTP_BASE/$repo/os" -if ! check_repo_permission $repo; then +if ! check_repo_permission "$repo"; then die "You don't have permission to remove packages from ${repo}" fi if [ "$arch" == "any" ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do - repo_lock $repo $tarch || exit 1 +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 done -for tarch in ${tarches[@]}; do - for pkgname in ${pkgnames[@]}; do +for tarch in "${tarches[@]}"; do + for pkgname in "${pkgnames[@]}"; do msg "Removing $pkgname from [$repo]..." done - arch_repo_remove "${repo}" "${tarch}" ${pkgnames[@]} - repo_unlock $repo $tarch + arch_repo_remove "${repo}" "${tarch}" "${pkgnames[@]}" + repo_unlock "$repo" "$tarch" done diff --git a/db-sync b/db-sync index 138328b..9b90219 100755 --- a/db-sync +++ b/db-sync @@ -16,7 +16,8 @@ # Run as `V=true db-sync` to get verbose output VERBOSE=${V} -${VERBOSE} && extra="-v" +extra=() +${VERBOSE} && extra+=(-v) WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT @@ -24,19 +25,19 @@ trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT # Returns contents of a repo get_repos() { # Exclude everything but db files - rsync ${extra} --no-motd -mrtlH --no-p --include="*/" \ + rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ --include="*.files" \ --include="*${FILESEXT}" \ --exclude="*" \ --delete-after \ - rsync://${mirror}/${mirrorpath}/ "$WORKDIR" + "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" } get_repo_content() { # Return all contents - bsdtar tf ${1} | \ + bsdtar tf "${1}" | \ cut -d "/" -f 1 | \ sort -u } @@ -68,12 +69,12 @@ init() { get_repos # Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do + for _repo in "${ARCHREPOS[@]}"; do + for _arch in "${ARCHARCHES[@]}"; do msg "Processing ${_repo}-${_arch}" - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} + files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} if [ ! -f "${db_file}" ]; then warning "%s doesn't exist, skipping this repo-arch" "${db_file}" @@ -93,7 +94,7 @@ init() { LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ |& sed -n 's/-> Removing/ &/p' # Get db contents - db=($(get_repo_content ${db_file})) + db=($(get_repo_content "${db_file}")) msg2 "Process clean db for syncing..." @@ -103,30 +104,30 @@ init() { # IMPORTANT: the . in the sed command is needed because an empty # whitelist would consist of a single * allowing any package to # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > /tmp/${_repo}-${_arch}.whitelist + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" # Sync excluding everything but whitelist # We delete here for cleanup - rsync ${extra} --no-motd -rtlH \ + rsync "${extra[@]}" --no-motd -rtlH \ --delete-after \ --delete-excluded \ --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --include-from="/tmp/${_repo}-${_arch}.whitelist" \ --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ + "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" # Add a new whitelist whitelists+=(/tmp/${_repo}-${_arch}.whitelist) msg "Putting databases back in place" - rsync ${extra} --no-motd -rtlH \ + rsync "${extra[@]}" --no-motd -rtlH \ --delay-updates \ --safe-links \ - ${WORKDIR}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ + "${WORKDIR}/${_repo}/os/${_arch}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" # Cleanup unset db @@ -136,7 +137,7 @@ init() { msg "Syncing package pool" # Concatenate all whitelists, check for single *s just in case - cat ${whitelists[@]} | grep -v "^\*$" | sort -u > /tmp/any.whitelist + cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" @@ -144,14 +145,14 @@ init() { # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too local pkgpool - for pkgpool in ${ARCHPKGPOOLS[@]}; do - rsync ${extra} --no-motd -rtlH \ + for pkgpool in "${ARCHPKGPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${pkgpool}/ \ - ${FTP_BASE}/${pkgpool}/ + "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ + "${FTP_BASE}/${pkgpool}/" done # Sync sources @@ -163,14 +164,14 @@ init() { # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too local srcpool - for srcpool in ${ARCHSRCPOOLS[@]}; do - rsync ${extra} --no-motd -rtlH \ + for srcpool in "${ARCHSRCPOOLS[@]}"; do + rsync "${extra[@]}" --no-motd -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${srcpool}/ \ - ${FTP_BASE}/${srcpool}/ + "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ + "${FTP_BASE}/${srcpool}/" done # Cleanup @@ -191,14 +192,13 @@ source "$(dirname "$(readlink -e "$0")")/libremessages" # Check variables presence for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do - test -z "${!var}" && fatal_error "Empty ${var}" done # From makepkg set -E for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" done trap 'trap_exit INT "Aborted by user! Exiting..."' INT trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR diff --git a/db-update b/db-update index 186ed55..18162db 100755 --- a/db-update +++ b/db-update @@ -19,7 +19,7 @@ fi # TODO: this might lock too much (architectures) for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do - repo_lock ${repo} ${pkgarch} || exit 1 + repo_lock "${repo}" "${pkgarch}" || exit 1 done done @@ -55,7 +55,7 @@ for repo in "${repos[@]}"; do any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do add_pkgs=() - arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) + arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-"${pkgarch}"${PKGEXT} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" msg2 "${pkgfile} (${pkgarch})" @@ -81,7 +81,7 @@ done for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do - repo_unlock ${repo} ${pkgarch} + repo_unlock "${repo}" "${pkgarch}" done done diff --git a/testing2x b/testing2x index b6828fc..37c4395 100755 --- a/testing2x +++ b/testing2x @@ -11,28 +11,28 @@ fi # Lock everything to reduce possibility of interfering task between the different repo-updates script_lock for repo in 'core' 'extra' 'testing'; do - for pkgarch in ${ARCHES[@]}; do - repo_lock ${repo} ${pkgarch} || exit 1 + for pkgarch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${pkgarch}" || exit 1 done done declare -A pkgs -for pkgbase in $*; do +for pkgbase in "$@"; do if [ ! -d "${WORKDIR}/${pkgbase}" ]; then /usr/bin/svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null found_source=false - for pkgarch in ${ARCHES[@]} 'any'; do + for pkgarch in "${ARCHES[@]}" 'any'; do svnrepo_from="${WORKDIR}/${pkgbase}/testing-${pkgarch}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then found_source=true break fi done - ${found_source} || die "${pkgbase} not found in [testing]" + "${found_source}" || die "${pkgbase} not found in [testing]" found_target=false - for pkgarch in ${ARCHES[@]} 'any'; do + for pkgarch in "${ARCHES[@]}" 'any'; do for repo in 'core' 'extra'; do svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" if [ -r "${svnrepo_to}/PKGBUILD" ]; then @@ -42,16 +42,16 @@ for pkgbase in $*; do fi done done - ${found_target} || die "${pkgbase} neither found in [core] nor [extra]" + "${found_target}" || die "${pkgbase} neither found in [core] nor [extra]" fi done -for pkgarch in ${ARCHES[@]}; do - repo_unlock 'testing' ${pkgarch} +for pkgarch in "${ARCHES[@]}"; do + repo_unlock 'testing' "${pkgarch}" done for repo in 'core' 'extra'; do - for pkgarch in ${ARCHES[@]}; do - repo_unlock ${repo} ${pkgarch} + for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${pkgarch}" done if [ -n "${pkgs[${repo}]}" ]; then "$(dirname "$(readlink -e "$0")")/db-move" 'testing' "${repo}" ${pkgs[${repo}]} -- cgit v1.2.3 From cc3720b21451fbf73e0075b0955653fb37d1e106 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 13:15:50 -0400 Subject: use abort() and die() from librelib --- db-functions | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/db-functions b/db-functions index a1e86d9..6bab5a0 100644 --- a/db-functions +++ b/db-functions @@ -29,8 +29,8 @@ WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") LOCKS=() REPO_MODIFIED=0 -# Used: plain, msg, msg2, warning, error, in_array, get_full_version -# Overwritten: cleanup, abort, die +# Used: plain, msg, msg2, warning, error, in_array, get_full_version, abort, die +# Overwritten: cleanup # Ignored: stat_busy, stat_done, # setup_workdir, trab_abort, trap_exit, # lock, slock, lock_close @@ -88,16 +88,6 @@ cleanup() { [ "$1" ] && exit "$1" } -abort() { - msg 'Aborting...' - cleanup 0 -} - -die() { - error "$*" - cleanup 1 -} - trap abort INT QUIT TERM HUP trap cleanup EXIT -- cgit v1.2.3 From ba575a4e64be157eeae1028ed86b29cb0042f41b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 13:29:51 -0400 Subject: More quoting fixes --- any-to-ours | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/any-to-ours b/any-to-ours index 6afc7b0..c203a6e 100755 --- a/any-to-ours +++ b/any-to-ours @@ -21,7 +21,7 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR BASEARCH='x86_64' # Traverse all Arch repos -for _repo in ${ARCHREPOS[@]}; do +for _repo in "${ARCHREPOS[@]}"; do msg "Processing ${_repo}..." # Find 'any' packages @@ -35,7 +35,7 @@ for _repo in ${ARCHREPOS[@]}; do continue fi - for _arch in ${OURARCHES[@]}; do + for _arch in "${OURARCHES[@]}"; do msg2 "Syncing ${_arch}..." # Sync 'any' only and extract the synced packages @@ -44,8 +44,8 @@ for _repo in ${ARCHREPOS[@]}; do --include='*-any.pkg.tar.?z' \ --include='*-any.pkg.tar.?z.sig' \ --exclude='*' \ - ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ + "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + "${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \ grep 'any\.pkg\.tar\..z$' | \ cut -d ' ' -f 1 )) @@ -58,11 +58,10 @@ for _repo in ${ARCHREPOS[@]}; do msg2 "Adding to db..." - pushd ${FTP_BASE}/${_repo}/os/${_arch}/ >/dev/null + pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null # Add the packages to the db - repo-add ${_repo}${DBEXT} \ - ${SYNCED[@]} + repo-add "${_repo}${DBEXT}" "${SYNCED[@]}" popd >/dev/null -- cgit v1.2.3 From 386c2d00249eb244bbcc9a173a64f9435b70180a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 13:45:10 -0400 Subject: Use printf formatters instead of string interpolation. I used this command to find them: egrep -r --exclude-dir={test,.git} '(plain|msg|msg2|warning|error|stat_busy|stat_done|abort|die)\s+"?[^"]*\$' --- any-to-ours | 8 +++---- create-repo | 8 +++---- cron-jobs/ftpdir-cleanup | 12 +++++----- cron-jobs/integrity-check | 2 +- cron-jobs/repo-sanity-check | 4 ++-- cron-jobs/sourceballs | 12 +++++----- cron-jobs/sourceballs2 | 4 ++-- db-check-nonfree | 6 ++--- db-cleanup | 4 ++-- db-functions | 58 ++++++++++++++++++++++----------------------- db-list-unsigned-packages | 2 +- db-move | 10 ++++---- db-remove | 10 ++++---- db-repo-add | 8 +++---- db-repo-remove | 6 ++--- db-sync | 8 +++---- db-update | 20 ++++++++-------- testing2x | 6 ++--- 18 files changed, 94 insertions(+), 94 deletions(-) diff --git a/any-to-ours b/any-to-ours index c203a6e..d98cc5d 100755 --- a/any-to-ours +++ b/any-to-ours @@ -22,7 +22,7 @@ BASEARCH='x86_64' # Traverse all Arch repos for _repo in "${ARCHREPOS[@]}"; do - msg "Processing ${_repo}..." + msg "Processing %s..." "${_repo}" # Find 'any' packages # This is hardcoded but it could release other arches... @@ -31,12 +31,12 @@ for _repo in "${ARCHREPOS[@]}"; do -printf "%f ")) if [ ${#PKGS[@]} -eq 0 ]; then - msg2 "No 'any' packages here" + msg2 "No '%s' packages here" any continue fi for _arch in "${OURARCHES[@]}"; do - msg2 "Syncing ${_arch}..." + msg2 "Syncing %s..." "${_arch}" # Sync 'any' only and extract the synced packages SYNCED=($( @@ -54,7 +54,7 @@ for _repo in "${ARCHREPOS[@]}"; do continue fi - msg2 "Synced ${#SYNCED[@]} packages: ${SYNCED[@]}" + msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" msg2 "Adding to db..." diff --git a/create-repo b/create-repo index cfdbc23..b218832 100755 --- a/create-repo +++ b/create-repo @@ -5,20 +5,20 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -eq 0 ]; then - msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" + msg "Usage: %s repo1 [repo2 ... repoX]" "${0##*/}" exit 1 fi msg "Creating repos..." for _repo in "$@"; do - msg2 "Creating [${_repo}]" + msg2 "Creating [%s]" "${_repo}" mkdir -p "${FTP_BASE}/staging/${_repo}" || \ error "Failed creating staging dir" for _arch in "${ARCHES[@]}"; do mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ - error "Failed creating ${_arch} dir" + error "Failed creating %s dir" "${_arch}" done done -msg "Don't forget to add them to the PKGREPOS array on %s/config" "$(dirname "$(readlink -e "$0")")" +msg "Don't forget to add them to the PKGREPOS array on %s" "$(dirname "$(readlink -e "$0")")/config" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index d5a277b..04ca6c9 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -36,17 +36,17 @@ for repo in "${PKGREPOS[@]}"; do missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#missing_pkgs[@]} -ge 1 ]; then - error "Missing packages in [${repo}] (${arch})..." + error "Missing packages in [%s] (%s)..." "${repo}" "${arch}" for missing_pkg in "${missing_pkgs[@]}"; do - msg2 "${missing_pkg}" + msg2 '%s' "${missing_pkg}" done fi old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old packages from [${repo}] (${arch})..." + msg "Removing old packages from [%s] (%s)..." "${repo}" "${arch}" for old_pkg in "${old_pkgs[@]}"; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" done fi @@ -62,7 +62,7 @@ old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from package pool..." for old_pkg in "${old_pkgs[@]}"; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" done fi @@ -71,7 +71,7 @@ old_pkgs=($(find "${CLEANUP_DESTDIR}" -type f -name "*${PKGEXT}" -mtime +"${CLEA if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from the cleanup directory..." for old_pkg in "${old_pkgs[@]}"; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" if ! "${CLEANUP_DRYRUN}"; then rm -f "${CLEANUP_DESTDIR}/${old_pkg}" rm -f "${CLEANUP_DESTDIR}/${old_pkg}.sig" diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 33a4eb6..7459380 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -8,7 +8,7 @@ dirname="$(dirname "$(readlink -e "$0")")" script_lock if [ $# -ne 1 ]; then - die "usage: ${0##*/} " + die "usage: %s " "${0##*/}" fi mailto=$1 diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 105bac4..012b544 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -6,7 +6,7 @@ # Traverse all repos for _repo in "${PKGREPOS[@]}"; do - msg "Cleaning up [${_repo}]" + msg "Cleaning up [%s]" "${_repo}" # Find all pkgnames on this repo's abs on_abs=($( @@ -30,7 +30,7 @@ for _repo in "${PKGREPOS[@]}"; do # quit if abs is empty if [ ${#on_abs[*]} -eq 0 ]; then - warning "[${_repo}]'s ABS tree is empty, skipping" + warning "[%s]'s ABS tree is empty, skipping" "${_repo}" break fi diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9c07c22..5ce7cfd 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -108,15 +108,15 @@ for repo in "${PKGREPOS[@]}"; do done < "${WORKDIR}/db-${repo}" if [ ${#newpkgs[@]} -ge 1 ]; then - msg "Adding source packages for [${repo}]..." + msg "Adding source packages for [%s]..." "${repo}" for new_pkg in "${newpkgs[@]}"; do - msg2 "${new_pkg}" + msg2 '%s' "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then - msg "Failed to create source packages for [${repo}]..." + msg "Failed to create source packages for [%s]..." "${repo}" for failed_pkg in "${failedpkgs[@]}"; do - msg2 "${failed_pkg}" + msg2 '%s' "${failed_pkg}" done fi done @@ -130,7 +130,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." "${SOURCE_CLEANUP_DRYRUN}" && warning 'dry run mode is active' for old_pkg in "${old_pkgs[@]}"; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" if ! "${SOURCE_CLEANUP_DRYRUN}"; then mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" @@ -142,7 +142,7 @@ old_pkgs=($(find "${SOURCE_CLEANUP_DESTDIR}" -type f -name "*${SRCEXT}" -mtime + if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." for old_pkg in "${old_pkgs[@]}"; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" "${SOURCE_CLEANUP_DRYRUN}" || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done fi diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index f7b3779..c431e9f 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -22,14 +22,14 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null for repo in "${PKGREPOS[@]}"; do - msg "Sourceballing [${repo}]" + msg "Sourceballing [%s]" "${repo}" pushd "$repo" >/dev/null find -maxdepth 1 -type d | while read pkg; do pushd "${SVNREPO}/$repo/$pkg" >/dev/null [[ ! -e ./PKGBUILD ]] && { - warning "$repo/$pkg is not a package" + warning "%s is not a package" "$repo/$pkg" continue } diff --git a/db-check-nonfree b/db-check-nonfree index cae4a14..c1673c3 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then - warning "Calling ${0##*/} with a specific repository is not supported" + warning "Calling %s with a specific repository is not supported" "${0##*/}" exit 1 fi @@ -19,7 +19,7 @@ msg "Check nonfree in repo:" nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u)) for repo in "${ARCHREPOS[@]}"; do for pkgarch in "${ARCHES[@]}"; do - msg2 "$repo $pkgarch" + msg2 "%s %s" "$repo" "$pkgarch" if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue fi @@ -33,7 +33,7 @@ for repo in "${ARCHREPOS[@]}"; do fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: ${cleanpkgs[*]}" + msg2 "Nonfree: %s" "${cleanpkgs[*]}" arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" fi done diff --git a/db-cleanup b/db-cleanup index b2f2e08..6ac9747 100755 --- a/db-cleanup +++ b/db-cleanup @@ -30,7 +30,7 @@ EXTRAFLAGS=() for _repo in "${PKGREPOS[@]}"; do for _arch in "${ARCHES[@]}"; do - msg "Getting ${_repo}-${_arch} database" + msg "Getting %s-%s database" "${_repo}" "${_arch}" dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" @@ -51,7 +51,7 @@ done msg "Removing old files:" for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do - msg2 "${POOL}" + msg2 '%s' "${POOL}" rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ --include-from="/tmp/${0##*/}.$$.filter" \ diff --git a/db-functions b/db-functions index 1384080..51fc1bd 100644 --- a/db-functions +++ b/db-functions @@ -97,7 +97,7 @@ script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then local _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" - error "Script ${0##*/} is already locked by $_owner." + error "Script %s is already locked by %s." "${0##*/}" "$_owner" exit 1 else set_umask @@ -108,7 +108,7 @@ script_lock() { script_unlock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if [ ! -d "$LOCKDIR" ]; then - warning "Script ${0##*/} was not locked!" + warning "Script %s was not locked!" "${0##*/}" restore_umask return 1 else @@ -128,12 +128,12 @@ cleanup() { repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then - msg "Removing left over lock from [${repo}] (${arch})" + msg "Removing left over lock from [%s] (%s)" "${repo}" "${arch}" repo_unlock "$repo" "$arch" fi done if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then - msg "Removing left over lock from ${0##*/}" + msg "Removing left over lock from %s" "${0##*/}" script_unlock fi rm -rf "$WORKDIR" @@ -146,7 +146,7 @@ abort() { } die() { - error "$*" + error "$@" cleanup 1 } @@ -167,11 +167,11 @@ repo_lock () { # This is the lock file used by repo-add and repo-remove if [ -f "${DBLOCKFILE}" ]; then - error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat "$DBLOCKFILE")" + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$DBLOCKFILE")" return 1 fi if [ -f "${FILESLOCKFILE}" ]; then - error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat "$FILESLOCKFILE")" + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$2" "$2" "$(<"$FILESLOCKFILE")" return 1 fi @@ -188,8 +188,8 @@ repo_lock () { while [ "$_count" -le "$_trial" ] || "$_lockblock" ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" - warning "Repo [${1}] (${2}) is already locked by $_owner. " - msg2 "Retrying in $LOCK_DELAY seconds..." + warning "Repo [%s] (%s) is already locked by %s." "$1" "$2" "$_owner" + msg2 "Retrying in %d seconds..." "$LOCK_DELAY" else LOCKS+=("$1.$2") set_umask @@ -199,14 +199,14 @@ repo_lock () { let _count=$_count+1 done - error "Repo [${1}] (${2}) is already locked by $_owner. Giving up!" + error "Repo [%s] (%s) is already locked by %s. Giving up!" "${1}" "${2}" "$_owner" return 1 } repo_unlock () { #repo_unlock local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [ ! -d "$LOCKDIR" ]; then - warning "Repo lock [${1}] (${2}) was not locked!" + warning "Repo lock [%s] (%s) was not locked!" "${1}" "${2}" restore_umask return 1 else @@ -254,7 +254,7 @@ getpkgname() { _name="$(_grep_pkginfo "$1" "pkgname")" if [ -z "$_name" ]; then - error "Package '$1' has no pkgname in the PKGINFO. Fail!" + error "Package '%s' has no pkgname in the PKGINFO. Fail!" "$1" exit 1 fi @@ -267,7 +267,7 @@ getpkgver() { _ver="$(_grep_pkginfo "$1" "pkgver")" if [ -z "$_ver" ]; then - error "Package '$1' has no pkgver in the PKGINFO. Fail!" + error "Package '%s' has no pkgver in the PKGINFO. Fail!" "$1" exit 1 fi @@ -279,7 +279,7 @@ getpkgarch() { _ver="$(_grep_pkginfo "$1" "arch")" if [ -z "$_ver" ]; then - error "Package '$1' has no arch in the PKGINFO. Fail!" + error "Package '%s' has no arch in the PKGINFO. Fail!" "$1" exit 1 fi @@ -291,10 +291,10 @@ getpkgfile() { error 'No canonical package found!' exit 1 elif [ ! -f "${1}" ]; then - error "Package ${1} not found!" + error "Package %s not found!" "${1}" exit 1 elif "${REQUIRE_SIGNATURE}" && [ ! -f "${1}.sig" ]; then - error "Package signature ${1}.sig not found!" + error "Package signature %s not found!" "${1}.sig" exit 1 fi @@ -310,10 +310,10 @@ getpkgfiles() { for f in "${@}"; do if [ ! -f "${f}" ]; then - error "Package ${f} not found!" + error "Package %s not found!" "${f}" exit 1 elif "${REQUIRE_SIGNATURE}" && [ ! -f "${f}.sig" ]; then - error "Package signature ${f}.sig not found!" + error "Package signature %s not found!" "${f}.sig" exit 1 fi done @@ -384,7 +384,7 @@ check_splitpkgs() { for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase "${pkgfile}")" - msg2 "Checking $_pkgbase" + msg2 "Checking %s" "$_pkgbase" local _pkgname="$(getpkgname "${pkgfile}")" local _pkgarch="$(getpkgarch "${pkgfile}")" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" @@ -490,12 +490,12 @@ set_repo_permission() { if [ -w "${dbfile}" ]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") - chgrp "$group" "${dbfile}" || error "Could not change group of ${dbfile} to $group" - chgrp "$group" "${filesfile}" || error "Could not change group of ${filesfile} to $group" - chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" - chmod g+w "${filesfile}" || error "Could not set write permission for group $group to ${filesfile}" + chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "${dbfile}" "$group" + chgrp "$group" "${filesfile}" || error "Could not change group of %s to %s" "${filesfile}" "$group" + chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "${dbfile}" + chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "${filesfile}" else - error "You don't have permission to change ${dbfile}" + error "You don't have permission to change %s" "${dbfile}" fi } @@ -507,9 +507,9 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ - || error "repo-add ${repo}${DBEXT} ${pkgs[*]}" + || error '%s' "repo-add ${repo}${DBEXT} ${pkgs[*]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ - || error "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" + || error '%s' "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" } @@ -522,12 +522,12 @@ arch_repo_remove() { local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ ! -f "${dbfile}" ]; then - error "No database found at '${dbfile}'" + error "No database found at '%s'" "${dbfile}" return 1 fi /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ - || error "repo-remove ${dbfile} ${pkgs[*]}" + || error '%s' "repo-remove ${dbfile} ${pkgs[*]}" /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ - || error "repo-remove ${filesfile} ${pkgs[*]}" + || error '%s' "repo-remove ${filesfile} ${pkgs[*]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index c88203b..095e1e6 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -24,7 +24,7 @@ set -e . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 1 ]; then - msg "usage: ${0##*/} " + msg "usage: %s " "${0##*/}" exit 1 fi diff --git a/db-move b/db-move index 53543bc..b057d28 100755 --- a/db-move +++ b/db-move @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi @@ -15,7 +15,7 @@ ftppath_from="${FTP_BASE}/${repo_from}/os/" ftppath_to="${FTP_BASE}/${repo_to}/os/" if ! check_repo_permission "$repo_to" || ! check_repo_permission "$repo_from"; then - die "You don't have permission to move packages from ${repo_from} to ${repo_to}" + die "You don't have permission to move packages from %s to %s" "${repo_from}" "${repo_to}" fi # TODO: this might lock too much (architectures) @@ -53,10 +53,10 @@ for pkgbase in "${args[@]:2}"; do continue 2 fi done - die "${pkgbase} not found in ${repo_from}" + die "%s not found in %s" "${pkgbase}" "${repo_from}" done -msg "Moving packages from [${repo_from}] to [${repo_to}]..." +msg "Moving packages from [%s] to [%s]..." "${repo_from}" "${repo_to}" declare -A add_pkgs declare -A remove_pkgs @@ -70,7 +70,7 @@ for pkgbase in "${args[@]:2}"; do else tarches=("${pkgarch}") fi - msg2 "${pkgbase} (${tarches[*]})" + msg2 '%s (%s)' "${pkgbase}" "${tarches[*]}" pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") diff --git a/db-remove b/db-remove index b0ed9bd..8fff9db 100755 --- a/db-remove +++ b/db-remove @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi @@ -13,7 +13,7 @@ arch="$2" pkgbases=("${@:3}") if ! check_repo_permission "$repo"; then - die "You don't have permission to remove packages from ${repo}" + die "You don't have permission to remove packages from %s" "${repo}" fi if [ "$arch" == "any" ]; then @@ -28,13 +28,13 @@ done remove_pkgs=() for pkgbase in "${pkgbases[@]}"; do - msg "Removing $pkgbase from [$repo]..." + msg "Removing %s from [%s]..." "$pkgbase" "$repo" if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo "${pkgname[@]}")) else - warning "$pkgbase not found in ABS(libre)" - warning "Removing only $pkgbase from the repo" + warning "%s not found in ABS(libre)" "$pkgbase" + warning "Removing only %s from the repo" "$pkgbase" warning "If it was a split package you have to remove the others yourself!" remove_pkgs+=("$pkgbase") fi diff --git a/db-repo-add b/db-repo-add index 92be22e..4611bdf 100755 --- a/db-repo-add +++ b/db-repo-add @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi @@ -15,7 +15,7 @@ pkgfiles=("${@:3}") ftppath="$FTP_BASE/$repo/os" if ! check_repo_permission "$repo"; then - die "You don't have permission to add packages to ${repo}" + die "You don't have permission to add packages to %s" "${repo}" fi if [ "$arch" == "any" ]; then @@ -31,9 +31,9 @@ done for tarch in "${tarches[@]}"; do for pkgfile in "${pkgfiles[@]}"; do if [[ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ]]; then - die "Package file ${pkgfile##*/} not found in ${FTP_BASE}/${repo}/os/${arch}/" + die "Package file %s not found in %s" "${pkgfile##*/}" "${FTP_BASE}/${repo}/os/${arch}/" else - msg "Adding $pkgfile to [$repo]..." + msg "Adding %s to [%s]..." "$pkgfile" "$repo" fi done arch_repo_add "${repo}" "${tarch}" "${pkgfiles[@]}" diff --git a/db-repo-remove b/db-repo-remove index 487abd8..aadc4ce 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi @@ -15,7 +15,7 @@ pkgnames=("${@:3}") ftppath="$FTP_BASE/$repo/os" if ! check_repo_permission "$repo"; then - die "You don't have permission to remove packages from ${repo}" + die "You don't have permission to remove packages from %s" "${repo}" fi if [ "$arch" == "any" ]; then @@ -30,7 +30,7 @@ done for tarch in "${tarches[@]}"; do for pkgname in "${pkgnames[@]}"; do - msg "Removing $pkgname from [$repo]..." + msg "Removing %s from [%s]..." "$pkgname" "$repo" done arch_repo_remove "${repo}" "${tarch}" "${pkgnames[@]}" repo_unlock "$repo" "$tarch" diff --git a/db-sync b/db-sync index 9b90219..58e211d 100755 --- a/db-sync +++ b/db-sync @@ -71,7 +71,7 @@ init() { # Traverse all repo-arch pairs for _repo in "${ARCHREPOS[@]}"; do for _arch in "${ARCHARCHES[@]}"; do - msg "Processing ${_repo}-${_arch}" + msg "Processing %s-%s" "${_repo}-${_arch}" db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} @@ -106,7 +106,7 @@ init() { # pass through printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" # Sync excluding everything but whitelist # We delete here for cleanup @@ -139,7 +139,7 @@ init() { # Concatenate all whitelists, check for single *s just in case cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our @@ -192,7 +192,7 @@ source "$(dirname "$(readlink -e "$0")")/libremessages" # Check variables presence for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do - test -z "${!var}" && fatal_error "Empty ${var}" + test -z "${!var}" && fatal_error "Empty %s" "${var}" done # From makepkg diff --git a/db-update b/db-update index 18162db..ced673e 100755 --- a/db-update +++ b/db-update @@ -6,14 +6,14 @@ shopt -s nullglob if [ $# -ge 1 ]; then - warning "Calling ${0##*/} with a specific repository is no longer supported" + warning "Calling %s with a specific repository is no longer supported" "${0##*/}" exit 1 fi # Find repos with packages to release repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) if [ $? -ge 1 ]; then - die "Could not read ${STAGING}" + die "Could not read %s" "${STAGING}" fi # TODO: this might lock too much (architectures) @@ -28,37 +28,37 @@ for repo in "${repos[@]}"; do pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then - die "You don't have permission to update packages in ${repo}" + die "You don't have permission to update packages in %s" "${repo}" fi for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then - die "Package ${repo}/${pkg##*/} is a symbolic link" + die "Package %s is a symbolic link" "${repo}/${pkg##*/}" fi if ! check_pkgfile "${pkg}"; then - die "Package ${repo}/${pkg##*/} is not consistent with its meta data" + die "Package %s is not consistent with its meta data" "${repo}/${pkg##*/}" fi if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/${pkg##*/} already exists in another repository" + die "Package %s already exists in another repository" "${repo}/${pkg##*/}" fi done # This is fucking obnoxious #if ! check_splitpkgs ${repo} "${pkgs[@]}"; then - # die "Missing split packages for ${repo}" + # die "Missing split packages for %s" "${repo}" #fi else - die "Could not read ${STAGING}" + die "Could not read %s" "${STAGING}" fi done for repo in "${repos[@]}"; do - msg "Updating [${repo}]..." + msg "Updating [%s]..." "${repo}" any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-"${pkgarch}"${PKGEXT} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" - msg2 "${pkgfile} (${pkgarch})" + msg2 '%s (%s)' "${pkgfile}" "${pkgarch}" # any packages might have been moved by the previous run if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" diff --git a/testing2x b/testing2x index 37c4395..0c5c8d7 100755 --- a/testing2x +++ b/testing2x @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 1 ]; then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi @@ -30,7 +30,7 @@ for pkgbase in "$@"; do break fi done - "${found_source}" || die "${pkgbase} not found in [testing]" + "${found_source}" || die "%s not found in [testing]" "${pkgbase}" found_target=false for pkgarch in "${ARCHES[@]}" 'any'; do for repo in 'core' 'extra'; do @@ -42,7 +42,7 @@ for pkgbase in "$@"; do fi done done - "${found_target}" || die "${pkgbase} neither found in [core] nor [extra]" + "${found_target}" || die "%s neither found in [core] nor [extra]" "${pkgbase}" fi done -- cgit v1.2.3 From 3abd3eaf4b3e6495d4603a8eb478e0f603ad51d9 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 13:46:25 -0400 Subject: create-repo: get rid of old staging logic --- create-repo | 3 --- 1 file changed, 3 deletions(-) diff --git a/create-repo b/create-repo index b218832..3feb098 100755 --- a/create-repo +++ b/create-repo @@ -12,9 +12,6 @@ fi msg "Creating repos..." for _repo in "$@"; do msg2 "Creating [%s]" "${_repo}" - mkdir -p "${FTP_BASE}/staging/${_repo}" || \ - error "Failed creating staging dir" - for _arch in "${ARCHES[@]}"; do mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ error "Failed creating %s dir" "${_arch}" -- cgit v1.2.3 From b2abcf95b33fc3c24855da83bc71112098e849b4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:19:49 -0400 Subject: use mv_acl --- cron-jobs/ftpdir-cleanup | 4 ++-- cron-jobs/sourceballs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 04ca6c9..4a2b418 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -12,9 +12,9 @@ clean_pkg() { if [ -h "$pkg" ]; then rm -f "$pkg" "$pkg.sig" else - mv -f "$pkg" "$CLEANUP_DESTDIR" + mv_acl "$pkg" "$CLEANUP_DESTDIR/${pkg##*/}" if [ -e "$pkg.sig" ]; then - mv -f "$pkg.sig" "$CLEANUP_DESTDIR" + mv_acl "$pkg.sig" "$CLEANUP_DESTDIR/${pkg##*/}.sig" fi touch "${CLEANUP_DESTDIR}/${pkg##*/}" fi diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 5ce7cfd..b172685 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -96,7 +96,7 @@ for repo in "${PKGREPOS[@]}"; do pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") @@ -132,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in "${old_pkgs[@]}"; do msg2 '%s' "${old_pkg}" if ! "${SOURCE_CLEANUP_DRYRUN}"; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done -- cgit v1.2.3 From b3c27405fb9e2686d3db9ee23ca616dc6b93b1f3 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:20:35 -0400 Subject: silly whitespace and similar fidling --- cron-jobs/repo-sanity-check | 8 +++----- cron-jobs/sourceballs2 | 2 +- db-check-nonfree | 2 +- db-functions | 4 ++-- db-update | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 012b544..86f9629 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,5 +1,5 @@ #!/bin/bash -# Solves issue165 +# Solves issue165... on the old flyspray install. I have no idea what issue that was. . "$(dirname "$(readlink -e "$0")")/../config" . "$(dirname "$(readlink -e "$0")")/../db-functions" @@ -46,12 +46,10 @@ for _repo in "${PKGREPOS[@]}"; do <(printf '%s\n' "${on_repo[@]}" | sort -u) )) # Remove them from databases, ftpdir-cleanup will take care of the rest - find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" -exec \ - repo-remove {} "${remove[@]}" >/dev/null 2>&1 \; + find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \ + -exec repo-remove {} "${remove[@]}" >/dev/null 2>&1 \; msg2 "Removed the following packages:" plain '%s' "${remove[@]}" done - -exit $? diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index c431e9f..e935f86 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -13,7 +13,7 @@ pushd "${WORKDIR}" >/dev/null script_lock -#adjust the nice level to run at a lower priority +# Adjust the nice level to run at a lower priority renice +10 -p $$ > /dev/null # Create a list of all available source package file names diff --git a/db-check-nonfree b/db-check-nonfree index c1673c3..37b7cf6 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then - warning "Calling %s with a specific repository is not supported" "${0##*/}" + error "Calling %s with a specific repository is not supported" "${0##*/}" exit 1 fi diff --git a/db-functions b/db-functions index 51fc1bd..accc508 100644 --- a/db-functions +++ b/db-functions @@ -188,7 +188,7 @@ repo_lock () { while [ "$_count" -le "$_trial" ] || "$_lockblock" ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" - warning "Repo [%s] (%s) is already locked by %s." "$1" "$2" "$_owner" + warning "Repo [%s] (%s) is already locked by %s." "${1}" "${2}" "$_owner" msg2 "Retrying in %d seconds..." "$LOCK_DELAY" else LOCKS+=("$1.$2") @@ -490,7 +490,7 @@ set_repo_permission() { if [ -w "${dbfile}" ]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") - chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "${dbfile}" "$group" + chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "${dbfile}" "$group" chgrp "$group" "${filesfile}" || error "Could not change group of %s to %s" "${filesfile}" "$group" chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "${dbfile}" chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "${filesfile}" diff --git a/db-update b/db-update index ced673e..4d2cace 100755 --- a/db-update +++ b/db-update @@ -58,7 +58,7 @@ for repo in "${repos[@]}"; do arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-"${pkgarch}"${PKGEXT} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" - msg2 '%s (%s)' "${pkgfile}" "${pkgarch}" + msg2 "%s (%s)" "${pkgfile}" "${pkgarch}" # any packages might have been moved by the previous run if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" -- cgit v1.2.3 From 12faf7365342bcfebb951dd748d48b5db8a1acba Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:20:54 -0400 Subject: oops --- db-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions b/db-functions index accc508..e45f673 100644 --- a/db-functions +++ b/db-functions @@ -171,7 +171,7 @@ repo_lock () { return 1 fi if [ -f "${FILESLOCKFILE}" ]; then - error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$2" "$2" "$(<"$FILESLOCKFILE")" + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(<"$FILESLOCKFILE")" return 1 fi -- cgit v1.2.3 From 108262939f3b80bc82e1a2eec983944f448f2afd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:21:18 -0400 Subject: db-functions:check_repo_permissions: ARCHES is an array --- db-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions b/db-functions index e45f673..ca43259 100644 --- a/db-functions +++ b/db-functions @@ -472,7 +472,7 @@ check_repo_permission() { [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 local arch - for arch in ${ARCHES}; do + for arch in "${ARCHES[@]}"; do local dir="${FTP_BASE}/${repo}/os/${arch}/" [ -w "${dir}" ] || return 1 [ -f "${dir}${repo}"${DBEXT} -a ! -w "${dir}${repo}"${DBEXT} ] && return 1 -- cgit v1.2.3 From 14dc8bc6bf985464c119910eeb22f65f534a57b0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:21:29 -0400 Subject: more quoting fixes --- db-functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-functions b/db-functions index ca43259..5ccaaad 100644 --- a/db-functions +++ b/db-functions @@ -298,7 +298,7 @@ getpkgfile() { exit 1 fi - echo ${1} + echo "${1}" } getpkgfiles() { @@ -425,11 +425,11 @@ check_splitpkgs() { check_pkgrepos() { local pkgfile=$1 - local pkgname="$(getpkgname ${pkgfile})" + local pkgname="$(getpkgname "${pkgfile}")" [ $? -ge 1 ] && return 1 - local pkgver="$(getpkgver ${pkgfile})" + local pkgver="$(getpkgver "${pkgfile}")" [ $? -ge 1 ] && return 1 - local pkgarch="$(getpkgarch ${pkgfile})" + local pkgarch="$(getpkgarch "${pkgfile}")" [ $? -ge 1 ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 -- cgit v1.2.3 From bb6df9198b3dba28e378acf738b0770fb159a1c8 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:21:50 -0400 Subject: db-cleanup: obey TMPDIR --- db-cleanup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/db-cleanup b/db-cleanup index 6ac9747..3b5ea13 100755 --- a/db-cleanup +++ b/db-cleanup @@ -28,6 +28,9 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR EXTRAFLAGS=() "${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) +filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") +trap "rm -f -- $(printf %q "$filter")" EXIT + for _repo in "${PKGREPOS[@]}"; do for _arch in "${ARCHES[@]}"; do msg "Getting %s-%s database" "${_repo}" "${_arch}" @@ -43,7 +46,7 @@ for _repo in "${PKGREPOS[@]}"; do bsdtar tf "${dbfile}" | \ cut -d'/' -f1 | \ sort -u | \ - sed "s|$|*|" >> "/tmp/${0##*/}.$$.filter" + sed "s|$|*|" >> "$filter" done done @@ -54,7 +57,7 @@ for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do msg2 '%s' "${POOL}" rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ - --include-from="/tmp/${0##*/}.$$.filter" \ + --include-from="$filter" \ --exclude="*" \ "${FTP_BASE}/${POOL}/" \ "${FTP_BASE}/${POOL}/" -- cgit v1.2.3 From f8296b65608ea5414de118653b3929493d81a94c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:22:10 -0400 Subject: db-cleanup: clarify message --- db-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-cleanup b/db-cleanup index 3b5ea13..989bda1 100755 --- a/db-cleanup +++ b/db-cleanup @@ -63,7 +63,7 @@ for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do "${FTP_BASE}/${POOL}/" done -msg "Removing symlinks:" +msg "Removing dead symlinks:" find -L "${FTP_BASE}/" -type l "${CLEANUP_DRYRUN}" || find -L "${FTP_BASE}/" -type l -delete -- cgit v1.2.3 From e5b35d14b8f81d1ecf1d6eb77aa20c7b7ca303e9 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:22:32 -0400 Subject: db-cleanup: delete symlinks in one pass --- db-cleanup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/db-cleanup b/db-cleanup index 989bda1..efe7397 100755 --- a/db-cleanup +++ b/db-cleanup @@ -64,7 +64,6 @@ for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do done msg "Removing dead symlinks:" -find -L "${FTP_BASE}/" -type l -"${CLEANUP_DRYRUN}" || find -L "${FTP_BASE}/" -type l -delete - -exit $? +actions=(-print) +"${CLEANUP_DRYRUN}" || actions+=(-delete) +find -L "${FTP_BASE}/" -type l "${actions[@]}" -- cgit v1.2.3 From 67e03640968bba58e40f22945f21e220425d6909 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:33:36 -0400 Subject: cron-jobs/repo-sanity-check: re-arrange redirectorys to be more clear --- cron-jobs/repo-sanity-check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 86f9629..c7ee377 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -22,7 +22,7 @@ for _repo in "${PKGREPOS[@]}"; do for _pkg in "${pkgname[@]}"; do unset "package_${pkg}" >/dev/null 2>&1 done - + # this fills the on_abs array echo "${pkgname[@]}" done @@ -47,7 +47,7 @@ for _repo in "${PKGREPOS[@]}"; do # Remove them from databases, ftpdir-cleanup will take care of the rest find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \ - -exec repo-remove {} "${remove[@]}" >/dev/null 2>&1 \; + -exec repo-remove {} "${remove[@]}" \; >/dev/null 2>&1 msg2 "Removed the following packages:" plain '%s' "${remove[@]}" -- cgit v1.2.3 From e0c3d7b0ba452d75a478f2c8505c9b46dabe29b5 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:34:09 -0400 Subject: sourceballs: (from Arch): create the workdir with mode 0770 --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b172685..2895725 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -79,7 +79,7 @@ for repo in "${PKGREPOS[@]}"; do fi # Get the sources from svn - mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" + mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 -- cgit v1.2.3 From 809e7ede093e6348552a9dc3d28b61e669bb49fd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 14:34:32 -0400 Subject: db-funtions:check_splitpkgs(): use printf instead of a loop --- db-functions | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/db-functions b/db-functions index 5ccaaad..6543d96 100644 --- a/db-functions +++ b/db-functions @@ -404,8 +404,7 @@ check_splitpkgs() { fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) - for svnname in "${svnnames[@]}"; do - echo "${svnname}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" + printf '%s\n' "${svnnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" done done popd >/dev/null -- cgit v1.2.3 From cdc18bf703f6f6a55053b86c36df3775ed02eea7 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 19:26:57 -0400 Subject: oops --- db-functions | 1 - 1 file changed, 1 deletion(-) diff --git a/db-functions b/db-functions index 6543d96..a4b072b 100644 --- a/db-functions +++ b/db-functions @@ -405,7 +405,6 @@ check_splitpkgs() { local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) printf '%s\n' "${svnnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" - done done popd >/dev/null -- cgit v1.2.3 From 441f81329e9b20775402aa7778f8ab1141cc4a62 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 19:33:37 -0400 Subject: remove extra whitespace --- cron-jobs/repo-sanity-check | 1 - testing2x | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index c7ee377..bd89240 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -51,5 +51,4 @@ for _repo in "${PKGREPOS[@]}"; do msg2 "Removed the following packages:" plain '%s' "${remove[@]}" - done diff --git a/testing2x b/testing2x index 0c5c8d7..2ed5c25 100755 --- a/testing2x +++ b/testing2x @@ -8,7 +8,7 @@ if [ $# -lt 1 ]; then exit 1 fi -# Lock everything to reduce possibility of interfering task between the different repo-updates +# Lock everything to reduce possibility of interfering task between the different repo-updates script_lock for repo in 'core' 'extra' 'testing'; do for pkgarch in "${ARCHES[@]}"; do -- cgit v1.2.3 From 906544cad702b9b8816717c8ba00d40bfe10a471 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 19:34:09 -0400 Subject: sourceballs: Only show makepkg output on errors. Also, set SRCPKGDEST. --- cron-jobs/sourceballs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 2895725..a9addc3 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -94,7 +94,7 @@ for repo in "${PKGREPOS[@]}"; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 + SRCPKGDEST=. makepkg --nocolor --allsource --ignorearch --skippgpcheck >"${WORKDIR}/${pkgbase}.log" 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" # Avoid creating the same source package for every arch @@ -102,6 +102,7 @@ for repo in "${PKGREPOS[@]}"; do newpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") else failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") + cat "${WORKDIR}/${pkgbase}.log" >> "${WORKDIR}/makepkg-fail.log" fi popd >/dev/null fi @@ -147,4 +148,9 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi +if [ -f "${WORKDIR}/makepkg-fail.log" ]; then + msg "Log of failed packages" + cat "${WORKDIR}/makepkg-fail.log" +fi + script_unlock -- cgit v1.2.3 From 3c2585392d46a3726d25696882d09ec0bf11d1f5 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 19:39:08 -0400 Subject: db-functions: update ${FTP_DIR}/lastupdate when we modify the repo. Of course, this only works for things that use db-functions, so db-sync won't touch it. --- db-functions | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/db-functions b/db-functions index a4b072b..2eeffbb 100644 --- a/db-functions +++ b/db-functions @@ -27,6 +27,7 @@ mv_acl() { # set up general environment WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") LOCKS=() +REPO_MODIFIED=0 # check if messages are to be printed using color unset ALL_OFF BOLD BLUE GREEN RED YELLOW @@ -137,6 +138,11 @@ cleanup() { script_unlock fi rm -rf "$WORKDIR" + + if (( REPO_MODIFIED )); then + date +%s > "${FTP_BASE}/lastupdate" + fi + [ "$1" ] && exit "$1" } @@ -510,6 +516,8 @@ arch_repo_add() { || error '%s' "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 } arch_repo_remove() { @@ -528,4 +536,6 @@ arch_repo_remove() { /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ || error '%s' "repo-remove ${filesfile} ${pkgs[*]}" set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 } -- cgit v1.2.3 From f4daa699ddc1adda97fc76e5f42000c9e631a7a0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 19:41:38 -0400 Subject: db-move: Remove code to handle the pre-pkgpool days. --- db-move | 5 ----- 1 file changed, 5 deletions(-) diff --git a/db-move b/db-move index b057d28..2ac3f4a 100755 --- a/db-move +++ b/db-move @@ -79,11 +79,6 @@ for pkgbase in "${args[@]:2}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}) pkgfile="${pkgpath##*/}" - # copy package to pool if needed - # TODO: can be removed once every package has been moved to the package pool - if [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkgfile}" ]; then - cp "${pkgpath}" "${FTP_BASE}/${PKGPOOL}" - fi ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" if [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" -- cgit v1.2.3 From cab7fb7288f6f3f02b80f6145dd5650b83f809fa Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 20:08:02 -0400 Subject: Remove db-sync and friends. Simplify the config accordingly --- any-to-ours | 71 ------------------- config | 38 ++++------ db-check-nonfree | 46 ------------ db-sync | 212 ------------------------------------------------------- db-sync.conf | 7 -- 5 files changed, 15 insertions(+), 359 deletions(-) delete mode 100755 any-to-ours delete mode 100755 db-check-nonfree delete mode 100755 db-sync delete mode 100644 db-sync.conf diff --git a/any-to-ours b/any-to-ours deleted file mode 100755 index 8a4e874..0000000 --- a/any-to-ours +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -# Releases 'any' packages from Arch arches to ours - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(librelib messages)" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -# The architecture to compare with -BASEARCH='x86_64' - -# Traverse all Arch repos -for _repo in "${ARCHREPOS[@]}"; do - msg "Processing %s..." "${_repo}" - - # Find 'any' packages - # This is hardcoded but it could release other arches... - PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - -iname '*-any.pkg.tar.?z' \ - -printf "%f ")) - - if [ ${#PKGS[@]} -eq 0 ]; then - msg2 "No '%s' packages here" any - continue - fi - - for _arch in "${OURARCHES[@]}"; do - msg2 "Syncing %s..." "${_arch}" - - # Sync 'any' only and extract the synced packages - SYNCED=($( - rsync -av \ - --include='*-any.pkg.tar.?z' \ - --include='*-any.pkg.tar.?z.sig' \ - --exclude='*' \ - "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \ - grep 'any\.pkg\.tar\..z$' | \ - cut -d ' ' -f 1 )) - - if [ ${#SYNCED[@]} -eq 0 ]; then - msg2 "Already synced (or error happened)" - continue - fi - - msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" - - msg2 "Adding to db..." - - pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null - - # Add the packages to the db - repo-add "${_repo}${DBEXT}" "${SYNCED[@]}" - - popd >/dev/null - - # Avoid mixups - unset SYNCED PKGS - done -done diff --git a/config b/config index e43c13d..87fbf2d 100644 --- a/config +++ b/config @@ -2,29 +2,26 @@ FTP_BASE="/srv/http/repo/public" SVNREPO="/var/abs" -# Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') TESTING_REPO='testing' -# Official Parabola repos -OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') -# User repos -USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') -# Community project repos -PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'nonprism') -# Remote repos -PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") +PKGREPOS=( + # Arch + {core,extra,testing,staging} + {gnome,kde}-unstable + {community,multilib}{,-testing,-staging} + # Parabola + libre{,-testing} + libre-multilib{,-testing} + pcr kernels cross java nonprism + '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') +) PKGPOOL='pool/parabola' SRCPOOL='sources/parabola' # Directories where packages are shared between repos # *relative to FTP_BASE* -ARCHPKGPOOLS=(pool/{packages,community}) -OURPKGPOOLS=(pool/parabola) -PKGPOOLS=(${OURPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) +PKGPOOLS=(pool/{packages,community,parabola}) # Directories where sources are stored -ARCHSRCPOOLS=(sources/{packages,community}) -OURPKGPOOLS=(sources/parabola) -SRCPOOLS=(${OURSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) +SRCPOOLS=(sources/{packages,community,parabola}) CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false @@ -43,16 +40,11 @@ LOCK_TIMEOUT=300 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) -OURARCHES=(mips64el) -ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) +ARCHES=(i686 x86_64 mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.xz" SRCEXT=".src.tar.gz" +# Used by cron-jobs/sourceballs2 MAKEPKGCONF="~/.makepkg.conf" -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" - -# parabolaweb root -WEB_DIR=/srv/http/parabolagnulinux.org/web diff --git a/db-check-nonfree b/db-check-nonfree deleted file mode 100755 index 37b7cf6..0000000 --- a/db-check-nonfree +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -ge 1 ]; then - error "Calling %s with a specific repository is not supported" "${0##*/}" - exit 1 -fi - -# TODO: this might lock too much (architectures) -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${pkgarch}" || exit 1 - done -done - -msg "Check nonfree in repo:" -nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u)) -for repo in "${ARCHREPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - msg2 "%s %s" "$repo" "$pkgarch" - if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then - continue - fi - unset dbpkgs - unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in "${dbpkgs[@]}"; do - if in_array "${pkgname}" "${nonfree[@]}"; then - cleanpkgs+=("${pkgname}") - fi - done - if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: %s" "${cleanpkgs[*]}" - arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" - fi - done -done - -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${pkgarch}" - done -done diff --git a/db-sync b/db-sync deleted file mode 100755 index b26e917..0000000 --- a/db-sync +++ /dev/null @@ -1,212 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics - -# Run as `V=true db-sync` to get verbose output -VERBOSE=${V} -extra=() -${VERBOSE} && extra+=(-v) - -WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT - -# Returns contents of a repo -get_repos() { - # Exclude everything but db files - rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" -} - -get_repo_content() { - # Return all contents - bsdtar tf "${1}" | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${WORKDIR}/${1}/os/${2}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "%d packages in blacklist" ${#blacklist[@]} - - test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in "${ARCHREPOS[@]}"; do - for _arch in "${ARCHARCHES[@]}"; do - msg "Processing %s-%s" "${_repo}-${_arch}" - - db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} - files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - # Get db contents - db=($(get_repo_content "${db_file}")) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - # IMPORTANT: the . in the sed command is needed because an empty - # whitelist would consist of a single * allowing any package to - # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" - - msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync "${extra[@]}" --no-motd -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from="/tmp/${_repo}-${_arch}.whitelist" \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - "${WORKDIR}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists, check for single *s just in case - cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist - - msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local pkgpool - for pkgpool in "${ARCHPKGPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ - "${FTP_BASE}/${pkgpool}/" - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local srcpool - for srcpool in "${ARCHSRCPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ - "${FTP_BASE}/${srcpool}/" - done - - # Cleanup - unset blacklist whitelists _arch _repo repo_file -} - -fatal_error() { - local mesg=$1; shift - error "$mesg" "$@" - exit 1 -} - -trap_exit() { - local signal=$1; shift - echo - error "$@" - trap -- "$signal" - kill "-$signal" "$$" -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-sync.conf" -source "$(librelib messages)" - -# Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do - test -z "${!var}" && fatal_error "Empty %s" "${var}" -done - -# From makepkg -set -E -for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" -done -trap 'trap_exit INT "Aborted by user! Exiting..."' INT -trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR - -init diff --git a/db-sync.conf b/db-sync.conf deleted file mode 100644 index 3d6c709..0000000 --- a/db-sync.conf +++ /dev/null @@ -1,7 +0,0 @@ -#mirror="mirrors.uk2.net" -mirror="mirrors.kernel.org" -#mirror="mirror.umd.edu" -#mirror="archlinux.c3sl.ufpr.br" -#mirror="mirror.us.leaseweb.net" -#mirror="mirror.de.leaseweb.net" -mirrorpath="archlinux" -- cgit v1.2.3 From c8c524bd20f3fc9e65af55b17e1bb2a8548a574a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 20:29:42 -0400 Subject: clean up cron-jobs/sourceballs* --- cron-jobs/sourceballs | 7 +++--- cron-jobs/sourceballs2 | 62 -------------------------------------------------- 2 files changed, 3 insertions(+), 66 deletions(-) delete mode 100755 cron-jobs/sourceballs2 diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index c12a128..3722449 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -63,11 +63,10 @@ for repo in "${PKGREPOS[@]}"; do if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi - # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package -# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then -# continue -# fi + if ! ([[ ${#ALLOWED_LICENSES[@]} -eq 0 ]] || chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + continue + fi # Store the expected file name of the source package echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 deleted file mode 100755 index e935f86..0000000 --- a/cron-jobs/sourceballs2 +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Steps -# Traverse ABSLibre -# Makepkg --allsource every package -# Remove the old sourceballs - -dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../config" -. "${dirname}/../db-functions" -. "${MAKEPKGCONF}" - -pushd "${WORKDIR}" >/dev/null - -script_lock - -# Adjust the nice level to run at a lower priority -renice +10 -p $$ > /dev/null - -# Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" - -pushd "${SVNREPO}" >/dev/null - -for repo in "${PKGREPOS[@]}"; do - msg "Sourceballing [%s]" "${repo}" - - pushd "$repo" >/dev/null - find -maxdepth 1 -type d | while read pkg; do - pushd "${SVNREPO}/$repo/$pkg" >/dev/null - - [[ ! -e ./PKGBUILD ]] && { - warning "%s is not a package" "$repo/$pkg" - continue - } - - # Unset the previous data - unset pkgbase pkgname pkgver pkgrel - source PKGBUILD - - unset build package url pkgdesc source md5sums depends makedepends \ - optdepends license arch options check mksource - - for _pkg in "${pkgname[@]}"; do - unset "package_${_pkg}" >/dev/null 2>&1 - done - - pkgbase=${pkgbase:-$pkgname} - srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" - - echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs" - - # Skip already sourceballed - [ -e "${SRCPKGDEST}/${srcfile}" ] && continue - - makepkg --allsource --ignorearch -c >/dev/null 2>&1 - - [ $? -ne 0 ] && plain '%s' "${srcfile}" - - done # end find pkgs - popd >/dev/null - -done # end repos -- cgit v1.2.3 From b82fbfc8761fda2a682e4037d08cf1a9730eebf8 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 20:31:31 -0400 Subject: rm cron-jobs/repo-sanity-check This was the last program using SVNREPO. Also, I want to do better than this. --- config | 1 - cron-jobs/repo-sanity-check | 54 --------------------------------------------- 2 files changed, 55 deletions(-) delete mode 100755 cron-jobs/repo-sanity-check diff --git a/config b/config index 87fbf2d..77df26f 100644 --- a/config +++ b/config @@ -1,6 +1,5 @@ #!/bin/bash # as a hint to text editors FTP_BASE="/srv/http/repo/public" -SVNREPO="/var/abs" TESTING_REPO='testing' PKGREPOS=( diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check deleted file mode 100755 index bd89240..0000000 --- a/cron-jobs/repo-sanity-check +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# Solves issue165... on the old flyspray install. I have no idea what issue that was. - -. "$(dirname "$(readlink -e "$0")")/../config" -. "$(dirname "$(readlink -e "$0")")/../db-functions" - -# Traverse all repos -for _repo in "${PKGREPOS[@]}"; do - msg "Cleaning up [%s]" "${_repo}" - - # Find all pkgnames on this repo's abs - on_abs=($( - find "${SVNREPO}/${_repo}" -name PKGBUILD | \ - while read pkgbuild; do - source "${pkgbuild}" >/dev/null 2>&1 - # cleanup to save memory - unset build package source md5sums pkgdesc pkgver pkgrel epoch \ - url license arch depends makedepends optdepends options \ - >/dev/null 2>&1 - - # also cleanup package functions - for _pkg in "${pkgname[@]}"; do - unset "package_${pkg}" >/dev/null 2>&1 - done - - # this fills the on_abs array - echo "${pkgname[@]}" - done - )) - - # quit if abs is empty - if [ ${#on_abs[*]} -eq 0 ]; then - warning "[%s]'s ABS tree is empty, skipping" "${_repo}" - break - fi - - # Find all pkgnames on repos - on_repo=($( - find "${FTP_BASE}/${_repo}" -name "*.pkg.tar.?z" \ - -printf "%f\n" | sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" - )) - - # Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 \ - <(printf '%s\n' "${on_abs[@]}" | sort -u) \ - <(printf '%s\n' "${on_repo[@]}" | sort -u) )) - - # Remove them from databases, ftpdir-cleanup will take care of the rest - find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \ - -exec repo-remove {} "${remove[@]}" \; >/dev/null 2>&1 - - msg2 "Removed the following packages:" - plain '%s' "${remove[@]}" -done -- cgit v1.2.3 From 4b9ab7834a9f0bff173a40932f681ac149049408 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 20:55:24 -0400 Subject: sourceballs: fix quoted array --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 3722449..b657813 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -64,7 +64,7 @@ for repo in "${PKGREPOS[@]}"; do continue fi # Check if the license or .force file does not enforce creating a source package - if ! ([[ ${#ALLOWED_LICENSES[@]} -eq 0 ]] || chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + if ! ([[ ${#ALLOWED_LICENSES[@]} -eq 0 ]] || chk_license "${pkglicense[@]}" || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then continue fi # Store the expected file name of the source package -- cgit v1.2.3 From f6b561c1ef3fd1ec0c0abba0e426ef769e14d0a0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 20:58:51 -0400 Subject: clean up config --- config | 50 ++++++++++++++++++++++++++++++++---------------- cron-jobs/devlist-mailer | 5 ++--- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/config b/config index 77df26f..fff797d 100644 --- a/config +++ b/config @@ -1,7 +1,6 @@ -#!/bin/bash # as a hint to text editors -FTP_BASE="/srv/http/repo/public" +#!/bin/bash # non-executable, but put this there as a hint to text editors -TESTING_REPO='testing' +FTP_BASE="/srv/http/repo/public" PKGREPOS=( # Arch {core,extra,testing,staging} @@ -11,24 +10,29 @@ PKGREPOS=( libre{,-testing} libre-multilib{,-testing} pcr kernels cross java nonprism - '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') + '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' ) PKGPOOL='pool/parabola' SRCPOOL='sources/parabola' +TESTING_REPO='testing' +STABLE_REPOS=( + # Arch + {core,extra} + {community,multilib} + # Parabola + libre + libre-multilib + pcr kernels cross java nonprism + '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' +) -# Directories where packages are shared between repos -# *relative to FTP_BASE* -PKGPOOLS=(pool/{packages,community,parabola}) -# Directories where sources are stored -SRCPOOLS=(sources/{packages,community,parabola}) - -CLEANUP_DESTDIR="$FTP_BASE/old/packages" +CLEANUP_DESTDIR="/srv/http/repo/private/cleanup-destdir" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" -SOURCE_CLEANUP_DRYRUN=true +SOURCE_CLEANUP_DESTDIR="/srv/http/repo/private/source-cleanup-destdir" +SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages SOURCE_CLEANUP_KEEP=30 @@ -38,12 +42,24 @@ LOCK_DELAY=10 LOCK_TIMEOUT=300 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" -TMPDIR="/tmp" +export TMPDIR="${TMPDIR:-/tmp}" ARCHES=(i686 x86_64 mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.xz" +PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" -# Used by cron-jobs/sourceballs2 -MAKEPKGCONF="~/.makepkg.conf" +# Allowed licenses: get sourceballs only for licenses in this array +# Empty (commented out) to get sourceballs for all packages +#ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') + +# For db-cleanup +PKGPOOLS=(pool/{packages,community,parabola}) +SRCPOOLS=(sources/{packages,community,parabola}) + +# Where to send error emails, and who they are from +LIST="dev@lists.parabolagnulinux.org" +FROM="dbscripts+$(whoami)@$(hostname -f)" + +# Override default config with config.local +[ -f "$(dirname "${BASH_SOURCE[0]}")/config.local" ] && . "$(dirname "${BASH_SOURCE[0]}")/config.local" diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 1a05521..cede4fc 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -2,9 +2,8 @@ #Dummy helper to send email to arch-dev # It does nothing if no output -LIST="arch-dev-public@archlinux.org" -#LIST="aaronmgriffin@gmail.com" -FROM="repomaint@archlinux.org" +# Load $LIST and $FROM from the config file +source "$(dirname "$(readlink -e "$0")")/../config" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if [ $# -ge 1 ]; then -- cgit v1.2.3 From d433fbd1da44acdf3dd50abce428fe57912f9fa7 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 21:01:23 -0400 Subject: remove db-cleanup in favor of ftpdir-cleanup --- config | 4 ---- db-cleanup | 69 -------------------------------------------------------------- 2 files changed, 73 deletions(-) delete mode 100755 db-cleanup diff --git a/config b/config index fff797d..8b4b8ab 100644 --- a/config +++ b/config @@ -53,10 +53,6 @@ SRCEXT=".src.tar.gz" # Empty (commented out) to get sourceballs for all packages #ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') -# For db-cleanup -PKGPOOLS=(pool/{packages,community,parabola}) -SRCPOOLS=(sources/{packages,community,parabola}) - # Where to send error emails, and who they are from LIST="dev@lists.parabolagnulinux.org" FROM="dbscripts+$(whoami)@$(hostname -f)" diff --git a/db-cleanup b/db-cleanup deleted file mode 100755 index 54c1ba8..0000000 --- a/db-cleanup +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Syncs pools against themselves using database contents as filter to cleanup -# them up -# License: GPLv3 - -# Principles -# * Get repos dbs contents -# * Make them a include list -# * Rsync pools against themselves removing excluded files -# * Instant cleanup! - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(librelib messages)" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -EXTRAFLAGS=() -"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) - -filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") -trap "rm -f -- $(printf %q "$filter")" EXIT - -for _repo in "${PKGREPOS[@]}"; do - for _arch in "${ARCHES[@]}"; do - msg "Getting %s-%s database" "${_repo}" "${_arch}" - - dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" - - if [ ! -r "${dbfile}" ]; then - warning "Not found" - continue - fi - - # Echo the contents into a filter file - bsdtar tf "${dbfile}" | \ - cut -d'/' -f1 | \ - sort -u | \ - sed "s|$|*|" >> "$filter" - - done -done - -msg "Removing old files:" - -for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do - msg2 '%s' "${POOL}" - - rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ - --include-from="$filter" \ - --exclude="*" \ - "${FTP_BASE}/${POOL}/" \ - "${FTP_BASE}/${POOL}/" -done - -msg "Removing dead symlinks:" -actions=(-print) -"${CLEANUP_DRYRUN}" || actions+=(-delete) -find -L "${FTP_BASE}/" -type l "${actions[@]}" -- cgit v1.2.3 From aae3b0a2490d65832547f29327f9e8828442a48d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 21:33:20 -0400 Subject: cleanup: - move some config into config.local.* - `mv createrepos db-init` - `mv {,db-}check-package-libraries.py` - `mv `list_nonfree_in_db.py db-list-nonfree.py` - `rm abslibre`: To be replaced by XBS+update abs tarballs script - `rm create-repo mkrepo`: Both did the same thing, just fancy `mkdir` - `rm `testing2x`: Just use db-move. --- abslibre | 124 --------------------------- check-package-libraries.py | 193 ------------------------------------------ config | 45 ++++------ config.local.import-community | 3 + config.local.import-packages | 3 + config.local.parabola | 8 ++ create-repo | 21 ----- createrepos | 8 -- db-check-package-libraries.py | 193 ++++++++++++++++++++++++++++++++++++++++++ db-init | 6 ++ db-list-nonfree.py | 28 ++++++ list_nonfree_in_db.py | 28 ------ mkrepo | 15 ---- testing2x | 59 ------------- 14 files changed, 257 insertions(+), 477 deletions(-) delete mode 100755 abslibre delete mode 100755 check-package-libraries.py create mode 100644 config.local.import-community create mode 100644 config.local.import-packages create mode 100644 config.local.parabola delete mode 100755 create-repo delete mode 100755 createrepos create mode 100755 db-check-package-libraries.py create mode 100755 db-init create mode 100755 db-list-nonfree.py delete mode 100755 list_nonfree_in_db.py delete mode 100755 mkrepo delete mode 100755 testing2x diff --git a/abslibre b/abslibre deleted file mode 100755 index 0b3e371..0000000 --- a/abslibre +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -ABSLIBRE=/srv/abslibre -ABSGIT=/srv/git/repositories/abslibre.git -# Remote -# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git -BLACKLIST='https://projects.parabolagnulinux.org/blacklist.git/plain/blacklist.txt' -SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' -BLFILE=/tmp/blacklist.txt - -# Variables from abs.conf -ABSROOT="/srv/abs/" -# DON'T CHANGE. WE NEED IT FOR ABSLIBRE -SYNCSERVER="rsync.archlinux.org" -ARCH="i686" -MIRRORLIST="/etc/pacman.d/mirrorlist" -REPOS=(core extra community testing community-testing !staging !community-staging) - -# Steps -# * Sync abs -# * Download blacklist.txt -# * Sync abslibre from abs excluding from blacklist -# * Create repo.abs.tar.gz tarballs - -function sync_abs() { - for ARCH in any i686 x86_64; do - rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? - done -} - -function get_blacklist() { - printf ":: Updating blacklist...\t" - wget -q -O - "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ - sed "s/^/**\//" > ${BLFILE} || { - printf "[FAILED]\n" - return 1 - } - - # Prevent using an empty blacklist - [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 - - printf "[OK]\n" -} - -function sync_abs_libre() { - - # Clone ABSLibre git repo - if [ -d /var/tmp/abslibre/.git ]; then - pushd /var/tmp/abslibre >/dev/null 2>&1 - git pull - popd >/dev/null 2>&1 - else - git clone /srv/git/abslibre.git /var/tmp/abslibre - fi - - # Sync from ABS and then sync from ABSLibre - printf ":: Syncing ABSLibre...\t" - (rsync ${SYNCARGS} --delete-excluded \ - --exclude-from=${BLFILE} \ - ${ABSROOT} \ - ${ABSLIBRE} \ - && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { - printf "[FAILED]\n" - return 1 - } - - printf "[OK]\n" -} - -# This part is very hacky and particular to the current setup :P -sync_pre_mips64el() { - pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null - - sudo -u fauno sh -c " - rsync ${SYNCARGS} \ - --exclude=.git* \ - --exclude=community-staging \ - --exclude=community-testing \ - --exclude=gnome-unstable \ - --exclude=kde-unstable \ - --exclude=multilib \ - --exclude=multilib-testing \ - --exclude=multilib-staging \ - --exclude=staging \ - --exclude=testing \ - ${ABSLIBRE}/x86_64/ \ - /home/fauno/Repos/abslibre-pre-mips64el/ && - git add . && - git commit -m \"$(date)\" -a - git push origin master - git gc - " -} - -# Create .abs.tar.gz tarballs -create_tarballs() { - for repo in ${ABSLIBRE}/{i686,x86_64}/*; do - baserepo=${repo##*/} - arch=$(basename $(dirname $repo)) - - # Remove the old one - mkdir -p /srv/http/web/media/abs/$baserepo/os/$arch/ - rm /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz - # Create a new one joining arch and any - # Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czvf /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz \ - -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ - $repo/* ${ABSLIBRE}/any/${baserepo}/* - - done -} - -sync_abs || exit 1 -get_blacklist || exit 1 -sync_abs_libre || exit 1 -# This is being done at repo server now -sync_pre_mips64el || exit 1 -#create_tarballs || exit 1 - -echo "Exclusion list used" -cat ${BLFILE} - -exit 0 diff --git a/check-package-libraries.py b/check-package-libraries.py deleted file mode 100755 index bc2349b..0000000 --- a/check-package-libraries.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2012 Michał Masłowski -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Check which libraries are provided or required by a package, store -this in a database, update and list broken packages. - -Dependencies: - -- Python 3.2 or later with SQLite 3 support - -- ``bsdtar`` - -- ``readelf`` -""" - - -import os.path -import re -import sqlite3 -import subprocess -import tempfile - - -#: Regexp matching an interesting dynamic entry. -_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" - "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") - - -def make_db(path): - """Make a new, empty, library database at *path*.""" - con = sqlite3.connect(path) - con.executescript(""" -create table provided( - library varchar not null, - package varchar not null -); -create table used( - library varchar not null, - package varchar not null -); -""") - con.close() - - -def begin(database): - """Connect to *database* and start a transaction.""" - con = sqlite3.connect(database) - con.execute("begin exclusive") - return con - - -def add_provided(con, package, libraries): - """Write that *package* provides *libraries*.""" - for library in libraries: - con.execute("insert into provided (package, library) values (?,?)", - (package, library)) - - -def add_used(con, package, libraries): - """Write that *package* uses *libraries*.""" - for library in libraries: - con.execute("insert into used (package, library) values (?,?)", - (package, library)) - - -def remove_package(con, package): - """Remove all entries for a package.""" - con.execute("delete from provided where package=?", (package,)) - con.execute("delete from used where package=?", (package,)) - - -def add_package(con, package): - """Add entries from a named *package*.""" - # Extract to a temporary directory. This could be done more - # efficiently, since there is no need to store more than one file - # at once. - with tempfile.TemporaryDirectory() as temp: - tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) - tar.communicate() - with open(os.path.join(temp, ".PKGINFO")) as pkginfo: - for line in pkginfo: - if line.startswith("pkgname ="): - pkgname = line[len("pkgname ="):].strip() - break - # Don't list previously removed libraries. - remove_package(con, pkgname) - provided = set() - used = set() - # Search for ELFs. - for dirname, dirnames, filenames in os.walk(temp): - assert dirnames is not None # unused, avoid pylint warning - for file_name in filenames: - path = os.path.join(dirname, file_name) - with open(path, "rb") as file_object: - if file_object.read(4) != b"\177ELF": - continue - readelf = subprocess.Popen(("readelf", "-d", path), - stdout=subprocess.PIPE) - for line in readelf.communicate()[0].split(b"\n"): - match = _DYNAMIC.match(line.decode("ascii")) - if match: - if match.group(1) == "SONAME": - provided.add(match.group(2)) - elif match.group(1) == "NEEDED": - used.add(match.group(2)) - else: - raise AssertionError("unknown entry type " - + match.group(1)) - add_provided(con, pkgname, provided) - add_used(con, pkgname, used) - - -def init(arguments): - """Initialize.""" - make_db(arguments.database) - - -def add(arguments): - """Add packages.""" - con = begin(arguments.database) - for package in arguments.packages: - add_package(con, package) - con.commit() - con.close() - - -def remove(arguments): - """Remove packages.""" - con = begin(arguments.database) - for package in arguments.packages: - remove_package(con, package) - con.commit() - con.close() - - -def check(arguments): - """List broken packages.""" - con = begin(arguments.database) - available = set(row[0] for row - in con.execute("select library from provided")) - for package, library in con.execute("select package, library from used"): - if library not in available: - print(package, "needs", library) - con.close() - - -def main(): - """Get arguments and run the command.""" - from argparse import ArgumentParser - parser = ArgumentParser(prog="check-package-libraries.py", - description="Check packages for " - "provided/needed libraries") - parser.add_argument("-d", "--database", type=str, - help="Database file to use", - default="package-libraries.sqlite") - subparsers = parser.add_subparsers() - subparser = subparsers.add_parser(name="init", - help="initialize the database") - subparser.set_defaults(command=init) - subparser = subparsers.add_parser(name="add", - help="add packages to database") - subparser.add_argument("packages", nargs="+", type=str, - help="package files to add") - subparser.set_defaults(command=add) - subparser = subparsers.add_parser(name="remove", - help="remove packages from database") - subparser.add_argument("packages", nargs="+", type=str, - help="package names to remove") - subparser.set_defaults(command=remove) - subparser = subparsers.add_parser(name="check", - help="list broken packages") - subparser.set_defaults(command=check) - arguments = parser.parse_args() - arguments.command(arguments) - - -if __name__ == "__main__": - main() diff --git a/config b/config index 8b4b8ab..51d89ea 100644 --- a/config +++ b/config @@ -1,40 +1,25 @@ #!/bin/bash # non-executable, but put this there as a hint to text editors +case "$USER" in + db-import-packages) _name=import-packages;; + db-import-community) _name=import-community;; + *) _name=parabola;; +esac + FTP_BASE="/srv/http/repo/public" -PKGREPOS=( - # Arch - {core,extra,testing,staging} - {gnome,kde}-unstable - {community,multilib}{,-testing,-staging} - # Parabola - libre{,-testing} - libre-multilib{,-testing} - pcr kernels cross java nonprism - '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' -) -PKGPOOL='pool/parabola' -SRCPOOL='sources/parabola' -TESTING_REPO='testing' -STABLE_REPOS=( - # Arch - {core,extra} - {community,multilib} - # Parabola - libre - libre-multilib - pcr kernels cross java nonprism - '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' -) - -CLEANUP_DESTDIR="/srv/http/repo/private/cleanup-destdir" +PKGREPOS=() +PKGPOOL='' +SRCPOOL='' + +CLEANUP_DESTDIR="/srv/http/repo/private/${_name}/package-cleanup" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/http/repo/private/source-cleanup-destdir" +SOURCE_CLEANUP_DESTDIR="/srv/http/repo/private/${_name}/source-cleanup" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=30 +SOURCE_CLEANUP_KEEP=14 REQUIRE_SIGNATURE=true @@ -55,7 +40,9 @@ SRCEXT=".src.tar.gz" # Where to send error emails, and who they are from LIST="dev@lists.parabolagnulinux.org" -FROM="dbscripts+$(whoami)@$(hostname -f)" +FROM="dbscripts+${_name}@$(hostname -f)" # Override default config with config.local [ -f "$(dirname "${BASH_SOURCE[0]}")/config.local" ] && . "$(dirname "${BASH_SOURCE[0]}")/config.local" +[ -f "$(dirname "${BASH_SOURCE[0]}")/config.local.${_name}" ] && . "$(dirname "${BASH_SOURCE[0]}")/config.local.${_name}" +unset _name diff --git a/config.local.import-community b/config.local.import-community new file mode 100644 index 0000000..63f4d2f --- /dev/null +++ b/config.local.import-community @@ -0,0 +1,3 @@ +PKGREPOS=({community,multilib}{,-testing,-staging}) +PKGPOOL='pool/community' +SRCPOOL='sources/community' diff --git a/config.local.import-packages b/config.local.import-packages new file mode 100644 index 0000000..432b3ab --- /dev/null +++ b/config.local.import-packages @@ -0,0 +1,3 @@ +PKGREPOS=('core' 'extra' 'testing' 'staging' {kde,gnome}-unstable) +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' diff --git a/config.local.parabola b/config.local.parabola new file mode 100644 index 0000000..2c52977 --- /dev/null +++ b/config.local.parabola @@ -0,0 +1,8 @@ +PKGREPOS=( + libre{,-testing} + libre-multilib{,-testing} + pcr kernels cross java nonprism + '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' +) +PKGPOOL='pool/parabola' +SRCPOOL='sources/parabola' diff --git a/create-repo b/create-repo deleted file mode 100755 index 3feb098..0000000 --- a/create-repo +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Creates repository structure - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -eq 0 ]; then - msg "Usage: %s repo1 [repo2 ... repoX]" "${0##*/}" - exit 1 -fi - -msg "Creating repos..." -for _repo in "$@"; do - msg2 "Creating [%s]" "${_repo}" - for _arch in "${ARCHES[@]}"; do - mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ - error "Failed creating %s dir" "${_arch}" - done -done - -msg "Don't forget to add them to the PKGREPOS array on %s" "$(dirname "$(readlink -e "$0")")/config" diff --git a/createrepos b/createrepos deleted file mode 100755 index 8da2455..0000000 --- a/createrepos +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Creates the repo structure defined in config - -source "$(dirname "$(readlink -e "$0")")/config" - -mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" - -"$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/db-check-package-libraries.py b/db-check-package-libraries.py new file mode 100755 index 0000000..bc2349b --- /dev/null +++ b/db-check-package-libraries.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Check which libraries are provided or required by a package, store +this in a database, update and list broken packages. + +Dependencies: + +- Python 3.2 or later with SQLite 3 support + +- ``bsdtar`` + +- ``readelf`` +""" + + +import os.path +import re +import sqlite3 +import subprocess +import tempfile + + +#: Regexp matching an interesting dynamic entry. +_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" + "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") + + +def make_db(path): + """Make a new, empty, library database at *path*.""" + con = sqlite3.connect(path) + con.executescript(""" +create table provided( + library varchar not null, + package varchar not null +); +create table used( + library varchar not null, + package varchar not null +); +""") + con.close() + + +def begin(database): + """Connect to *database* and start a transaction.""" + con = sqlite3.connect(database) + con.execute("begin exclusive") + return con + + +def add_provided(con, package, libraries): + """Write that *package* provides *libraries*.""" + for library in libraries: + con.execute("insert into provided (package, library) values (?,?)", + (package, library)) + + +def add_used(con, package, libraries): + """Write that *package* uses *libraries*.""" + for library in libraries: + con.execute("insert into used (package, library) values (?,?)", + (package, library)) + + +def remove_package(con, package): + """Remove all entries for a package.""" + con.execute("delete from provided where package=?", (package,)) + con.execute("delete from used where package=?", (package,)) + + +def add_package(con, package): + """Add entries from a named *package*.""" + # Extract to a temporary directory. This could be done more + # efficiently, since there is no need to store more than one file + # at once. + with tempfile.TemporaryDirectory() as temp: + tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) + tar.communicate() + with open(os.path.join(temp, ".PKGINFO")) as pkginfo: + for line in pkginfo: + if line.startswith("pkgname ="): + pkgname = line[len("pkgname ="):].strip() + break + # Don't list previously removed libraries. + remove_package(con, pkgname) + provided = set() + used = set() + # Search for ELFs. + for dirname, dirnames, filenames in os.walk(temp): + assert dirnames is not None # unused, avoid pylint warning + for file_name in filenames: + path = os.path.join(dirname, file_name) + with open(path, "rb") as file_object: + if file_object.read(4) != b"\177ELF": + continue + readelf = subprocess.Popen(("readelf", "-d", path), + stdout=subprocess.PIPE) + for line in readelf.communicate()[0].split(b"\n"): + match = _DYNAMIC.match(line.decode("ascii")) + if match: + if match.group(1) == "SONAME": + provided.add(match.group(2)) + elif match.group(1) == "NEEDED": + used.add(match.group(2)) + else: + raise AssertionError("unknown entry type " + + match.group(1)) + add_provided(con, pkgname, provided) + add_used(con, pkgname, used) + + +def init(arguments): + """Initialize.""" + make_db(arguments.database) + + +def add(arguments): + """Add packages.""" + con = begin(arguments.database) + for package in arguments.packages: + add_package(con, package) + con.commit() + con.close() + + +def remove(arguments): + """Remove packages.""" + con = begin(arguments.database) + for package in arguments.packages: + remove_package(con, package) + con.commit() + con.close() + + +def check(arguments): + """List broken packages.""" + con = begin(arguments.database) + available = set(row[0] for row + in con.execute("select library from provided")) + for package, library in con.execute("select package, library from used"): + if library not in available: + print(package, "needs", library) + con.close() + + +def main(): + """Get arguments and run the command.""" + from argparse import ArgumentParser + parser = ArgumentParser(prog="check-package-libraries.py", + description="Check packages for " + "provided/needed libraries") + parser.add_argument("-d", "--database", type=str, + help="Database file to use", + default="package-libraries.sqlite") + subparsers = parser.add_subparsers() + subparser = subparsers.add_parser(name="init", + help="initialize the database") + subparser.set_defaults(command=init) + subparser = subparsers.add_parser(name="add", + help="add packages to database") + subparser.add_argument("packages", nargs="+", type=str, + help="package files to add") + subparser.set_defaults(command=add) + subparser = subparsers.add_parser(name="remove", + help="remove packages from database") + subparser.add_argument("packages", nargs="+", type=str, + help="package names to remove") + subparser.set_defaults(command=remove) + subparser = subparsers.add_parser(name="check", + help="list broken packages") + subparser.set_defaults(command=check) + arguments = parser.parse_args() + arguments.command(arguments) + + +if __name__ == "__main__": + main() diff --git a/db-init b/db-init new file mode 100755 index 0000000..e25dbff --- /dev/null +++ b/db-init @@ -0,0 +1,6 @@ +#!/bin/bash +# Creates the repo structure defined in config + +source "$(dirname "$(readlink -e "$0")")/config" + +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" diff --git a/db-list-nonfree.py b/db-list-nonfree.py new file mode 100755 index 0000000..4e1b164 --- /dev/null +++ b/db-list-nonfree.py @@ -0,0 +1,28 @@ +#! /usr/bin/python2 +#-*- encoding: utf-8 -*- +from filter import * +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="nonfree_in_db", + description="Cleans nonfree files on repo",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean", + required=True,) + + args=parser.parse_args() + + if not (args.blacklist_file and args.database): + parser.print_help() + exit(1) + + blacklist=listado(args.blacklist_file) + pkgs=get_pkginfo_from_db(args.database) + + print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) diff --git a/list_nonfree_in_db.py b/list_nonfree_in_db.py deleted file mode 100755 index 4e1b164..0000000 --- a/list_nonfree_in_db.py +++ /dev/null @@ -1,28 +0,0 @@ -#! /usr/bin/python2 -#-*- encoding: utf-8 -*- -from filter import * -import argparse - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="nonfree_in_db", - description="Cleans nonfree files on repo",) - - parser.add_argument("-k", "--blacklist-file", type=str, - help="File containing blacklisted names", - required=True,) - - parser.add_argument("-b", "--database", type=str, - help="dabatase to clean", - required=True,) - - args=parser.parse_args() - - if not (args.blacklist_file and args.database): - parser.print_help() - exit(1) - - blacklist=listado(args.blacklist_file) - pkgs=get_pkginfo_from_db(args.database) - - print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) diff --git a/mkrepo b/mkrepo deleted file mode 100755 index b11dc0b..0000000 --- a/mkrepo +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Author: Nicolás Reynolds -# License: GPLv3+ -# Description: A script to quickly create new [repos] - -source "$(dirname "$(readlink -e "$0")")/config" - -for repo in "$@"; do - echo ":: Creating [$repo]" - for arch in "${ARCHES[@]}"; do - mkdir -pv "${FTP_BASE}/${repo}/os/${arch}" - done -done - -echo ":: All done. Add the repo to the ParabolaWeb admin page." diff --git a/testing2x b/testing2x deleted file mode 100755 index b76438b..0000000 --- a/testing2x +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -lt 1 ]; then - msg "usage: %s ..." "${0##*/}" - exit 1 -fi - -# Lock everything to reduce possibility of interfering task between the different repo-updates -script_lock -for repo in "${TESTING_REPO}" "${STABLE_REPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${pkgarch}" || exit 1 - done -done - -declare -A pkgs - -for pkgbase in "$@"; do - if [ ! -d "${WORKDIR}/${pkgbase}" ]; then - found_source=false - for pkgarch in "${ARCHES[@]}" 'any'; do - xbsrepo_from="$(xbs releasepath "${pkgbase}" "${TESTING_REPO}" "${pkgarch}")" - if [ -r "${xbsrepo_from}/PKGBUILD" ]; then - found_source=true - break - fi - done - "${found_source}" || die "%s not found in [%s]" "${pkgbase}" "${TESTING_REPO}" - found_target=false - for pkgarch in "${ARCHES[@]}" 'any'; do - for repo in "${STABLE_REPOS[@]}"; do - xbsrepo_to="$(xbs releasepath "${pkgbase}" "${repo}" "${pkgarch}")" - if [ -r "${xbsrepo_to}/PKGBUILD" ]; then - found_target=true - pkgs[${repo}]+="${pkgbase} " - break 2 - fi - done - done - "${found_target}" || die "%s not found in any of these repos: %s" "${pkgbase}" "${STABLE_REPOS[*]}" - fi -done - -for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${TESTING_REPO}" "${pkgarch}" -done -for repo in "${STABLE_REPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${pkgarch}" - done - if [ -n "${pkgs[${repo}]}" ]; then - "$(dirname "$(readlink -e "$0")")/db-move" ${TESTING_REPO} "${repo}" ${pkgs[${repo}]} - fi -done - -script_unlock -- cgit v1.2.3 From 5137df3e0e41abdcacc9f55ca40880de045fdbce Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 21:34:15 -0400 Subject: Remove useless README --- README | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 README diff --git a/README b/README deleted file mode 100644 index 9ce2d3f..0000000 --- a/README +++ /dev/null @@ -1,3 +0,0 @@ -* Script to run if there is error and for update - - - db-sync -- cgit v1.2.3 From 6b162f5156f1456541c160233b6a2145df500898 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 21:43:43 -0400 Subject: db-move: fix some quoting --- db-move | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-move b/db-move index 2c7c19e..f48ce4f 100755 --- a/db-move +++ b/db-move @@ -27,7 +27,7 @@ done # First loop is to check that all necessary files exist for pkgbase in "${args[@]:2}"; do for pkgarch in "${ARCHES[@]}" 'any'; do - xbsrepo_from="$(xbs releasepath ${pkgbase} ${repo_from} ${pkgarch})" + xbsrepo_from="$(xbs releasepath "${pkgbase}" "${repo_from}" "${pkgarch}")" if [ -r "${xbsrepo_from}/PKGBUILD" ]; then pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) if [ ${#pkgnames[@]} -lt 1 ]; then -- cgit v1.2.3 From b4ddeae83daca2e29d9b978abd50f49d2e6692ef Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 24 Jun 2014 22:43:54 -0400 Subject: foo --- cron-jobs/update-abs-tarballs | 5 ----- db-functions | 5 ++--- 2 files changed, 2 insertions(+), 8 deletions(-) delete mode 100755 cron-jobs/update-abs-tarballs diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs deleted file mode 100755 index e710f7c..0000000 --- a/cron-jobs/update-abs-tarballs +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/../config" - -rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ "${FTP_BASE}/" diff --git a/db-functions b/db-functions index 5d13022..1febb12 100644 --- a/db-functions +++ b/db-functions @@ -315,7 +315,6 @@ check_splitpkgs() { for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase "${pkgfile}")" - msg2 "Checking %s" "$_pkgbase" local _pkgname="$(getpkgname "${pkgfile}")" local _pkgarch="$(getpkgarch "${pkgfile}")" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" @@ -411,7 +410,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" \ || error '%s' "repo-add ${repo}${DBEXT} ${pkgs[*]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ || error '%s' "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" @@ -432,7 +431,7 @@ arch_repo_remove() { error "No database found at '%s'" "${dbfile}" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" \ || error '%s' "repo-remove ${dbfile} ${pkgs[*]}" /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ || error '%s' "repo-remove ${filesfile} ${pkgs[*]}" -- cgit v1.2.3 From 8e670c23359134af39b1c97173895c09948c2acb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 24 Jun 2014 22:51:14 -0400 Subject: db-update: get the list of repos the same way as upstream Arch The largest advantage of this is that it implicitly ignores directories we don't want to consider repos. --- config | 2 +- db-update | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/config b/config index 75e88cb..f4c1e15 100644 --- a/config +++ b/config @@ -47,7 +47,7 @@ OURARCHES=(mips64el) ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.xz" +PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" diff --git a/db-update b/db-update index 4d2cace..a0a3e39 100755 --- a/db-update +++ b/db-update @@ -11,11 +11,18 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)) if [ $? -ge 1 ]; then die "Could not read %s" "${STAGING}" fi +repos=() +for staging_repo in "${staging_repos[@]##*/}"; do + if in_array "${staging_repo}" "${PKGREPOS[@]}"; then + repos+=("${staging_repo}") + fi +done + # TODO: this might lock too much (architectures) for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do -- cgit v1.2.3 From e029bd5f834a5973ba3be45fd06eed911f26477b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 25 Jun 2014 02:23:04 -0300 Subject: revert changes on config * fix package extension on PKGEXT variable, because it's generating a bug (e.g. "ERROR Package... already exists in another repository") on all the packages that are being uploaded to the staging folder. --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index f4c1e15..75e88cb 100644 --- a/config +++ b/config @@ -47,7 +47,7 @@ OURARCHES=(mips64el) ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" +PKGEXT=".pkg.tar.xz" SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" -- cgit v1.2.3 From ee08c7abc25647e8ad038df03ac35144401b6091 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 29 Jun 2014 15:34:37 +0100 Subject: Added an additional tracker. --- cron-jobs/make_repo_torrents | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index 0798a5e..b5c6749 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -34,8 +34,8 @@ esac torrent_location='/srv/http/repo/public/torrents/' public_location='/srv/http/repo/public/' -# Tracker announce URL -tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon +# Tracker announce URLs, comma seperated +tracker='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' # t67.eu is run by Xylon # All mirrors go here #declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') -- cgit v1.2.3 From b85f896c2cb614ac22b11c14bd9e7994773309eb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 4 Jul 2014 22:51:46 -0400 Subject: Fix having a flexible PKGEXT --- config | 2 +- db-update | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config b/config index 75e88cb..f4c1e15 100644 --- a/config +++ b/config @@ -47,7 +47,7 @@ OURARCHES=(mips64el) ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.xz" +PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" diff --git a/db-update b/db-update index a0a3e39..3c06f63 100755 --- a/db-update +++ b/db-update @@ -3,7 +3,10 @@ . "$(dirname "$(readlink -e "$0")")/config" . "$(dirname "$(readlink -e "$0")")/db-functions" -shopt -s nullglob +if [[ $STAGING = *luke* ]]; then + set -x + PKGEXT='.pkg.tar.?z' +fi if [ $# -ge 1 ]; then warning "Calling %s with a specific repository is no longer supported" "${0##*/}" @@ -32,11 +35,11 @@ done # check if packages are valid for repo in "${repos[@]}"; do + if ! check_repo_permission "${repo}"; then + die "You don't have permission to update packages in %s" "${repo}" + fi pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then - if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then - die "You don't have permission to update packages in %s" "${repo}" - fi for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then die "Package %s is a symbolic link" "${repo}/${pkg##*/}" @@ -95,10 +98,10 @@ done cd "${STAGING}" while read -r file; do pub="${FTP_BASE}/${file}" - if [[ -f $pub ]]; then + if [[ -f "$pub" ]]; then warning "file already exists: %s" "${file}" else mkdir -p -- "${pub%/*}" mv -vn "$file" "$pub" fi -done < <(find other sources -type f) +done < <(find other sources -type f 2>/dev/null) -- cgit v1.2.3 From 5fad74e56b72810c9d7f86956236ff45654fb684 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 26 Jul 2014 20:20:33 -0400 Subject: make_repo_torrents: load the location from the common config file --- cron-jobs/make_repo_torrents | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 cron-jobs/make_repo_torrents diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents old mode 100644 new mode 100755 index b5c6749..faee191 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -31,8 +31,9 @@ esac # pacman doesn't support multiple different packages of the same name, # so it's OK to just stuff all the torrents into a single directory. -torrent_location='/srv/http/repo/public/torrents/' -public_location='/srv/http/repo/public/' +. "$(dirname "$(readlink -e "$0")")/../config" +public_location="$FTP_BASE/" +torrent_location="$FTP_BASE/torrents/" # Tracker announce URLs, comma seperated tracker='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' # t67.eu is run by Xylon -- cgit v1.2.3 From 52951d35c69766d0d196fc4cf59fb4864f46d334 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 26 Jul 2014 20:21:14 -0400 Subject: config: set FTP_BASE=~repo/public/main instead of ~repo/public This is to do fancy union stuff to make the web server happy --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index f4c1e15..51d044e 100644 --- a/config +++ b/config @@ -1,5 +1,5 @@ #!/bin/bash # as a hint to text editors -FTP_BASE="/srv/http/repo/public" +FTP_BASE="/srv/http/repo/public/main" SVNREPO="/var/abs" # Repos from Arch -- cgit v1.2.3 From b2a763cee1f67751065c39800a59c431ec872bfa Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 27 Jul 2014 11:55:01 +0100 Subject: Dividing make_repo_torrents into two scripts, one to make the actual torrent and a wrapper to find any updated packages and call the first script for each... --- cron-jobs/make_repo_torrents | 57 ++++++-------------------------------------- 1 file changed, 7 insertions(+), 50 deletions(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index faee191..c94d073 100755 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -14,7 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# This script depends on `mktorrent' +# This script finds any updated packages and makes calls +# `make_indivudual_torrent' for each of them. username=$( id -un ) @@ -35,71 +36,27 @@ esac public_location="$FTP_BASE/" torrent_location="$FTP_BASE/torrents/" -# Tracker announce URLs, comma seperated -tracker='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' # t67.eu is run by Xylon - -# All mirrors go here -#declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') - -# I'm removing all mirrors but one since pacman2pacman now re-writes -# the webseeds list to just point to the user's chosen mirror -declare -a array=('http://repo.parabolagnulinux.org/') - -# I got this function from http://mywiki.wooledge.org/BashFAQ/026 . It -# shuffles an array. Uses a global array variable. Must be compact -# (not a sparse array). The array must be called `array'. -shuffle() { - local i tmp size max rand - - # $RANDOM % (i+1) is biased because of the limited range of $RANDOM - # Compensate by using a range which is a multiple of the array size. - size=${#array[*]} - max=$(( 32768 / size * size )) - - for ((i=size-1; i>0; i--)); do - while (( (rand=$RANDOM) >= max )); do :; done - rand=$(( rand % (i+1) )) - tmp=${array[i]} array[i]=${array[rand]} array[rand]=$tmp - done -} - cd "${torrent_location}" +# Find any directories that might have packages in then find "${public_location}" -name 'os' -type 'd' | while read dir do + # Find any packages find "${dir}" -name '*\.pkg\.tar\.xz' | while read pkg do - pkg_name="${pkg##*/}" - if [[ -h "${pkg}" ]] # check if it's a symbolic link then # We get the target of the symlink pkg=$( readlink -f "${pkg}" ) fi + # If a .torrent file does not already exist for this package, we call + # `make_individual_torrent' to make it. if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] then - # We need to make a comma seperated list of webseeds (this is passed - # as a single argument to mktorrent) - webseeds='' - - # Randomize the order of the list of webseeds because I - # don't know if transmission might always use the one at - # the top otherwize. - shuffle - - for prefix in "${array[@]}" - do - webseeds+="${prefix}${pkg#${public_location}}," - done - - # There should not be a random comma at the end of the webseeds - webseeds="${webseeds%,}" - - mktorrent -a "${tracker}" "${pkg}" -w "${webseeds}" >/dev/null || - echo "Error making torrent for \"${pkg}\"" + /srv/http/repo/dbscripts/make_individual_torrent "${pkg}" fi done done -- cgit v1.2.3 From db2ea03e53a3aed1c8350811be70a31c7ea0ef53 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 27 Jul 2014 11:55:08 +0100 Subject: Added new file --- make_individual_torrent | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 make_individual_torrent diff --git a/make_individual_torrent b/make_individual_torrent new file mode 100644 index 0000000..0cfee56 --- /dev/null +++ b/make_individual_torrent @@ -0,0 +1,44 @@ +#! /bin/bash +# Copyright (C) 2014 Joseph Graham +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script is called by `make_repo_torrents' to make a torrent. It +# depends on `mktorrent'. It takes the following arg: +# $1 - path of package + +# Comma seperated list of trackers, no spaces +# t67.eu is run by Xylon, hackcoop by fauno & friends +trackers='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' + +# This mirror is put as a webseed. Which mirror we use for a webseed +# doesn't really matter since it's re-written on the client machine by +# pacman2pacman so it won't normally be used anyway. +seed_url='http://repo.parabolagnulinux.org/' + +if [[ -z "${1}" ]] +then + echo "Error. First arg must be the path of the package." + echo 1 +fi + +pkg="${1}" + +pkg_name="${pkg##*/}" + +# URL of the actual package for the webseed +webseed="${seed_url}${pkg#${public_location}}" + +mktorrent -a "${trackers}" "${pkg}" -w "${webseed}" >/dev/null || +echo "Error making torrent for \"${pkg}\"" -- cgit v1.2.3 From 6790ac43f2f79fea6adcfef2a10d3c4105574216 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 27 Jul 2014 12:07:06 +0100 Subject: fixed loads of mistakes I make --- cron-jobs/make_repo_torrents | 4 +++- make_individual_torrent | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index c94d073..a5d72a5 100755 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -46,6 +46,8 @@ do find "${dir}" -name '*\.pkg\.tar\.xz' | while read pkg do + pkg_name="${pkg##*/}" + if [[ -h "${pkg}" ]] # check if it's a symbolic link then # We get the target of the symlink @@ -56,7 +58,7 @@ do # `make_individual_torrent' to make it. if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] then - /srv/http/repo/dbscripts/make_individual_torrent "${pkg}" + /srv/http/repo/dbscripts/make_individual_torrent "${pkg}" "${public_location}" fi done done diff --git a/make_individual_torrent b/make_individual_torrent index 0cfee56..e5b7d8c 100644 --- a/make_individual_torrent +++ b/make_individual_torrent @@ -15,8 +15,9 @@ # along with this program. If not, see . # This script is called by `make_repo_torrents' to make a torrent. It -# depends on `mktorrent'. It takes the following arg: +# depends on `mktorrent'. It takes the following args: # $1 - path of package +# $2 - public location # Comma seperated list of trackers, no spaces # t67.eu is run by Xylon, hackcoop by fauno & friends @@ -33,7 +34,14 @@ then echo 1 fi +if [[ -z "${2}" ]] +then + echo "Error. Second arg must be the public location." + echo 1 +fi + pkg="${1}" +public_location="${2}" pkg_name="${pkg##*/}" -- cgit v1.2.3 From 01f85e9950540e6718e140882051dd8f3a9ba9a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Wed, 3 Sep 2014 17:02:14 -0300 Subject: add nonprism-testing repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 51d044e..92c059c 100644 --- a/config +++ b/config @@ -9,7 +9,7 @@ OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') # User repos USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos -PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'nonprism') +PROJREPOS=('nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") PKGPOOL='pool/parabola' -- cgit v1.2.3 From a264b078cde98c6ad3766d1bdf69ca331f01baa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Mon, 29 Sep 2014 02:22:28 -0300 Subject: config: set FTP_BASE=/srv/repo/main instead of /srv/http/repo/public/main --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 92c059c..544a8e9 100644 --- a/config +++ b/config @@ -1,5 +1,5 @@ #!/bin/bash # as a hint to text editors -FTP_BASE="/srv/http/repo/public/main" +FTP_BASE="/srv/repo/main" SVNREPO="/var/abs" # Repos from Arch -- cgit v1.2.3 From 5e9c23a6cc430138358c97544b1a9219d643b6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Tue, 30 Sep 2014 21:18:41 +0200 Subject: Fix make_repo_torrents for the new server. --- cron-jobs/make_repo_torrents | 3 ++- make_individual_torrent | 0 2 files changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 make_individual_torrent diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index a5d72a5..fec4698 100755 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -32,6 +32,7 @@ esac # pacman doesn't support multiple different packages of the same name, # so it's OK to just stuff all the torrents into a single directory. +script_directory="$(dirname "$(readlink -e "$0")")/.." . "$(dirname "$(readlink -e "$0")")/../config" public_location="$FTP_BASE/" torrent_location="$FTP_BASE/torrents/" @@ -58,7 +59,7 @@ do # `make_individual_torrent' to make it. if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] then - /srv/http/repo/dbscripts/make_individual_torrent "${pkg}" "${public_location}" + "$script_directory/make_individual_torrent" "${pkg}" "${public_location}" fi done done diff --git a/make_individual_torrent b/make_individual_torrent old mode 100644 new mode 100755 -- cgit v1.2.3 From 724674000a5c4ac1ce87db0b5f86b51d1d070a41 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Fri, 3 Oct 2014 22:21:44 +0100 Subject: Fixed typo in comments --- cron-jobs/make_repo_torrents | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index fec4698..fc723f1 100755 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# This script finds any updated packages and makes calls +# This script finds any updated packages and calls # `make_indivudual_torrent' for each of them. username=$( id -un ) -- cgit v1.2.3 From 2c72fef7bd097105e57e05a4a49d0eda060735ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Sat, 4 Oct 2014 21:21:49 +0200 Subject: Update lastsync after db-sync or cleanup. --- db-functions | 1 + db-sync | 2 ++ 2 files changed, 3 insertions(+) diff --git a/db-functions b/db-functions index 2eeffbb..dd8e2dc 100644 --- a/db-functions +++ b/db-functions @@ -141,6 +141,7 @@ cleanup() { if (( REPO_MODIFIED )); then date +%s > "${FTP_BASE}/lastupdate" + date -u +%s > "${FTP_BASE}/lastsync" fi [ "$1" ] && exit "$1" diff --git a/db-sync b/db-sync index 58e211d..35b6489 100755 --- a/db-sync +++ b/db-sync @@ -174,6 +174,8 @@ init() { "${FTP_BASE}/${srcpool}/" done + date -u +%s > "${FTP_BASE}/lastsync" + # Cleanup unset blacklist whitelists _arch _repo repo_file } -- cgit v1.2.3 From 548dad674dd2e2e59402981522676284eee6cee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Sun, 26 Oct 2014 20:22:07 +0100 Subject: db-list-unsigned-packages.py: support listing keys that signed the packages. --- db-list-unsigned-packages.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/db-list-unsigned-packages.py b/db-list-unsigned-packages.py index 36be93a..80cff51 100755 --- a/db-list-unsigned-packages.py +++ b/db-list-unsigned-packages.py @@ -21,23 +21,35 @@ unsigned packages in the database at standard input of repo named in the first argument and specified for architectures listed in the following arguments (usually the one of the database or any, default is to list all). + +If the --keyset argument is passed, print the key fingerprint of every +signed package. """ +import base64 +import subprocess import sys import tarfile def main(): """Do the job.""" + check_keys = False + if "--keyset" in sys.argv: + sys.argv.remove("--keyset") + check_keys = True repo = sys.argv[1] pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) + packages = [] + keys = [] with tarfile.open(fileobj=sys.stdin.buffer) as archive: for entry in archive: if entry.name.endswith("/desc"): content = archive.extractfile(entry) skip = False is_arch = False + key = None for line in content: if is_arch: is_arch = False @@ -46,12 +58,38 @@ def main(): break if line == b"%PGPSIG%\n": skip = True # signed - break + key = b"" + if check_keys: + continue + else: + break if line == b"%ARCH%\n": is_arch = True + continue + if key is not None: + if line.strip(): + key += line.strip() + else: + break + if check_keys and key: + key_binary = base64.b64decode(key) + keys.append(key_binary) + packages.append(repo + "/" + entry.name[:-5]) if skip: continue print(repo + "/" + entry.name[:-5]) + if check_keys and keys: + # We have collected all signed package names in packages and + # all keys in keys. Let's now ask gpg to list all signatures + # and find which keys made them. + packets = subprocess.check_output(("gpg", "--list-packets"), + input=b"".join(keys)) + i = 0 + for line in packets.decode("latin1").split("\n"): + if line.startswith(":signature packet:"): + keyid = line[line.index("keyid ") + len("keyid "):] + print(packages[i], keyid) + i += 1 if __name__ == "__main__": -- cgit v1.2.3 From dd63290fd4c15b6220ade6ef4d6f8f0e4878944e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 15 Apr 2015 11:40:23 -0300 Subject: add nonsystemd and nonsystemd-testing repos --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 544a8e9..fdf3ef9 100644 --- a/config +++ b/config @@ -9,7 +9,7 @@ OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') # User repos USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos -PROJREPOS=('nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') +PROJREPOS=('nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") PKGPOOL='pool/parabola' -- cgit v1.2.3 From de330518db13ca0a6bb9d0a4227f842d2a8ca941 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 May 2015 21:20:56 -0400 Subject: clean up configuration --- config | 8 ++++---- config.local.import-community | 2 ++ config.local.import-packages | 2 ++ config.local.parabola | 2 ++ db-import.conf | 20 +++++++++++++------- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/config b/config index 51d89ea..f494a53 100644 --- a/config +++ b/config @@ -1,4 +1,4 @@ -#!/bin/bash # non-executable, but put this there as a hint to text editors +#!/hint/bash case "$USER" in db-import-packages) _name=import-packages;; @@ -11,12 +11,12 @@ PKGREPOS=() PKGPOOL='' SRCPOOL='' -CLEANUP_DESTDIR="/srv/http/repo/private/${_name}/package-cleanup" +CLEANUP_DESTDIR="/srv/repo/private/${_name}/package-cleanup" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/http/repo/private/${_name}/source-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/repo/private/${_name}/source-cleanup" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages SOURCE_CLEANUP_KEEP=14 @@ -39,7 +39,7 @@ SRCEXT=".src.tar.gz" #ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Where to send error emails, and who they are from -LIST="dev@lists.parabolagnulinux.org" +LIST="dev@lists.parabola.nu" FROM="dbscripts+${_name}@$(hostname -f)" # Override default config with config.local diff --git a/config.local.import-community b/config.local.import-community index 63f4d2f..393fd57 100644 --- a/config.local.import-community +++ b/config.local.import-community @@ -1,3 +1,5 @@ +#!/hint/bash + PKGREPOS=({community,multilib}{,-testing,-staging}) PKGPOOL='pool/community' SRCPOOL='sources/community' diff --git a/config.local.import-packages b/config.local.import-packages index 432b3ab..c699b28 100644 --- a/config.local.import-packages +++ b/config.local.import-packages @@ -1,3 +1,5 @@ +#!/hint/bash + PKGREPOS=('core' 'extra' 'testing' 'staging' {kde,gnome}-unstable) PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/config.local.parabola b/config.local.parabola index b96f801..648dfbb 100644 --- a/config.local.parabola +++ b/config.local.parabola @@ -1,3 +1,5 @@ +#!/hint/bash + PKGREPOS=( # Main repos libre{,-testing} diff --git a/db-import.conf b/db-import.conf index 4330fa9..aaf7b0f 100644 --- a/db-import.conf +++ b/db-import.conf @@ -1,13 +1,19 @@ -#/bin/bash # as a hint to text editors +#!/hint/bash IMPORTDIR=/srv/repo/import -_archrepos=( - {core,extra,testing,staging}-{i686,x86_64} - {gnome,kde}-unstable-{i686,x86_64} - community{,-testing,-staging}-{i686,x86_64} - multilib{,-testing,-staging}-x86_64 -) +case "$USER" in + db-import-packages) + _archrepos=( + {core,extra,testing,staging}-{i686,x86_64} + {gnome,kde}-unstable-{i686,x86_64} + );; + db-import-community) + _archrepos=( + community{,-testing,-staging}-{i686,x86_64} + multilib{,-testing,-staging}-x86_64 + );; +esac _archpkgmirror=$(db-pick-mirror rsync https://www.archlinux.org/mirrors/status/tier/1/json/) -- cgit v1.2.3 From 1918eeec794d764ce021996ab66e923fc50195b4 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 24 May 2015 08:02:12 +0100 Subject: db-update: Remove debugging code --- db-update | 5 ----- 1 file changed, 5 deletions(-) diff --git a/db-update b/db-update index 3c06f63..0c4f824 100755 --- a/db-update +++ b/db-update @@ -3,11 +3,6 @@ . "$(dirname "$(readlink -e "$0")")/config" . "$(dirname "$(readlink -e "$0")")/db-functions" -if [[ $STAGING = *luke* ]]; then - set -x - PKGEXT='.pkg.tar.?z' -fi - if [ $# -ge 1 ]; then warning "Calling %s with a specific repository is no longer supported" "${0##*/}" exit 1 -- cgit v1.2.3 From cbefaa1b5750a1a24ac343633e9ad3ba794f2938 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 24 May 2015 08:03:06 +0100 Subject: db-sync.conf: mess with mirrors --- db-sync.conf | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/db-sync.conf b/db-sync.conf index 3d6c709..f7748c3 100644 --- a/db-sync.conf +++ b/db-sync.conf @@ -1,7 +1,11 @@ -#mirror="mirrors.uk2.net" -mirror="mirrors.kernel.org" -#mirror="mirror.umd.edu" -#mirror="archlinux.c3sl.ufpr.br" +#mirror="mirrors.kernel.org" +mirror="mirrors.niyawe.de" + +## mirrors without sources folder +#mirror="mirror.nl.leaseweb.net" +#mirror="mirror.one.com" #mirror="mirror.us.leaseweb.net" +#mirror="mirror.bytemark.co.uk" #mirror="mirror.de.leaseweb.net" + mirrorpath="archlinux" -- cgit v1.2.3 From 8a57cf2f7a1f83083dec391a2c157f552c0725a7 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 24 May 2015 08:06:42 +0100 Subject: abslibre: pass '--quiet' to rsync --- abslibre | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/abslibre b/abslibre index 0b3e371..78ec81e 100755 --- a/abslibre +++ b/abslibre @@ -5,7 +5,7 @@ ABSGIT=/srv/git/repositories/abslibre.git # Remote # ABSGIT=http://projects.parabolagnulinux.org/abslibre.git BLACKLIST='https://projects.parabolagnulinux.org/blacklist.git/plain/blacklist.txt' -SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' +SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' BLFILE=/tmp/blacklist.txt # Variables from abs.conf @@ -60,7 +60,7 @@ function sync_abs_libre() { ${ABSROOT} \ ${ABSLIBRE} \ && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { printf "[FAILED]\n" return 1 } -- cgit v1.2.3 From 0527d9f1f600387f3397ebcbaff1a6742862cf9f Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 24 May 2015 08:09:56 +0100 Subject: abslibre: use `set -e` for cleaner error handling --- abslibre | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/abslibre b/abslibre index 78ec81e..493a266 100755 --- a/abslibre +++ b/abslibre @@ -1,5 +1,7 @@ #!/bin/bash +set -e -x + ABSLIBRE=/srv/abslibre ABSGIT=/srv/git/repositories/abslibre.git # Remote @@ -111,14 +113,9 @@ create_tarballs() { done } -sync_abs || exit 1 -get_blacklist || exit 1 -sync_abs_libre || exit 1 +sync_abs +get_blacklist +sync_abs_libre # This is being done at repo server now -sync_pre_mips64el || exit 1 -#create_tarballs || exit 1 - -echo "Exclusion list used" -cat ${BLFILE} - -exit 0 +sync_pre_mips64el +#create_tarballs -- cgit v1.2.3 From 60c97dd73fa93fffaef35ce169330daf0dbc8ce4 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 24 May 2015 08:18:17 +0100 Subject: abslibre: remember these things called variables, so we don't need to chage the same thing in a dozen places? --- abslibre | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/abslibre b/abslibre index 493a266..a4af041 100755 --- a/abslibre +++ b/abslibre @@ -2,6 +2,7 @@ set -e -x +FTP_BASE=/srv/repo/main ABSLIBRE=/srv/abslibre ABSGIT=/srv/git/repositories/abslibre.git # Remote @@ -52,7 +53,7 @@ function sync_abs_libre() { git pull popd >/dev/null 2>&1 else - git clone /srv/git/abslibre.git /var/tmp/abslibre + git clone "$ABSGIT" /var/tmp/abslibre fi # Sync from ABS and then sync from ABSLibre @@ -102,11 +103,11 @@ create_tarballs() { arch=$(basename $(dirname $repo)) # Remove the old one - mkdir -p /srv/http/web/media/abs/$baserepo/os/$arch/ - rm /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz + mkdir -p $FTP_BASE/$baserepo/os/$arch/ + rm $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz # Create a new one joining arch and any # Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czvf /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + bsdtar -czvf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ $repo/* ${ABSLIBRE}/any/${baserepo}/* -- cgit v1.2.3 From 4595d68c985d05340f2e316823a34b759b043e69 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 24 May 2015 08:19:40 +0100 Subject: abslibre: misc changes, look at the diff --- abslibre | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/abslibre b/abslibre index a4af041..af57ec2 100755 --- a/abslibre +++ b/abslibre @@ -4,10 +4,10 @@ set -e -x FTP_BASE=/srv/repo/main ABSLIBRE=/srv/abslibre -ABSGIT=/srv/git/repositories/abslibre.git +ABSGIT=/srv/git/abslibre/abslibre.git # Remote # ABSGIT=http://projects.parabolagnulinux.org/abslibre.git -BLACKLIST='https://projects.parabolagnulinux.org/blacklist.git/plain/blacklist.txt' +BLACKLIST=/home/repo/blacklist/blacklist.txt SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet' BLFILE=/tmp/blacklist.txt @@ -29,11 +29,15 @@ function sync_abs() { for ARCH in any i686 x86_64; do rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? done + + # fix some permissions + find "${ABSROOT}" -type d -print0 | xargs -0 chmod 755 + find "${ABSROOT}" -type f -print0 | xargs -0 chmod 644 } function get_blacklist() { printf ":: Updating blacklist...\t" - wget -q -O - "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ + cat "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ sed "s/^/**\//" > ${BLFILE} || { printf "[FAILED]\n" return 1 @@ -68,6 +72,10 @@ function sync_abs_libre() { return 1 } + # fix some permissions + find "${ABSLIBRE}" -type d -print0 | xargs -0 chmod 755 + find "${ABSLIBRE}" -type f -print0 | xargs -0 chmod 644 + printf "[OK]\n" } @@ -104,10 +112,10 @@ create_tarballs() { # Remove the old one mkdir -p $FTP_BASE/$baserepo/os/$arch/ - rm $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz + rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz # Create a new one joining arch and any # Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czvf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ $repo/* ${ABSLIBRE}/any/${baserepo}/* @@ -117,6 +125,5 @@ create_tarballs() { sync_abs get_blacklist sync_abs_libre -# This is being done at repo server now -sync_pre_mips64el -#create_tarballs +#sync_pre_mips64el +create_tarballs -- cgit v1.2.3 From bce56345e8df88782139179cc260bbffeb47290d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 20 May 2015 20:35:07 -0600 Subject: testing2x: Remove; it has no place in Parabola. It was for moving packages from testing to [core] or [extra] (automatically picking the right one). We, of course, don't do that. --- testing2x | 61 ------------------------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100755 testing2x diff --git a/testing2x b/testing2x deleted file mode 100755 index 2ed5c25..0000000 --- a/testing2x +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -lt 1 ]; then - msg "usage: %s ..." "${0##*/}" - exit 1 -fi - -# Lock everything to reduce possibility of interfering task between the different repo-updates -script_lock -for repo in 'core' 'extra' 'testing'; do - for pkgarch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${pkgarch}" || exit 1 - done -done - -declare -A pkgs - -for pkgbase in "$@"; do - if [ ! -d "${WORKDIR}/${pkgbase}" ]; then - /usr/bin/svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null - - found_source=false - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/${pkgbase}/testing-${pkgarch}" - if [ -r "${svnrepo_from}/PKGBUILD" ]; then - found_source=true - break - fi - done - "${found_source}" || die "%s not found in [testing]" "${pkgbase}" - found_target=false - for pkgarch in "${ARCHES[@]}" 'any'; do - for repo in 'core' 'extra'; do - svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" - if [ -r "${svnrepo_to}/PKGBUILD" ]; then - found_target=true - pkgs[${repo}]+="${pkgbase} " - break 2 - fi - done - done - "${found_target}" || die "%s neither found in [core] nor [extra]" "${pkgbase}" - fi -done - -for pkgarch in "${ARCHES[@]}"; do - repo_unlock 'testing' "${pkgarch}" -done -for repo in 'core' 'extra'; do - for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${pkgarch}" - done - if [ -n "${pkgs[${repo}]}" ]; then - "$(dirname "$(readlink -e "$0")")/db-move" 'testing' "${repo}" ${pkgs[${repo}]} - fi -done - -script_unlock -- cgit v1.2.3 From 2b25f7944ef903ede14d573b2ea5e7c9d736017e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 24 May 2015 16:28:46 -0600 Subject: cron-jobs/update-abs-tarballs: Remove; everything is on the same server now. It just ran `rsync` to sync things between servers. --- cron-jobs/update-abs-tarballs | 5 ----- 1 file changed, 5 deletions(-) delete mode 100755 cron-jobs/update-abs-tarballs diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs deleted file mode 100755 index e710f7c..0000000 --- a/cron-jobs/update-abs-tarballs +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/../config" - -rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ "${FTP_BASE}/" -- cgit v1.2.3 From 70a3f4fb4d6994e73b5ad19b4218bde609150249 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 24 May 2015 16:29:07 -0600 Subject: repo-sanity-check: expand on the comment at the top. --- cron-jobs/repo-sanity-check | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index bd89240..239f042 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,5 +1,8 @@ #!/bin/bash -# Solves issue165... on the old flyspray install. I have no idea what issue that was. +# Solves issue165... on the old roundup install. From the database +# backups, the title was "Older/deprecated packages never leave the +# repo", I don't know how the body of the issue is stored in the DB, +# but the title says enough, I think. . "$(dirname "$(readlink -e "$0")")/../config" . "$(dirname "$(readlink -e "$0")")/../db-functions" -- cgit v1.2.3 From 7efc4b2b48f9ed1669a17b3cd307db0f17e193e9 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 24 May 2015 12:32:44 -0600 Subject: db-remove: remove_pkgs= should have been remove_pkgs+= Because of this mistake, it didn't properly handle specifying multiple pkgbases. --- db-remove | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-remove b/db-remove index 8fff9db..2f18ff7 100755 --- a/db-remove +++ b/db-remove @@ -31,7 +31,7 @@ for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then - remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo "${pkgname[@]}")) + remove_pkgs+=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo "${pkgname[@]}")) else warning "%s not found in ABS(libre)" "$pkgbase" warning "Removing only %s from the repo" "$pkgbase" -- cgit v1.2.3 From 0be51cb3e78592984ea8152cac176448d1765fb0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 24 May 2015 12:35:53 -0600 Subject: db-remove: pull the path ${SVNREPO}/$repo/$pkgbase into a variable. This way, if it changes, it only needs to be changed in one place. --- db-remove | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db-remove b/db-remove index 2f18ff7..53fdb2a 100755 --- a/db-remove +++ b/db-remove @@ -30,8 +30,9 @@ remove_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" - if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then - remove_pkgs+=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo "${pkgname[@]}")) + path="${SVNREPO}/$repo/$pkgbase" + if [ -d "$path" ]; then + remove_pkgs+=($(. "$path/PKGBUILD"; echo "${pkgname[@]}")) else warning "%s not found in ABS(libre)" "$pkgbase" warning "Removing only %s from the repo" "$pkgbase" -- cgit v1.2.3 From d0834f7e6bd53ea729374eab138bb38a36c73996 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 20 May 2015 20:41:17 -0600 Subject: Clean up shebangs --- config | 2 +- cron-jobs/check_archlinux/check_packages.py | 2 +- cron-jobs/make_repo_torrents | 2 +- db-functions | 2 +- list_nonfree_in_db.py | 2 +- make_individual_torrent | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config b/config index fdf3ef9..ec732c8 100644 --- a/config +++ b/config @@ -1,4 +1,4 @@ -#!/bin/bash # as a hint to text editors +#!/hint/bash FTP_BASE="/srv/repo/main" SVNREPO="/var/abs" diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index d233bf6..ac0194f 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python2 # # check_archlinux.py # diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index fc723f1..2eb0978 100755 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -1,4 +1,4 @@ -#! /bin/bash +#!/bin/bash # Copyright (C) 2014 Joseph Graham # # This program is free software: you can redistribute it and/or modify diff --git a/db-functions b/db-functions index dd8e2dc..a0b20cd 100644 --- a/db-functions +++ b/db-functions @@ -1,4 +1,4 @@ -#!/bin/bash +#!/hint/bash # Some PKGBUILDs need CARCH to be set CARCH="x86_64" diff --git a/list_nonfree_in_db.py b/list_nonfree_in_db.py index 4e1b164..a486fa5 100755 --- a/list_nonfree_in_db.py +++ b/list_nonfree_in_db.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2 +#!/usr/bin/env python2 #-*- encoding: utf-8 -*- from filter import * import argparse diff --git a/make_individual_torrent b/make_individual_torrent index e5b7d8c..0a7e778 100755 --- a/make_individual_torrent +++ b/make_individual_torrent @@ -1,4 +1,4 @@ -#! /bin/bash +#!/bin/bash # Copyright (C) 2014 Joseph Graham # # This program is free software: you can redistribute it and/or modify -- cgit v1.2.3 From 148bfaf36c04106e4ff1043dff2c5f6c3d75149e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 25 May 2015 01:08:20 -0600 Subject: Handle empty things (from Arch Linux) --- cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/sourceballs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 560689a..4063c09 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -64,7 +64,7 @@ done # get a list of all available packages in the pacakge pool find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" # create a list of packages in our db -cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" +find "${WORKDIR}" -maxdepth 1 -type f -name 'db-*' -exec cat {} \; | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if [ ${#old_pkgs[@]} -ge 1 ]; then diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b657813..b696dd4 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -117,8 +117,8 @@ for repo in "${PKGREPOS[@]}"; do done # Cleanup old source packages -cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'expected-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'available-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/available-src-pkgs.sort" old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) if [ ${#old_pkgs[@]} -ge 1 ]; then -- cgit v1.2.3 From f1fa54c8637f00587193879f13ab2060d4e7afbb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 25 May 2015 01:08:36 -0600 Subject: Remove pointless differences from Arch Linux --- cron-jobs/devlist-mailer | 2 +- cron-jobs/sourceballs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index cede4fc..7f298b9 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -3,7 +3,7 @@ # It does nothing if no output # Load $LIST and $FROM from the config file -source "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../config" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if [ $# -ge 1 ]; then diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b696dd4..c02912a 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -64,7 +64,7 @@ for repo in "${PKGREPOS[@]}"; do continue fi # Check if the license or .force file does not enforce creating a source package - if ! ([[ ${#ALLOWED_LICENSES[@]} -eq 0 ]] || chk_license "${pkglicense[@]}" || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + if ! ([[ -z ${ALLOWED_LICENSES[*]} ]] || chk_license "${pkglicense[@]}" || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then continue fi # Store the expected file name of the source package -- cgit v1.2.3 From 0b44acfc457cd954297d36851c712f0149851699 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 25 May 2015 01:11:21 -0600 Subject: db-update: Remove debug code --- db-update | 5 ----- 1 file changed, 5 deletions(-) diff --git a/db-update b/db-update index 0a46128..3b63708 100755 --- a/db-update +++ b/db-update @@ -3,11 +3,6 @@ . "$(dirname "$(readlink -e "$0")")/config" . "$(dirname "$(readlink -e "$0")")/db-functions" -if [[ $STAGING = *luke* ]]; then - set -x - PKGEXT='.pkg.tar.?z' -fi - if [ $# -ge 1 ]; then warning "Calling %s with a specific repository is no longer supported" "${0##*/}" exit 1 -- cgit v1.2.3 From af138659e5042debbb57deba5c0419ba744f75fd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 25 May 2015 01:29:34 -0600 Subject: Pull in more changes from lukeshu/archlinux+cleanup+librelib --- db-functions | 18 +++++++++++++++--- db-update | 3 +++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/db-functions b/db-functions index c1bf6e8..d0c45a4 100644 --- a/db-functions +++ b/db-functions @@ -26,15 +26,21 @@ mv_acl() { # set up general environment WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +if [ -n "${SVNUSER}" ]; then + setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" + setfacl -m d:u:"${USER}":rwx "${WORKDIR}" + setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" +fi LOCKS=() REPO_MODIFIED=0 # Used: plain, msg, msg2, warning, error, in_array, get_full_version, abort, die # Overwritten: cleanup # Ignored: stat_busy, stat_done, -# setup_workdir, trab_abort, trap_exit, +# setup_workdir, trap_abort, trap_exit, # lock, slock, lock_close -. $(librelib common) +# pkgver_equal, find_cached_package, check_root +. "$(librelib common)" script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" @@ -83,7 +89,6 @@ cleanup() { if (( REPO_MODIFIED )); then date +%s > "${FTP_BASE}/lastupdate" - date -u +%s > "${FTP_BASE}/lastsync" fi [ "$1" ] && exit "$1" @@ -225,6 +230,13 @@ getpkgarch() { echo "$_ver" } +check_packager() { + local _packager + + _packager=$(_grep_pkginfo "$1" "packager") + [[ $_packager && $_packager != 'Unknown Packager' ]] +} + getpkgfile() { if [[ ${#} -ne 1 ]]; then error 'No canonical package found!' diff --git a/db-update b/db-update index 3b63708..57a2251 100755 --- a/db-update +++ b/db-update @@ -51,6 +51,9 @@ for repo in "${repos[@]}"; do if ! check_pkgrepos "${pkg}"; then die "Package %s already exists in another repository" "${repo}/${pkg##*/}" fi + if ! check_packager "${pkg}"; then + die "Package ${repo}/${pkg##*/} does not have a valid packager" + fi done if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then die "Missing split packages for %s" "${repo}" -- cgit v1.2.3 From de1f6d3b16d6e833ec5df6e9f403aec7e2193962 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 25 May 2015 01:30:09 -0600 Subject: Use $(xbs name) instead of saying XBS. --- db-remove | 2 +- db-update | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db-remove b/db-remove index c4439a6..05b1750 100755 --- a/db-remove +++ b/db-remove @@ -34,7 +34,7 @@ for pkgbase in "${pkgbases[@]}"; do remove_pkgs+=($(. "$path/PKGBUILD"; echo "${pkgname[@]}")) xbs unrelease "$pkgbase" "$repo" "$arch" else - warning "%s not found in XBS %s" "$pkgbase" "$repo-$arch" + warning "%s not found in %s for %s" "$pkgbase" "$(xbs name)" "$repo-$arch" warning "Removing only %s from the repo" "$pkgbase" warning "If it was a split package you have to remove the others yourself!" remove_pkgs+=("$pkgbase") diff --git a/db-update b/db-update index 57a2251..9b9b7b4 100755 --- a/db-update +++ b/db-update @@ -46,7 +46,7 @@ for repo in "${repos[@]}"; do die "Package %s does not have a valid signature" "${repo}/${pkg##*/}" fi if ! check_pkgxbs "${pkg}" "${repo}"; then - die "Package %s is not consistent with XBS" "${repo}/${pkg##*/}" + die "Package %s is not consistent with %s" "${repo}/${pkg##*/}" "$(xbs name)" fi if ! check_pkgrepos "${pkg}"; then die "Package %s already exists in another repository" "${repo}/${pkg##*/}" -- cgit v1.2.3 From e960e11bfb0d74cf545595ebfabe4965e3a5b1ed Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 25 May 2015 01:52:20 -0600 Subject: Backport simple changes from the lukeshu/cleanup+xbs2+stuff branch - db-move: rename dir_to to xbsrepo_to - db-move: fix typo in comment - db-move: add a diagnostic message - db-remove: wrap a line --- db-move | 9 +++++---- db-remove | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/db-move b/db-move index f48ce4f..89a0ad6 100755 --- a/db-move +++ b/db-move @@ -64,17 +64,18 @@ for pkgbase in "${args[@]:2}"; do arches=($(xbs move "${repo_from}" "${repo_to}" "${pkgbase}")) # move the package in ftp for pkgarch in "${arches[@]}"; do - dir_to="$(xbs releasepath "$pkgbase" "$repo_to" "$pkgarch")" - if true; then # to add in indent level to make merging easier + xbsrepo_to="$(xbs releasepath "$pkgbase" "$repo_to" "$pkgarch")" + if true; then # to add an indent level to make merging easier if [ "${pkgarch}" == 'any' ]; then tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi - pkgnames=($(. "${dir_to}/PKGBUILD"; echo "${pkgname[@]}")) + msg2 "%s (%s)" "${pkgbase}" "${tarches[*]}" + pkgnames=($(. "${xbsrepo_to}/PKGBUILD"; echo "${pkgname[@]}")) for pkgname in "${pkgnames[@]}"; do - pkgver=$(. "${dir_to}/PKGBUILD"; get_full_version "${pkgname}") + pkgver=$(. "${xbsrepo_to}/PKGBUILD"; get_full_version "${pkgname}") for tarch in "${tarches[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}) pkgfile="${pkgpath##*/}" diff --git a/db-remove b/db-remove index f761f22..dcbe4b4 100755 --- a/db-remove +++ b/db-remove @@ -35,7 +35,8 @@ for pkgbase in "${pkgbases[@]}"; do remove_pkgs+=($(. "$path/PKGBUILD"; echo "${pkgname[@]}")) xbs unrelease "$pkgbase" "$repo" "$arch" else - warning "%s not found in %s for %s" "$pkgbase" "$(xbs name)" "$repo-$arch" + warning "%s not found in %s for %s" \ + "$pkgbase" "$(xbs name)" "$repo-$arch" warning "Removing only %s from the repo" "$pkgbase" warning "If it was a split package you have to remove the others yourself!" remove_pkgs+=("$pkgbase") -- cgit v1.2.3 From f338cb024ffad54b051000bba3d42fd343e363ae Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 5 Jun 2015 00:49:37 -0600 Subject: db-functions: do better printing of whitespace-separated filenames --- db-functions | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/db-functions b/db-functions index a0b20cd..deaa008 100644 --- a/db-functions +++ b/db-functions @@ -509,12 +509,13 @@ arch_repo_add() { local arch=$2 local pkgs=("${@:3}") + printf -v pkgs_str -- '%q ' "${pkgs[@]}" # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ - || error '%s' "repo-add ${repo}${DBEXT} ${pkgs[*]}" + || error 'repo-add %q %s' "${repo}${DBEXT}" "${pkgs_str% }" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ - || error '%s' "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" + || error 'repo-add -f %q %s' "${repo}${FILESEXT}" "${pkgs_str% }" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -532,10 +533,11 @@ arch_repo_remove() { error "No database found at '%s'" "${dbfile}" return 1 fi + printf -v pkgs_str -- '%q ' "${pkgs[@]}" /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ - || error '%s' "repo-remove ${dbfile} ${pkgs[*]}" + || error 'repo-remove %q %s' "${dbfile}" "${pkgs_str% }" /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ - || error '%s' "repo-remove ${filesfile} ${pkgs[*]}" + || error 'repo-remove %q %s' "${filesfile}" "${pkgs_str% }" set_repo_permission "${repo}" "${arch}" REPO_MODIFIED=1 -- cgit v1.2.3 From 17a94df0c108f09cada98535138136779b4e13a4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 24 May 2015 12:41:09 -0600 Subject: Use XBS instead of the SVN/ABS grossness. * config: drop SVNREPO * cron-jobs/sourceballs: Replace commented out SVN code and active ABS code with XBS code. The XBS code is fairly similar to the SVN code, the difference being that it uses `xbs releasepath` instead of `svn export`. * db-functions: - Rename check_pkgsvn to check_pkgxbs - check_pkgxbs: Drop the `svn export` bit, as `xbs releasepath` assumes that a working directory already exists. Replace the paths created by the `svn export` with calls to `xbs releasepath`. - check_splitpkgs: Drop the ABS `cp` -r bit, as `xbs releasepath` assumes that a working directory already exists. Replace the paths created by the `cp -r` with calls to `xbs releasepath`. Rename the variables and temporary files s/svn/xbs/ . * db-move: - First loop: Rename the variable svnrepo_from to xbsrepo_from, and get the value from `xbs releasepath`. - Second loop: Run `xbs move` before the inner loop to get a list of architectures. Rename the variable `svnrepo_from` to `xbsrepo_to`, and get the value for it from `xbs releasepath`. Because xbs guarantees that the PKGBUILD exists for the architectures listed, replace the check for whether the PKGBUILD exists with `if true`, to keep merging easy (as opposed to removing the if, and de-indenting the whole thing). * db-remove: Get the location of the PKGBUILD from `xbs releasepath`, call `xbs unrelease`, and adjust a message to use `xbs name` and mention the appropriate repo/arch pair. --- config | 2 +- cron-jobs/sourceballs | 11 +++-------- db-functions | 40 ++++++++++------------------------------ db-move | 22 ++++++++++++---------- db-remove | 6 ++++-- 5 files changed, 30 insertions(+), 51 deletions(-) diff --git a/config b/config index ec732c8..f1b09eb 100644 --- a/config +++ b/config @@ -1,6 +1,6 @@ #!/hint/bash + FTP_BASE="/srv/repo/main" -SVNREPO="/var/abs" # Repos from Arch ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index a9addc3..c12a128 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -78,15 +78,10 @@ for repo in "${PKGREPOS[@]}"; do continue fi - # Get the sources from svn + # Get the sources from xbs mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 - - # If it's on official repos, nor [libre], nor [libre-testing] - cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ - cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ - cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 + cp -a "$(xbs releasepath "${pkgbase}" "${repo}" "${pkgarch}")" \ + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue diff --git a/db-functions b/db-functions index deaa008..d76aa41 100644 --- a/db-functions +++ b/db-functions @@ -347,7 +347,7 @@ check_pkgfile() { fi } -check_pkgsvn() { +check_pkgxbs() { local pkgfile="${1}" local _pkgbase="$(getpkgbase "${pkgfile}")" [ $? -ge 1 ] && return 1 @@ -361,18 +361,11 @@ check_pkgsvn() { in_array "${repo}" "${PKGREPOS[@]}" || return 1 - if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then - mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 - fi - - local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}")" - [ "${svnver}" == "${_pkgver}" ] || return 1 + local xbsver="$(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; get_full_version "${_pkgname}")" + [ "${xbsver}" == "${_pkgver}" ] || return 1 - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) - in_array "${_pkgname}" "${svnnames[@]}" || return 1 + local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) + in_array "${_pkgname}" "${xbsnames[@]}" || return 1 return 0 } @@ -383,7 +376,7 @@ check_splitpkgs() { local pkgfiles=("${@}") local pkgfile local pkgdir - local svnname + local xbsname mkdir -p "${WORKDIR}/check_splitpkgs/" pushd "${WORKDIR}/check_splitpkgs" >/dev/null @@ -397,29 +390,16 @@ check_splitpkgs() { mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" - if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then - mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - - cp -r "${SVNREPO}/$repo/$_pkgbase/PKGBUILD" "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r "${SVNREPO}/libre/$_pkgbase/PKGBUILD" "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r "${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD" "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 - - [[ $? -ge 1 ]] && { - echo "Failed $_pkgbase-$_pkgver-$_pkgarch" - return 1 - } - fi - - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) - printf '%s\n' "${svnnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" + local xbsnames=($(. "$(xbs releasepath "${_pkgbase}" "${repo}" "${_pkgarch}")/PKGBUILD"; echo "${pkgname[@]}")) + printf '%s\n' "${xbsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" done popd >/dev/null for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do [ ! -d "${pkgdir}" ] && continue sort -u "${pkgdir}/staging" -o "${pkgdir}/staging" - sort -u "${pkgdir}/svn" -o "${pkgdir}/svn" - if [ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/svn")" ]; then + sort -u "${pkgdir}/xbs" -o "${pkgdir}/xbs" + if [ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/xbs")" ]; then return 1 fi done diff --git a/db-move b/db-move index 2ac3f4a..275a11a 100755 --- a/db-move +++ b/db-move @@ -27,14 +27,14 @@ done # First loop is to check that all necessary files exist for pkgbase in "${args[@]:2}"; do for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" - if [ -r "${svnrepo_from}/PKGBUILD" ]; then - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) + xbsrepo_from="$(xbs releasepath "${pkgbase}" "${repo_from}" "${pkgarch}")" + if [ -r "${xbsrepo_from}/PKGBUILD" ]; then + pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) if [ ${#pkgnames[@]} -lt 1 ]; then die "Could not read pkgname" fi - pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") + pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") if [ -z "${pkgver}" ]; then die "Could not read pkgver" fi @@ -61,18 +61,20 @@ msg "Moving packages from [%s] to [%s]..." "${repo_from}" "${repo_to}" declare -A add_pkgs declare -A remove_pkgs for pkgbase in "${args[@]:2}"; do - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" - - if [ -f "${svnrepo_from}/PKGBUILD" ]; then + # move the package in xbs + arches=($(xbs move "${repo_from}" "${repo_to}" "${pkgbase}")) + # move the package in ftp + for pkgarch in "${arches[@]}"; do + xbsrepo_to="$(xbs releasepath "$pkgbase" "$repo_to" "$pkgarch")" + if true; then # to add an indent level to make merging easier if [ "${pkgarch}" == 'any' ]; then tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi msg2 '%s (%s)' "${pkgbase}" "${tarches[*]}" - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) - pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") + pkgnames=($(. "${xbsrepo_to}/PKGBUILD"; echo "${pkgname[@]}")) + pkgver=$(. "${xbsrepo_to}/PKGBUILD"; get_full_version "${epoch:-0}" "${pkgver}" "${pkgrel}") for pkgname in "${pkgnames[@]}"; do for tarch in "${tarches[@]}"; do diff --git a/db-remove b/db-remove index 53fdb2a..dcbe4b4 100755 --- a/db-remove +++ b/db-remove @@ -30,11 +30,13 @@ remove_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" - path="${SVNREPO}/$repo/$pkgbase" + path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" if [ -d "$path" ]; then remove_pkgs+=($(. "$path/PKGBUILD"; echo "${pkgname[@]}")) + xbs unrelease "$pkgbase" "$repo" "$arch" else - warning "%s not found in ABS(libre)" "$pkgbase" + warning "%s not found in %s for %s" \ + "$pkgbase" "$(xbs name)" "$repo-$arch" warning "Removing only %s from the repo" "$pkgbase" warning "If it was a split package you have to remove the others yourself!" remove_pkgs+=("$pkgbase") -- cgit v1.2.3 From 003288a90c042c56d86258f60ac205b1d3053eab Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 24 May 2015 15:37:05 -0600 Subject: Remove cron-jobs/sourceballs2. It was "simpler" than cron-jobs/sourceballs because it iterated over the files in $SVNREPO directly, rather than getting the list from $FTP_BASE, and then getting the corresponding files from $SVNREPO. This fails with XBS because there is no single `abstree` path, there is one for each architecture. --- cron-jobs/sourceballs2 | 62 -------------------------------------------------- 1 file changed, 62 deletions(-) delete mode 100755 cron-jobs/sourceballs2 diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 deleted file mode 100755 index e935f86..0000000 --- a/cron-jobs/sourceballs2 +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Steps -# Traverse ABSLibre -# Makepkg --allsource every package -# Remove the old sourceballs - -dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../config" -. "${dirname}/../db-functions" -. "${MAKEPKGCONF}" - -pushd "${WORKDIR}" >/dev/null - -script_lock - -# Adjust the nice level to run at a lower priority -renice +10 -p $$ > /dev/null - -# Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" - -pushd "${SVNREPO}" >/dev/null - -for repo in "${PKGREPOS[@]}"; do - msg "Sourceballing [%s]" "${repo}" - - pushd "$repo" >/dev/null - find -maxdepth 1 -type d | while read pkg; do - pushd "${SVNREPO}/$repo/$pkg" >/dev/null - - [[ ! -e ./PKGBUILD ]] && { - warning "%s is not a package" "$repo/$pkg" - continue - } - - # Unset the previous data - unset pkgbase pkgname pkgver pkgrel - source PKGBUILD - - unset build package url pkgdesc source md5sums depends makedepends \ - optdepends license arch options check mksource - - for _pkg in "${pkgname[@]}"; do - unset "package_${_pkg}" >/dev/null 2>&1 - done - - pkgbase=${pkgbase:-$pkgname} - srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" - - echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs" - - # Skip already sourceballed - [ -e "${SRCPKGDEST}/${srcfile}" ] && continue - - makepkg --allsource --ignorearch -c >/dev/null 2>&1 - - [ $? -ne 0 ] && plain '%s' "${srcfile}" - - done # end find pkgs - popd >/dev/null - -done # end repos -- cgit v1.2.3 From b96b9d95dd4d2214a428f1435a9a72ef50f43cf5 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 5 Jun 2015 00:48:12 -0600 Subject: db-update: Hook in to `xbs release-server` as it exists today. Obviously, this is an exercise in noticing a leaky abstraction. - assumes that the PKGBUILD is staged to a real directory going in - assumes where that is - assumes how to clean that up I believe the correct solution will require modifying XBS. Following is what I believe the changes should be. - change release-server to take "REPO ARCH DIR1 [DIR2...]", instead of using the CWD. - but this still assumes that the caller knows where release-client put the PKGBUILD, so change it to take "REPO ARCH PKGBASE1 [PKGBASE2...]". - this means that release-server knows to check if a PKGBUILD exists at the specified arch, and fall back to 'any' if it doesn't. - an alternative would be passing in the .pkg.tar filenames, and either parsing the filename or using db-functions to figure out the arch. - ??? how to get rid of cleaning up assumptions. --- db-update | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/db-update b/db-update index 0c4f824..f22eb15 100755 --- a/db-update +++ b/db-update @@ -55,13 +55,16 @@ for repo in "${repos[@]}"; do fi done +dirs=() for repo in "${repos[@]}"; do msg "Updating [%s]..." "${repo}" any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do + add_dirs=() add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-"${pkgarch}"${PKGEXT} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do + add_dirs+=("${STAGING}/abslibre/$(getpkgarch "$pkg")/$(getpkgbase "$pkg")") pkgfile="${pkg##*/}" msg2 "%s (%s)" "${pkgfile}" "${pkgarch}" # any packages might have been moved by the previous run @@ -78,9 +81,14 @@ for repo in "${repos[@]}"; do fi add_pkgs+=("${pkgfile}") done + for add_dir in "${add_dirs[@]}"; do + (cd "${add_dir}" && xbs release-server "${repo}" "${pkgarch}") || + error 'cd %q && xbs release-server %q %q' "${add_dir}" "${repo}" "${pkgarch}" + done if [ ${#add_pkgs[@]} -ge 1 ]; then arch_repo_add "${repo}" "${pkgarch}" "${add_pkgs[@]}" fi + dirs+=("${add_dirs[@]}") done done @@ -91,6 +99,14 @@ for repo in "${repos[@]}"; do done cd "${STAGING}" + +# Remove left over XBS files +rm -rf -- "${dirs[@]}" +dirname -z -- "${dirs[@]}" | + xargs -0 realpath -zm --relative-to="${STAGING}" -- | + xargs -0 rmdir -p -- 2>/dev/null + +# Stage generated source files while read -r file; do pub="${FTP_BASE}/${file}" if [[ -f "$pub" ]]; then -- cgit v1.2.3 From 6fd5374e66877d8d1b44aaf6f70e8a00ad8c2d7b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 8 Jun 2015 07:01:18 +0100 Subject: rename libremessages to db-libremessages to avoid masking libretools libremessages --- any-to-ours | 2 +- db-cleanup | 2 +- db-libremessages | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ db-sync | 2 +- libremessages | 83 -------------------------------------------------------- 5 files changed, 86 insertions(+), 86 deletions(-) create mode 100755 db-libremessages delete mode 100755 libremessages diff --git a/any-to-ours b/any-to-ours index d98cc5d..a901d54 100755 --- a/any-to-ours +++ b/any-to-ours @@ -8,7 +8,7 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/libremessages" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" # From makepkg set -E diff --git a/db-cleanup b/db-cleanup index efe7397..ffa2601 100755 --- a/db-cleanup +++ b/db-cleanup @@ -16,7 +16,7 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/libremessages" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" # From makepkg set -E diff --git a/db-libremessages b/db-libremessages new file mode 100755 index 0000000..37df149 --- /dev/null +++ b/db-libremessages @@ -0,0 +1,83 @@ +# Copyright (c) 2006-2010 Pacman Development Team +# Copyright (c) 2002-2006 by Judd Vinet +# Copyright (c) 2005 by Aurelien Foret +# Copyright (c) 2006 by Miklos Vajna +# Copyright (c) 2005 by Christian Hamar +# Copyright (c) 2006 by Alex Smith +# Copyright (c) 2006 by Andras Voroskoi +# Copyright (c) 2011 by Joshua Haase +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# gettext initialization +export TEXTDOMAIN='libretools' +export TEXTDOMAINDIR='/usr/share/locale' + +# check if messages are to be printed using color +unset ALL_OFF BOLD BLUE GREEN RED YELLOW + +if tput setaf 0 &>/dev/null; then + ALL_OFF="$(tput sgr0)" + BOLD="$(tput bold)" + BLUE="${BOLD}$(tput setaf 4)" + GREEN="${BOLD}$(tput setaf 2)" + RED="${BOLD}$(tput setaf 1)" + YELLOW="${BOLD}$(tput setaf 3)" + PURPLE="${ALL_OFF}$(tput setaf 5)" +else + ALL_OFF="\033[1;0m" + BOLD="\033[1;1m" + BLUE="${BOLD}\033[1;34m" + GREEN="${BOLD}\033[1;32m" + RED="${BOLD}\033[1;31m" + YELLOW="${BOLD}\033[1;33m" + PURPLE="${BOLD}\033[1;30;40m" +fi + +stdnull() { + local action=$1; + eval "${action} >/dev/null 2>&1" +} + +plain() { + local mesg=$1; shift + printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg() { + local mesg=$1; shift + printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg2() { + local mesg=$1; shift + printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +warning() { + local mesg=$1; shift + printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +fatal_error() { + local mesg=$1; shift + error "$mesg" "$@" + + exit 1 +} diff --git a/db-sync b/db-sync index 35b6489..2194fe6 100755 --- a/db-sync +++ b/db-sync @@ -190,7 +190,7 @@ trap_exit() { source "$(dirname "$(readlink -e "$0")")/config" source "$(dirname "$(readlink -e "$0")")/db-sync.conf" -source "$(dirname "$(readlink -e "$0")")/libremessages" +source "$(dirname "$(readlink -e "$0")")/db-libremessages" # Check variables presence for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do diff --git a/libremessages b/libremessages deleted file mode 100755 index 37df149..0000000 --- a/libremessages +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2006-2010 Pacman Development Team -# Copyright (c) 2002-2006 by Judd Vinet -# Copyright (c) 2005 by Aurelien Foret -# Copyright (c) 2006 by Miklos Vajna -# Copyright (c) 2005 by Christian Hamar -# Copyright (c) 2006 by Alex Smith -# Copyright (c) 2006 by Andras Voroskoi -# Copyright (c) 2011 by Joshua Haase -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# gettext initialization -export TEXTDOMAIN='libretools' -export TEXTDOMAINDIR='/usr/share/locale' - -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW - -if tput setaf 0 &>/dev/null; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" - PURPLE="${ALL_OFF}$(tput setaf 5)" -else - ALL_OFF="\033[1;0m" - BOLD="\033[1;1m" - BLUE="${BOLD}\033[1;34m" - GREEN="${BOLD}\033[1;32m" - RED="${BOLD}\033[1;31m" - YELLOW="${BOLD}\033[1;33m" - PURPLE="${BOLD}\033[1;30;40m" -fi - -stdnull() { - local action=$1; - eval "${action} >/dev/null 2>&1" -} - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -fatal_error() { - local mesg=$1; shift - error "$mesg" "$@" - - exit 1 -} -- cgit v1.2.3 From 969e2de31319c95913ec52f85b97552c3d26414f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 8 Jun 2015 07:02:15 +0100 Subject: db-update: get the correct directory for xbs --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index f22eb15..4830791 100755 --- a/db-update +++ b/db-update @@ -64,7 +64,6 @@ for repo in "${repos[@]}"; do add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-"${pkgarch}"${PKGEXT} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do - add_dirs+=("${STAGING}/abslibre/$(getpkgarch "$pkg")/$(getpkgbase "$pkg")") pkgfile="${pkg##*/}" msg2 "%s (%s)" "${pkgfile}" "${pkgarch}" # any packages might have been moved by the previous run @@ -79,6 +78,7 @@ for repo in "${repos[@]}"; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi + add_dirs+=("${STAGING}/abslibre/$(getpkgarch "$FTP_BASE/$PKGPOOL/$pkgfile")/$repo/$(getpkgbase "$FTP_BASE/$PKGPOOL/$pkgfile")") add_pkgs+=("${pkgfile}") done for add_dir in "${add_dirs[@]}"; do -- cgit v1.2.3 From 00c15eb70600b5024973fd19528761381a89f759 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 8 Jun 2015 07:03:07 +0100 Subject: config: remove mips64el from ARCHES --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index f1b09eb..287f5b0 100644 --- a/config +++ b/config @@ -43,7 +43,7 @@ LOCK_TIMEOUT=300 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) -OURARCHES=(mips64el) +OURARCHES=() ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -- cgit v1.2.3 From 780c57ec14e7ccbf8695ccc159bbee49cf17e237 Mon Sep 17 00:00:00 2001 From: Parabola Repo Date: Mon, 8 Jun 2015 07:04:15 +0100 Subject: These changes to abslibre were sitting on the server --- abslibre | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/abslibre b/abslibre index af57ec2..6b21d24 100755 --- a/abslibre +++ b/abslibre @@ -1,6 +1,6 @@ #!/bin/bash -set -e -x +set -e FTP_BASE=/srv/repo/main ABSLIBRE=/srv/abslibre @@ -52,12 +52,12 @@ function get_blacklist() { function sync_abs_libre() { # Clone ABSLibre git repo - if [ -d /var/tmp/abslibre/.git ]; then - pushd /var/tmp/abslibre >/dev/null 2>&1 + if [ -d /tmp/abslibre/.git ]; then + pushd /tmp/abslibre >/dev/null 2>&1 git pull popd >/dev/null 2>&1 else - git clone "$ABSGIT" /var/tmp/abslibre + git clone "$ABSGIT" /tmp/abslibre fi # Sync from ABS and then sync from ABSLibre @@ -67,7 +67,7 @@ function sync_abs_libre() { ${ABSROOT} \ ${ABSLIBRE} \ && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { printf "[FAILED]\n" return 1 } -- cgit v1.2.3 From 91b26994d40f7ea23f5bc7ef8e4e40e1af115e7a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 16 Apr 2016 14:30:39 -0400 Subject: db-pick-mirror: use Net::HTTP instead of RestClient --- db-pick-mirror | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-pick-mirror b/db-pick-mirror index 9474ed7..6adcdaf 100755 --- a/db-pick-mirror +++ b/db-pick-mirror @@ -1,12 +1,12 @@ #!/usr/bin/env ruby require 'json' -require 'rest_client' +require 'net/http' protocol = ARGV[0] jsonurl = ARGV[1] -data = JSON::parse(RestClient.get(jsonurl)) +data = JSON::parse(Net::HTTP.get(URI(jsonurl))) if data["version"] != 3 print "Data format version != 3" -- cgit v1.2.3 From 752bd700e9da464006bfbd9877cc858dfad545ba Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 16 Apr 2016 14:31:34 -0400 Subject: db-pick-mirror: filter out URLs with incomplete information --- db-pick-mirror | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db-pick-mirror b/db-pick-mirror index 6adcdaf..4d01b95 100755 --- a/db-pick-mirror +++ b/db-pick-mirror @@ -13,7 +13,8 @@ if data["version"] != 3 exit 1 end -urls = data["urls"] +# Filter out URLs with incomplete information +urls = data["urls"].select{|a| a.none?{|k,v|v.nil?}} rsync_urls = urls.select{|a| a["protocol"]==protocol} # By score ( (delay+speed)/completion ) -- cgit v1.2.3