From 122c52fa9acb2b38853f41f3060e7d0474d376a9 Mon Sep 17 00:00:00 2001 From: Isaac David Date: Mon, 22 Aug 2016 16:14:57 -0500 Subject: create clean databases when importing arm packages The package databases provided upstream contain mismatches (e.g. package descs with a %PGPSIG% field in .db but not in .files), which causes parabolaweb to fail to add file lists. A new database with whitelisted packages is created to work around this. --- db-import-archlinuxarm-pkg | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/db-import-archlinuxarm-pkg b/db-import-archlinuxarm-pkg index 8d8cb3d..0268526 100755 --- a/db-import-archlinuxarm-pkg +++ b/db-import-archlinuxarm-pkg @@ -6,10 +6,9 @@ # * Get repo.db from an Arch-like repo # * Generate a list of available packages # * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo +# * Get packages and signatures +# * Create new repo.db with them +# * Sync repo.db => repo.db # TODO # * make a tarball of files used for forensics @@ -121,6 +120,16 @@ init() { "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ "${FTP_BASE}/${PKGPOOLARM}/" + # Avoid sloppy upstream databases with corrupt package descriptions, + # exclude .sig files from consideration. + msg2 "Adding whitelisted packages to clean %s and %s databases..." .db .files + # Optional, repo-add overwrites entries by default + rm "$db_file" "$files_file" + LC_ALL=C repo-add "$db_file" \ + $(sed "s|^|${FTP_BASE}/${PKGPOOLARM}/|; s|$|${PKGEXT}|" \ + "/tmp/${_repo}-${_arch}.whitelist") \ + |& sed -n 's/==> Adding/ -> Adding/p' + msg "Putting databases back in place" rsync "${extra[@]}" --no-motd -rtlH \ --delay-updates \ @@ -129,7 +138,7 @@ init() { "${FTP_BASE}/${_repo}/os/${_arch}/" # Cleanup - unset db + unset db done done -- cgit v1.2.3 From 7cb14da25387cfb3c39e9064d2885e0246d9d183 Mon Sep 17 00:00:00 2001 From: Isaac David Date: Mon, 22 Aug 2016 16:25:50 -0500 Subject: remove some redundant logic from import-*-pkg scripts repo-remove handles both .db and .files at once, there's no need to use it again. A second time produces the following error, it was hiding behind the pipe to sed: ERROR: '${_repo}.files.tar.gz' doesn't have a valid database archive extension. --- db-import-archlinux-pkg | 5 +---- db-import-archlinuxarm-pkg | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/db-import-archlinux-pkg b/db-import-archlinux-pkg index aae7862..17fa38d 100755 --- a/db-import-archlinux-pkg +++ b/db-import-archlinux-pkg @@ -89,12 +89,9 @@ init() { # Remove blacklisted packages and count them # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db + msg2 "Removing blacklisted packages from %s and %s databases..." .db .files LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' # Get db contents db=($(get_repo_content "${db_file}")) diff --git a/db-import-archlinuxarm-pkg b/db-import-archlinuxarm-pkg index 0268526..ef8c810 100755 --- a/db-import-archlinuxarm-pkg +++ b/db-import-archlinuxarm-pkg @@ -88,12 +88,9 @@ init() { # Remove blacklisted packages and count them # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db + msg2 "Removing blacklisted packages from %s and %s databases..." .db .files LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' # Get db contents db=($(get_repo_content "${db_file}")) -- cgit v1.2.3 From cb6eaefaf50af4f7936af2139bf5b48262a083f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Fri, 14 Oct 2016 10:18:45 -0300 Subject: add pcr-testing repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 43bef5e..e57698d 100644 --- a/config +++ b/config @@ -9,7 +9,7 @@ PKGREPOS=( 'core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing' 'libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing' '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' - 'nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') + 'nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'pcr-testing' 'kernels' 'cross' 'java') PKGPOOL='pool/parabola_gnu+linux' SRCPOOL='src/parabola_gnu+linux' -- cgit v1.2.3 From ef542f77bed6f8123e1272a935d3dfdb8e8c70ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Fri, 14 Oct 2016 10:28:06 -0300 Subject: add unmaintained repo --- config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config b/config index e57698d..5ed1357 100644 --- a/config +++ b/config @@ -9,7 +9,8 @@ PKGREPOS=( 'core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing' 'libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing' '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan' - 'nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'pcr-testing' 'kernels' 'cross' 'java') + 'nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'pcr-testing' 'kernels' + 'cross' 'java' 'unmaintained') PKGPOOL='pool/parabola_gnu+linux' SRCPOOL='src/parabola_gnu+linux' -- cgit v1.2.3 From 05e783a97c1cbba33fa3c86440ab46f182fda708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Fri, 14 Oct 2016 10:37:35 -0300 Subject: add community-testing repo --- db-import-archlinux.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-import-archlinux.conf b/db-import-archlinux.conf index fdf7a6a..8361121 100644 --- a/db-import-archlinux.conf +++ b/db-import-archlinux.conf @@ -1,4 +1,4 @@ -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') +ARCHREPOS=('core' 'extra' 'testing' 'community' 'community-testing' 'multilib' 'multilib-testing') ARCHPKGPOOLS=(pool/{packages,community}) ARCHSRCPOOLS=(sources/{packages,community}) ARCHARCHES=(i686 x86_64) -- cgit v1.2.3