summaryrefslogtreecommitdiff
path: root/extra/xbs-lukeshu/db-import
blob: a8a073dee7f6ebda754b66d5c7b9ff653e556742 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
#!/bin/bash
set -euE
# Imports Arch-like repos, running them through a blacklist
# License: GPLv3

. "$(dirname "$(readlink -e "$0")")/config"         # for: FTP_BASE DBEXT
. "$(dirname "$(readlink -e "$0")")/db-import.conf" # for: IMPORTDIR IMPORTS
. "$(librelib messages)"
. "$(librelib blacklist)"

# DBs = pacman DataBases

# This replaces two scripts:
#  - abslibre : imported ABS tree from Arch
#  - db-sync  : imported pacman DBs from Arch

# The flow here is:
#  1. "${IMPORTDIR}/cache/${name}/dbs/" # Download the pacman databases
#  2. "${IMPORTDIR}/cache/${name}/abs/" # Download the ABS tree
#  3. "${IMPORTDIR}/clean/${name}/dbs/" # Run the pacman DBs through the blacklist
#  4. "${IMPORTDIR}/clean/${name}/pkgs/" # Download the pkg files mentioned in "clean/${name}/dbs/"
#  5. "${IMPORTDIR}/staging/${tag}" # Copy all the package files we just downloaded to here
#  6. Run `db-update on` with STAGING="${IMPORTDIR}/staging/${tag}"

# generic arguments to pass to rsync, borrowed from `abs`
SYNCARGS='-mrtvlH --no-motd --no-p --no-o --no-g'

main() {
	blacklist-update

	local importStr
	for importStr in "${IMPORTS[@]}"; do
		local importAry=($importStr)
		local name=${importAry[0]}
		local pkgmirror=${importAry[1]}
		local absmirror=${importAry[2]}
		local tags=("${importAry[@]:3}")

		msg "Fetching remote package source: %s" "$name"
		fetch_dbs "$name" "$pkgmirror"
		fetch_abs "$name" "$absmirror" "${tags[@]}"
		msg "Filtering blacklisted packages from remote package source: %s" "$name"
		clean_dbs "$name" "${tags[@]}"
		fetch_pkgs "$name" "${tags[@]}"
		msg "Publishing changes from remote package source: %s" "$name"
		publish "$name" "${tags[@]}"
	done
	return $r
}

fetch_dbs() {
	local name=$1
	local pkgmirror=$2

	msg2 'Synchronizing package databases...'

	mkdir -p -- "${IMPORTDIR}/cache/${name}/dbs"
	# Grab just the .db files from $pkgmirror
	rsync $SYNCARGS --delete-after \
		--include="*/" \
		--include="*.db" \
		--include="*${DBEXT}" \
		--exclude="*" \
		"rsync://${pkgmirror}/" "${IMPORTDIR}/cache/${name}/dbs"
}

fetch_abs() {
	local name=$1
	local absmirror=$2
	local tags=("${@:3}")

	local fake_home
	local absroot

	# Sync the ABS tree from $absmirror
	local arch
	for arch in $(list_arches "${tags[@]}"); do
		msg2 'Synchronizing %s ABS tree...' "$arch"

		absroot="${IMPORTDIR}/cache/${name}/abs/${arch}"
		mkdir -p -- "$absroot"

		# Configure `abs` for this mirror
		fake_home="${IMPORTDIR}/homes/${name}/${arch}"
		mkdir -p -- "$fake_home"
		{
			printf "ABSROOT='%s'\n" "$absroot"
			printf "SYNCSERVER='%s'\n" "$absmirror"
			printf "ARCH='%s'\n" "$arch"
			printf 'REPOS=(\n'
			list_repos "$arch" "${tags[@]}"
			printf ')\n'
		} > "${fake_home}/.abs.conf"

		# Run `abs`
		HOME=$fake_home abs
	done
}

clean_dbs() {
	local name=$1
	local tags=("${@:2}")

	rm -rf -- "${IMPORTDIR}/clean/$name"

	local tag
	for tag in "${tags[@]}"; do
		msg2 'Creating clean version of %s package database...' "$tag"

		local cache="${IMPORTDIR}/cache/$name/dbs/$(db_file "$tag")"
		local clean="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
		install -Dm644 "$cache" "$clean"

		blacklist-cat | blacklist-get-pkg | xargs -d '\n' repo-remove "$clean"
	done
}

fetch_pkgs() {
	local name=$1
	local tags=("${@:2}")

	local repo arch dbfile whitelist

	local tag
	for tag in "${tags[@]}"; do
		msg2 'Syncronizing package files for %s...' "$tag"
		repo=${tag%-*}
		arch=${tag##*-}

		dbfile="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
		whitelist="${IMPORTDIR}/clean/$name/dbs/$tag.whitelist"

		list_pkgs "$dbfile" > "$whitelist"

		# fetch the architecture-specific packages
		rsync $SYNCARGS --delete-after --delete-excluded \
			--delay-updates \
			--include-from=<(sed "s|\$|-$arch.tar.?z|" "$whitelist") \
			--exclude='*' \
			"rsync://${pkgmirror}/$(db_dir "$tag")/" \
			"${IMPORTDIR}/clean/${name}/pkgs/${tag}/"

		# fetch the architecture-independent packages
		rsync $SYNCARGS --delete-after --delete-excluded \
			--delay-updates \
			--include-from=<(sed "s|\$|-any.tar.?z|" "$whitelist") \
			--exclude='*' \
			"rsync://${pkgmirror}/$(db_dir "$tag")/" \
			"${IMPORTDIR}/clean/${name}/pkgs/${repo}-any/"
	done
}

publish() {
	local name=$1
	local tags=("${@:2}")

	local tag
	for tag in "${tags[@]}"; do
		msg2 'Publishing changes to %s...' "$tag"
		publish_tag "$name" "$tag"
	done
}

publish_tag() {
	local name=$1
	local tag=$2

	local repo=${tag%-*}
	local arch=${tag##*-}
	local dir="${IMPORTDIR}/clean/${name}/pkgs/${tag}"

	local found
	local error=false
	local files=()

	local pkgid pkgarch
	for pkgid in $(list_added_pkgs "$name" "$tag"); do
		found=false

		for pkgarch in "${arch}" any; do
			file="${dir}/${pkgid}-${arch}".pkg.tar.?z
			if ! $found && [[ -r $file ]]; then
				files+=("$file")
				found=true
			fi
		done

		if ! $found; then
			error 'Could not find package file for %s' "$pkgid"
			error=true
		fi
	done

	if $error; then
		error 'Quitting...'
		return 1
	fi

	mkdir -p -- "${IMPORTDIR}/staging/${tag}/${repo}"
	cp -al -- "${files[@]}" "${IMPORTDIR}/staging/${tag}/${repo}/"
	STAGING="${IMPORTDIR}/staging/${tag}" db-update

	# XXX: db-remove wants pkgbase, not pkgname
	list_removed_pkgs "$name" "$tag" | xargs -d '\n' db-remove "$repo" "$arch"
}

################################################################################

# Usage: list_arches repo-arch...
# Returns a list of the architectures mentioned in a list of "repo-arch" pairs.
list_arches() {
	local tags=("$@")
	printf '%s\n' "${tags[@]##*-}" | sort -u
}

# Usage: list_repos arch repo-arch...
# Returns a list of all the repositories mentioned for a given architecture in a
# list of "repo-arch" pairs.
list_repos() {
	local arch=$1
	local tags=("${@:2}")
	printf '%s\n' "${tags[@]}" | sed -n "s/-$arch\$//p"
}

# Usage: db_dir repo-arch
db_dir() {
	local tag=$1
	local repo=${tag%-*}
	local arch=${tag##*-}
	echo "${repo}/os/${arch}"
}

# Usage; db_file repo-arch
db_file() {
	local tag=$1
	local repo=${tag%-*}
	local arch=${tag##*-}
	echo "${repo}/os/${arch}/${repo}${DBEXT}"
}

# Usage: list_pkgs dbfile
# Prints "$pkgname-$(get_full_version "$pkgname")" for every package in $dbfile
list_pkgs() {
	local dbfile=$1
	bsdtar tf "$dbfile" | cut -d/ -f1
}

# Usage: list_pkgs | sep_ver
# Separates the pkgname from the version (replaces the '-' with ' ') for the
# list provided on stdin.
sep_ver() {
	sed -r 's/-([^-]*-[^-]*)$/ \1/'
}

# Usage: list_removed_pkgs importsrc repo-arch
# Prints "$pkgname-$(get_full_version "$pkgname")" for every removed package.
list_removed_pkgs() {
	local name=$1
	local tag=$2

	local old="${FTP_BASE}/$(db_file "$tag")"
	local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"

	# make a list of:
	# pkgname oldver[ newver]
	# It will include removed or updated packages (changed packages)
	join -a1 \
		<(list_pkgs "$old"|sep_ver|sort) \
		<(list_pkgs "$new"|sep_ver|sort)
	| grep -v ' .* ' # remove updated packages
	| sed 's/ /-/' # re-combine the pkgname and version
}

# Usage: list_added_pkgs importsrc repo-arch
# slightly a misnomer; added and updated
# Prints "$pkgname-$(get_full_version "$pkgname")" for every added or updated
# package.
list_added_pkgs() {
	local name=$1
	local tag=$2

	local old="${FTP_BASE}/$(db_file "$tag")"
	local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"

	comm -13 <(list_pkgs "$old") <(list_pkgs "$new")
}

main "$@"