PATH:=$(CURDIR)/bin:$(PATH)
export PATH

all: each-cdx

dat:
	mkdir -p $@
dat/ftp.txt: | dat
	cdxget 'url=ftp.unicode.org/Public/*' 'fl=urlkey' 'filter=statuscode:200' 'filter=urlkey:.*(cvt|convert)utf.*' > $@
dat/www.txt: | dat
	cdxget 'url=www.unicode.org/Public/*' 'fl=urlkey' 'filter=statuscode:200' 'filter=urlkey:.*(cvt|convert)utf.*' > $@
dat/urlkeys.txt: dat/ftp.txt dat/www.txt 
	cat $^ | cut -d '?' -f1 | sed 's/,ftp)/)/' | sort -u > $@
dat/urlkeys.mk: dat/urlkeys.txt
	cat $^ | sed 's/^/urlkeys+=/' < $< > $@

-include dat/urlkeys.mk
rp = )
c = ,
all_urlkeys = $(urlkeys) $(subst $(rp),$(c)ftp$(rp),$(urlkeys))

dat/each-cdx/%.txt:
	mkdir -p '$(@D)'
	cdxget "url=$$(urlkey2url '$*')" 'filter=statuscode:200' 'collapse=digest' 'fl=timestamp,original' > '$@'

each-cdx: $(addprefix dat/each-cdx/,$(addsuffix .txt,$(all_urlkeys)))
.PHONY: each-cdx

.DELETE_ON_ERROR:
.SECONDARY: