From e46a74fe8a143936eee2b9be1fd6b5f963357d9d Mon Sep 17 00:00:00 2001
From: Luke Shumaker <lukeshu@lukeshu.com>
Date: Fri, 30 Jun 2017 20:05:23 -0400
Subject: work on it

---
 Makefile | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

(limited to 'Makefile')

diff --git a/Makefile b/Makefile
index 14db651..eb8ae8e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,24 +1,29 @@
+PATH:=$(CURDIR)/bin:$(PATH)
+export PATH
+
 all: each-cdx
 
-ftp.json: cdx_json
-	./cdx_json 'url=ftp.unicode.org/Public/*' 'collapse=urlkey' > $@
-www.json: cdx_json
-	./cdx_json 'url=www.unicode.org/Public/*' 'collapse=urlkey' > $@
-urlkeys.txt: ftp.json www.json Makefile
-	cat $^ | cut -d '"' -f2,10 | sed -n 's/"200$$//p' | cut -d '?' -f1 | sed 's/,ftp)/)/' | sort -u | grep -i -e cvtutf -e convertutf > $@
-urlkeys.mk: urlkeys.txt
-	sed 's/^/urlkeys+=/' < $< > $@
+dat:
+	mkdir -p $@
+dat/ftp.txt: | dat
+	cdxget 'url=ftp.unicode.org/Public/*' 'fl=urlkey' 'filter=statuscode:200' 'filter=urlkey:.*(cvt|convert)utf.*' > $@
+dat/www.txt: | dat
+	cdxget 'url=www.unicode.org/Public/*' 'fl=urlkey' 'filter=statuscode:200' 'filter=urlkey:.*(cvt|convert)utf.*' > $@
+dat/urlkeys.txt: dat/ftp.txt dat/www.txt 
+	cat $^ | cut -d '?' -f1 | sed 's/,ftp)/)/' | sort -u > $@
+dat/urlkeys.mk: dat/urlkeys.txt
+	cat $^ | sed 's/^/urlkeys+=/' < $< > $@
 
--include urlkeys.mk
+-include dat/urlkeys.mk
 rp = )
 c = ,
 all_urlkeys = $(urlkeys) $(subst $(rp),$(c)ftp$(rp),$(urlkeys))
 
-each-cdx/%.json: cdx_json urlkey2url
+dat/each-cdx/%.txt:
 	mkdir -p '$(@D)'
-	./cdx_json "url=$$(./urlkey2url '$*')" 'collapse=digest' > '$@'
+	cdxget "url=$$(urlkey2url '$*')" 'filter=statuscode:200' 'collapse=digest' 'fl=timestamp,original' > '$@'
 
-each-cdx: $(addprefix each-cdx/,$(addsuffix .json,$(all_urlkeys)))
+each-cdx: $(addprefix dat/each-cdx/,$(addsuffix .txt,$(all_urlkeys)))
 .PHONY: each-cdx
 
 .DELETE_ON_ERROR:
-- 
cgit v1.2.3