[ci-skip] gen-posix: Use the HTTP cache

2 files changed, 41 insertions, 4 deletions

diff --git a/cmd/gen-posix/data.go b/cmd/gen-posix/data.go
index 9ec3de4..ac92095 100644
--- a/cmd/gen-posix/data.go
+++ b/cmd/gen-posix/data.go
@@ -2,25 +2,48 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"os/exec"
+	"regexp"
 	"strings"
+
+	"git.lukeshu.com/www/lib/httpcache"
 )
 
 var IEEESA = Vendor{
 	Name:   "IEEE-SA",
 	GetURL: func(id string) string { return fmt.Sprintf("http://standards.ieee.org/findstds/standard/%s.html", id) },
 	GetName: func(id string, url string) string {
-		d, _ := exec.Command("nokogiri", url, "-e", `puts $_.css("meta[name=\"des\"]").first["content"]`).Output()
+		html, err := httpcache.Get(url, nil)
+		if err != nil {
+			panic(fmt.Errorf("URL=%q: %v", url, err))
+		}
+		cmd := exec.Command("nokogiri", "-e", `puts $_.css("meta[name=\"des\"]").first["content"]`)
+		cmd.Stderr = os.Stderr
+		cmd.Stdin = strings.NewReader(html)
+		d, err := cmd.Output()
+		if err != nil {
+			panic(fmt.Errorf("URL=%q: %v", url, err))
+		}
 		return strings.TrimSuffix(string(d), "\n")
 	},
 }
 
+var reIEEE = regexp.MustCompile(`standardNumber":"([^"]*)"`)
+
 var IEEEXplore = Vendor{
 	Name:   "IEEE Xplore",
 	GetURL: func(id string) string { return fmt.Sprintf("http://ieeexplore.ieee.org/servlet/opac?punumber=%s", id) },
 	GetName: func(id string, url string) string {
-		d, _ := exec.Command("sh", "-c", `curl -sL "$1"|grep -o '"standardNumber":"[^"]*"'|cut -d'"' -f4`, "--", url).Output()
-		return strings.TrimSuffix(string(d), "\n")
+		html, err := httpcache.Get(url, nil)
+		if err != nil {
+			panic(fmt.Errorf("URL=%q: %v", url, err))
+		}
+		m := reIEEE.FindStringSubmatch(html)
+		if m == nil {
+			panic(fmt.Errorf("URL=%q did not contain expected JSON", url))
+		}
+		return m[1]
 	},
 }
 
@@ -42,7 +65,17 @@ var ISO = Vendor{
 		return fmt.Sprintf("http://www.iso.org/iso/home/store/catalogue_tc/catalogue_detail.htm?csnumber=%s", id)
 	},
 	GetName: func(id string, url string) string {
-		d, _ := exec.Command("nokogiri", url, "-e", `puts $_.css("[itemprop=\"name\"]").first.text`).Output()
+		html, err := httpcache.Get(url, nil)
+		if err != nil {
+			panic(fmt.Errorf("URL=%q: %v", url, err))
+		}
+		cmd := exec.Command("nokogiri", "-e", `puts $_.css("[itemprop=\"name\"]").first.text`)
+		cmd.Stderr = os.Stderr
+		cmd.Stdin = strings.NewReader(html)
+		d, err := cmd.Output()
+		if err != nil {
+			panic(fmt.Errorf("URL=%q: %v", url, err))
+		}
 		return strings.TrimSuffix(string(d), "\n")
 	},
 }
diff --git a/cmd/gen-posix/main.go b/cmd/gen-posix/main.go
index 354fd3f..7525719 100644
--- a/cmd/gen-posix/main.go
+++ b/cmd/gen-posix/main.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"html/template"
 	"os"
+
+	"git.lukeshu.com/www/lib/httpcache"
 )
 
 var urls = map[string]string{}
@@ -182,6 +184,8 @@ var tmpl = `{{define "document"}}{{if .}}
 `
 
 func mainWithError() error {
+	httpcache.UserAgent = "https://git.lukeshu.com/www/tree/cmd/gen-posix"
+
 	tmpl := template.Must(template.New("page").Parse(tmpl))
 
 	var out bytes.Buffer