diff options
Diffstat (limited to 'cmd/generate/httpcache.go')
-rw-r--r-- | cmd/generate/httpcache.go | 95 |
1 files changed, 0 insertions, 95 deletions
diff --git a/cmd/generate/httpcache.go b/cmd/generate/httpcache.go deleted file mode 100644 index 04762e3..0000000 --- a/cmd/generate/httpcache.go +++ /dev/null @@ -1,95 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "path/filepath" -) - -var httpCache = map[string]string{} - -func httpGet(u string) (string, error) { - if cache, ok := httpCache[u]; ok { - fmt.Printf("CACHE-GET %q\n", u) - return cache, nil - } - if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) { - return "", err - } - cacheFile := filepath.Join(".http-cache", url.QueryEscape(u)) - if bs, err := os.ReadFile(cacheFile); err == nil { - httpCache[u] = string(bs) - fmt.Printf("CACHE-GET %q\n", u) - return httpCache[u], nil - } else if !os.IsNotExist(err) { - return "", err - } - - fmt.Printf("GET %q...", u) - resp, err := http.Get(u) - if err != nil { - fmt.Printf(" err\n") - return "", err - } - if resp.StatusCode != http.StatusOK { - fmt.Printf(" err\n") - return "", fmt.Errorf("unexpected HTTP status: %v", resp.Status) - } - bs, err := io.ReadAll(resp.Body) - if err != nil { - fmt.Printf(" err\n") - return "", err - } - fmt.Printf(" ok\n") - if err := os.WriteFile(cacheFile, bs, 0666); err != nil { - return "", err - } - httpCache[u] = string(bs) - return httpCache[u], nil -} - -func httpGetJSON(u string, out any) error { - str, err := httpGet(u) - if err != nil { - return err - } - return json.Unmarshal([]byte(str), out) -} - -func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.Values) error { - u, err := url.Parse(uStr) - if err != nil { - return err - } - query := u.Query() - - for i := 0; true; i++ { - pageParams := pageFn(i) - for k, v := range pageParams { - query[k] = v - } - - u.RawQuery = query.Encode() - var resp []T - if err := httpGetJSON(u.String(), &resp); err != nil { - return err - } - fmt.Printf(" -> %d records\n", len(resp)) - if len(resp) == 0 { - break - } - *out = append(*out, resp...) - } - - return nil -} - -func githubPagination(i int) url.Values { - params := make(url.Values) - params.Set("page", fmt.Sprintf("%v", i+1)) - return params -} |