summaryrefslogtreecommitdiff
path: root/cmd/generate/httpcache.go
diff options
context:
space:
mode:
Diffstat (limited to 'cmd/generate/httpcache.go')
-rw-r--r--cmd/generate/httpcache.go95
1 files changed, 0 insertions, 95 deletions
diff --git a/cmd/generate/httpcache.go b/cmd/generate/httpcache.go
deleted file mode 100644
index 04762e3..0000000
--- a/cmd/generate/httpcache.go
+++ /dev/null
@@ -1,95 +0,0 @@
-package main
-
-import (
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "os"
- "path/filepath"
-)
-
-var httpCache = map[string]string{}
-
-func httpGet(u string) (string, error) {
- if cache, ok := httpCache[u]; ok {
- fmt.Printf("CACHE-GET %q\n", u)
- return cache, nil
- }
- if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) {
- return "", err
- }
- cacheFile := filepath.Join(".http-cache", url.QueryEscape(u))
- if bs, err := os.ReadFile(cacheFile); err == nil {
- httpCache[u] = string(bs)
- fmt.Printf("CACHE-GET %q\n", u)
- return httpCache[u], nil
- } else if !os.IsNotExist(err) {
- return "", err
- }
-
- fmt.Printf("GET %q...", u)
- resp, err := http.Get(u)
- if err != nil {
- fmt.Printf(" err\n")
- return "", err
- }
- if resp.StatusCode != http.StatusOK {
- fmt.Printf(" err\n")
- return "", fmt.Errorf("unexpected HTTP status: %v", resp.Status)
- }
- bs, err := io.ReadAll(resp.Body)
- if err != nil {
- fmt.Printf(" err\n")
- return "", err
- }
- fmt.Printf(" ok\n")
- if err := os.WriteFile(cacheFile, bs, 0666); err != nil {
- return "", err
- }
- httpCache[u] = string(bs)
- return httpCache[u], nil
-}
-
-func httpGetJSON(u string, out any) error {
- str, err := httpGet(u)
- if err != nil {
- return err
- }
- return json.Unmarshal([]byte(str), out)
-}
-
-func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.Values) error {
- u, err := url.Parse(uStr)
- if err != nil {
- return err
- }
- query := u.Query()
-
- for i := 0; true; i++ {
- pageParams := pageFn(i)
- for k, v := range pageParams {
- query[k] = v
- }
-
- u.RawQuery = query.Encode()
- var resp []T
- if err := httpGetJSON(u.String(), &resp); err != nil {
- return err
- }
- fmt.Printf(" -> %d records\n", len(resp))
- if len(resp) == 0 {
- break
- }
- *out = append(*out, resp...)
- }
-
- return nil
-}
-
-func githubPagination(i int) url.Values {
- params := make(url.Values)
- params.Set("page", fmt.Sprintf("%v", i+1))
- return params
-}