diff options
author | Luke T. Shumaker <lukeshu@lukeshu.com> | 2024-05-04 13:31:22 -0400 |
---|---|---|
committer | Luke T. Shumaker <lukeshu@lukeshu.com> | 2024-05-04 13:31:22 -0400 |
commit | 618cfc7cddf91d01fe6636c81809a4f90b40c00c (patch) | |
tree | a08f5f0765f4cbb1c8901f2251bd7d8db264aeae | |
parent | 6e29a14f10b51c6f7005b4bfb352c57d5dda0ef7 (diff) |
imworkingon: Properly do pagination with the GH APIs
-rw-r--r-- | cmd/generate/httpcache.go | 36 | ||||
-rw-r--r-- | cmd/generate/src_contribs.go | 4 | ||||
-rw-r--r-- | cmd/generate/src_mastodon.go | 14 |
3 files changed, 43 insertions, 11 deletions
diff --git a/cmd/generate/httpcache.go b/cmd/generate/httpcache.go index cff0c7c..04762e3 100644 --- a/cmd/generate/httpcache.go +++ b/cmd/generate/httpcache.go @@ -14,6 +14,7 @@ var httpCache = map[string]string{} func httpGet(u string) (string, error) { if cache, ok := httpCache[u]; ok { + fmt.Printf("CACHE-GET %q\n", u) return cache, nil } if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) { @@ -22,6 +23,7 @@ func httpGet(u string) (string, error) { cacheFile := filepath.Join(".http-cache", url.QueryEscape(u)) if bs, err := os.ReadFile(cacheFile); err == nil { httpCache[u] = string(bs) + fmt.Printf("CACHE-GET %q\n", u) return httpCache[u], nil } else if !os.IsNotExist(err) { return "", err @@ -57,3 +59,37 @@ func httpGetJSON(u string, out any) error { } return json.Unmarshal([]byte(str), out) } + +func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.Values) error { + u, err := url.Parse(uStr) + if err != nil { + return err + } + query := u.Query() + + for i := 0; true; i++ { + pageParams := pageFn(i) + for k, v := range pageParams { + query[k] = v + } + + u.RawQuery = query.Encode() + var resp []T + if err := httpGetJSON(u.String(), &resp); err != nil { + return err + } + fmt.Printf(" -> %d records\n", len(resp)) + if len(resp) == 0 { + break + } + *out = append(*out, resp...) + } + + return nil +} + +func githubPagination(i int) url.Values { + params := make(url.Values) + params.Set("page", fmt.Sprintf("%v", i+1)) + return params +} diff --git a/cmd/generate/src_contribs.go b/cmd/generate/src_contribs.go index 4e65665..6db6764 100644 --- a/cmd/generate/src_contribs.go +++ b/cmd/generate/src_contribs.go @@ -292,7 +292,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) { HTMLURL string `json:"html_url"` } `json:"user"` } - if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/comments", &comments); err != nil { + if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/comments", &comments, githubPagination); err != nil { return time.Time{}, User{}, err } for _, comment := range comments { @@ -312,7 +312,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) { HTMLURL string `json:"html_url"` } `json:"user"` } - if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum+"/comments", &reviewComments); err != nil { + if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum+"/comments", &reviewComments, githubPagination); err != nil { return time.Time{}, User{}, err } for _, comment := range reviewComments { diff --git a/cmd/generate/src_mastodon.go b/cmd/generate/src_mastodon.go index 42ae8b2..b4b54a8 100644 --- a/cmd/generate/src_mastodon.go +++ b/cmd/generate/src_mastodon.go @@ -22,22 +22,18 @@ func ReadStandups(server, username string) ([]*MastodonStatus, error) { if err := httpGetJSON(server+"/api/v1/accounts/lookup?acct="+username, &account); err != nil { return nil, err } + var statuses []*MastodonStatus - for { + if err := httpGetPaginatedJSON(server+"/api/v1/accounts/"+account.ID+"/statuses", &statuses, func(_ int) url.Values { params := make(url.Values) params.Set("tagged", "DailyStandUp") params.Set("exclude_reblogs", "true") if len(statuses) > 0 { params.Set("max_id", statuses[len(statuses)-1].ID) } - var resp []*MastodonStatus - if err := httpGetJSON(server+"/api/v1/accounts/"+account.ID+"/statuses?"+params.Encode(), &resp); err != nil { - return nil, err - } - if len(resp) == 0 { - break - } - statuses = append(statuses, resp...) + return params + }); err != nil { + return nil, err } ignoreList := []string{ |