summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLuke T. Shumaker <lukeshu@lukeshu.com>2024-05-04 13:31:22 -0400
committerLuke T. Shumaker <lukeshu@lukeshu.com>2024-05-04 13:31:22 -0400
commit618cfc7cddf91d01fe6636c81809a4f90b40c00c (patch)
treea08f5f0765f4cbb1c8901f2251bd7d8db264aeae
parent6e29a14f10b51c6f7005b4bfb352c57d5dda0ef7 (diff)
imworkingon: Properly do pagination with the GH APIs
-rw-r--r--cmd/generate/httpcache.go36
-rw-r--r--cmd/generate/src_contribs.go4
-rw-r--r--cmd/generate/src_mastodon.go14
3 files changed, 43 insertions, 11 deletions
diff --git a/cmd/generate/httpcache.go b/cmd/generate/httpcache.go
index cff0c7c..04762e3 100644
--- a/cmd/generate/httpcache.go
+++ b/cmd/generate/httpcache.go
@@ -14,6 +14,7 @@ var httpCache = map[string]string{}
func httpGet(u string) (string, error) {
if cache, ok := httpCache[u]; ok {
+ fmt.Printf("CACHE-GET %q\n", u)
return cache, nil
}
if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) {
@@ -22,6 +23,7 @@ func httpGet(u string) (string, error) {
cacheFile := filepath.Join(".http-cache", url.QueryEscape(u))
if bs, err := os.ReadFile(cacheFile); err == nil {
httpCache[u] = string(bs)
+ fmt.Printf("CACHE-GET %q\n", u)
return httpCache[u], nil
} else if !os.IsNotExist(err) {
return "", err
@@ -57,3 +59,37 @@ func httpGetJSON(u string, out any) error {
}
return json.Unmarshal([]byte(str), out)
}
+
+func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.Values) error {
+ u, err := url.Parse(uStr)
+ if err != nil {
+ return err
+ }
+ query := u.Query()
+
+ for i := 0; true; i++ {
+ pageParams := pageFn(i)
+ for k, v := range pageParams {
+ query[k] = v
+ }
+
+ u.RawQuery = query.Encode()
+ var resp []T
+ if err := httpGetJSON(u.String(), &resp); err != nil {
+ return err
+ }
+ fmt.Printf(" -> %d records\n", len(resp))
+ if len(resp) == 0 {
+ break
+ }
+ *out = append(*out, resp...)
+ }
+
+ return nil
+}
+
+func githubPagination(i int) url.Values {
+ params := make(url.Values)
+ params.Set("page", fmt.Sprintf("%v", i+1))
+ return params
+}
diff --git a/cmd/generate/src_contribs.go b/cmd/generate/src_contribs.go
index 4e65665..6db6764 100644
--- a/cmd/generate/src_contribs.go
+++ b/cmd/generate/src_contribs.go
@@ -292,7 +292,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
HTMLURL string `json:"html_url"`
} `json:"user"`
}
- if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/comments", &comments); err != nil {
+ if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/comments", &comments, githubPagination); err != nil {
return time.Time{}, User{}, err
}
for _, comment := range comments {
@@ -312,7 +312,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
HTMLURL string `json:"html_url"`
} `json:"user"`
}
- if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum+"/comments", &reviewComments); err != nil {
+ if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum+"/comments", &reviewComments, githubPagination); err != nil {
return time.Time{}, User{}, err
}
for _, comment := range reviewComments {
diff --git a/cmd/generate/src_mastodon.go b/cmd/generate/src_mastodon.go
index 42ae8b2..b4b54a8 100644
--- a/cmd/generate/src_mastodon.go
+++ b/cmd/generate/src_mastodon.go
@@ -22,22 +22,18 @@ func ReadStandups(server, username string) ([]*MastodonStatus, error) {
if err := httpGetJSON(server+"/api/v1/accounts/lookup?acct="+username, &account); err != nil {
return nil, err
}
+
var statuses []*MastodonStatus
- for {
+ if err := httpGetPaginatedJSON(server+"/api/v1/accounts/"+account.ID+"/statuses", &statuses, func(_ int) url.Values {
params := make(url.Values)
params.Set("tagged", "DailyStandUp")
params.Set("exclude_reblogs", "true")
if len(statuses) > 0 {
params.Set("max_id", statuses[len(statuses)-1].ID)
}
- var resp []*MastodonStatus
- if err := httpGetJSON(server+"/api/v1/accounts/"+account.ID+"/statuses?"+params.Encode(), &resp); err != nil {
- return nil, err
- }
- if len(resp) == 0 {
- break
- }
- statuses = append(statuses, resp...)
+ return params
+ }); err != nil {
+ return nil, err
}
ignoreList := []string{