imworkingon: Properly do pagination with the GH APIs

3 files changed, 43 insertions, 11 deletions

diff --git a/cmd/generate/httpcache.go b/cmd/generate/httpcache.go
index cff0c7c..04762e3 100644
--- a/cmd/generate/httpcache.go
+++ b/cmd/generate/httpcache.go
@@ -14,6 +14,7 @@ var httpCache = map[string]string{}
 
 func httpGet(u string) (string, error) {
 	if cache, ok := httpCache[u]; ok {
+		fmt.Printf("CACHE-GET %q\n", u)
 		return cache, nil
 	}
 	if err := os.Mkdir(".http-cache", 0777); err != nil && !os.IsExist(err) {
@@ -22,6 +23,7 @@ func httpGet(u string) (string, error) {
 	cacheFile := filepath.Join(".http-cache", url.QueryEscape(u))
 	if bs, err := os.ReadFile(cacheFile); err == nil {
 		httpCache[u] = string(bs)
+		fmt.Printf("CACHE-GET %q\n", u)
 		return httpCache[u], nil
 	} else if !os.IsNotExist(err) {
 		return "", err
@@ -57,3 +59,37 @@ func httpGetJSON(u string, out any) error {
 	}
 	return json.Unmarshal([]byte(str), out)
 }
+
+func httpGetPaginatedJSON[T any](uStr string, out *[]T, pageFn func(i int) url.Values) error {
+	u, err := url.Parse(uStr)
+	if err != nil {
+		return err
+	}
+	query := u.Query()
+
+	for i := 0; true; i++ {
+		pageParams := pageFn(i)
+		for k, v := range pageParams {
+			query[k] = v
+		}
+
+		u.RawQuery = query.Encode()
+		var resp []T
+		if err := httpGetJSON(u.String(), &resp); err != nil {
+			return err
+		}
+		fmt.Printf(" -> %d records\n", len(resp))
+		if len(resp) == 0 {
+			break
+		}
+		*out = append(*out, resp...)
+	}
+
+	return nil
+}
+
+func githubPagination(i int) url.Values {
+	params := make(url.Values)
+	params.Set("page", fmt.Sprintf("%v", i+1))
+	return params
+}
diff --git a/cmd/generate/src_contribs.go b/cmd/generate/src_contribs.go
index 4e65665..6db6764 100644
--- a/cmd/generate/src_contribs.go
+++ b/cmd/generate/src_contribs.go
@@ -292,7 +292,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
 					HTMLURL string `json:"html_url"`
 				} `json:"user"`
 			}
-			if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/comments", &comments); err != nil {
+			if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/issues/"+prnum+"/comments", &comments, githubPagination); err != nil {
 				return time.Time{}, User{}, err
 			}
 			for _, comment := range comments {
@@ -312,7 +312,7 @@ func (c Contribution) fetchLastUpdated() (time.Time, User, error) {
 					HTMLURL string `json:"html_url"`
 				} `json:"user"`
 			}
-			if err := httpGetJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum+"/comments", &reviewComments); err != nil {
+			if err := httpGetPaginatedJSON("https://api.github.com/repos/"+user+"/"+repo+"/pulls/"+prnum+"/comments", &reviewComments, githubPagination); err != nil {
 				return time.Time{}, User{}, err
 			}
 			for _, comment := range reviewComments {
diff --git a/cmd/generate/src_mastodon.go b/cmd/generate/src_mastodon.go
index 42ae8b2..b4b54a8 100644
--- a/cmd/generate/src_mastodon.go
+++ b/cmd/generate/src_mastodon.go
@@ -22,22 +22,18 @@ func ReadStandups(server, username string) ([]*MastodonStatus, error) {
 	if err := httpGetJSON(server+"/api/v1/accounts/lookup?acct="+username, &account); err != nil {
 		return nil, err
 	}
+
 	var statuses []*MastodonStatus
-	for {
+	if err := httpGetPaginatedJSON(server+"/api/v1/accounts/"+account.ID+"/statuses", &statuses, func(_ int) url.Values {
 		params := make(url.Values)
 		params.Set("tagged", "DailyStandUp")
 		params.Set("exclude_reblogs", "true")
 		if len(statuses) > 0 {
 			params.Set("max_id", statuses[len(statuses)-1].ID)
 		}
-		var resp []*MastodonStatus
-		if err := httpGetJSON(server+"/api/v1/accounts/"+account.ID+"/statuses?"+params.Encode(), &resp); err != nil {
-			return nil, err
-		}
-		if len(resp) == 0 {
-			break
-		}
-		statuses = append(statuses, resp...)
+		return params
+	}); err != nil {
+		return nil, err
 	}
 
 	ignoreList := []string{