Adding pagination to get records func (#21)

* adding pagination for get records

* setting maxPageSize as const

---------

Co-authored-by: Ray Janoka <[email protected]>

Author: rayjanoka

provider.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"net/url"
 	"sync"
 
 	"github.com/libdns/libdns"
@@ -14,7 +15,7 @@ type HTTPClient interface {
 }
 
 // Provider implements the libdns interfaces for Cloudflare.
-// TODO: Support pagination and retries, handle rate limits.
+// TODO: Support retries and handle rate limits.
 type Provider struct {
 	// API tokens are used for authentication. Make sure to use
 	// scoped API **tokens**, NOT a global API **key**.
@@ -36,20 +37,39 @@ func (p *Provider) GetRecords(ctx context.Context, zone string) ([]libdns.Record
 		return nil, err
 	}
 
-	reqURL := fmt.Sprintf("%s/zones/%s/dns_records", baseURL, zoneInfo.ID)
-	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
-	if err != nil {
-		return nil, err
-	}
+	page := 1
+	const maxPageSize = 100
 
-	var result []cfDNSRecord
-	_, err = p.doAPIRequest(req, &result)
-	if err != nil {
-		return nil, err
+	var allRecords []cfDNSRecord
+	for {
+		qs := make(url.Values)
+		qs.Set("page", fmt.Sprintf("%d", page))
+		qs.Set("per_page", fmt.Sprintf("%d", maxPageSize))
+		reqURL := fmt.Sprintf("%s/zones/%s/dns_records?%s", baseURL, zoneInfo.ID, qs.Encode())
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		var pageRecords []cfDNSRecord
+		response, err := p.doAPIRequest(req, &pageRecords)
+		if err != nil {
+			return nil, err
+		}
+
+		allRecords = append(allRecords, pageRecords...)
+
+		lastPage := (response.ResultInfo.TotalCount + response.ResultInfo.PerPage - 1) / response.ResultInfo.PerPage
+		if response.ResultInfo == nil || page >= lastPage || len(pageRecords) == 0 {
+			break
+		}
+
+		page++
 	}
 
-	recs := make([]libdns.Record, 0, len(result))
-	for _, rec := range result {
+	recs := make([]libdns.Record, 0, len(allRecords))
+	for _, rec := range allRecords {
 		libdnsRec, err := rec.libdnsRecord(zone)
 		if err != nil {
 			return nil, fmt.Errorf("parsing Cloudflare DNS record %+v: %v", rec, err)