add pagination and timeout

Author: eumel8

README.md

@@ -14,6 +14,14 @@ Requires following secrets in your Github repository:
   WALLABAG_PASSWORD: Wallabag password
 ```
 
+Optional environment variables for timeout configuration:
+
+```
+  WALLABAG_API_TIMEOUT: Timeout for Wallabag API requests in seconds (default: 30)
+  HTTP_CHECK_TIMEOUT: Timeout for HTTP link checking in seconds (default: 15)
+  TLS_HANDSHAKE_TIMEOUT: Timeout for TLS handshake in seconds (default: 10)
+```
+
 Refer to the [Wallabag Documentation](https://doc.wallabag.org/developer/api/oauth/) to create API credentials on your instance.
 
 After run you can see job output for results or check tagged articles in your Wallabag instance with `dead`
@@ -44,7 +52,7 @@ jobs:
 
 ## tipps & tricks
 
-* wallabag-checklinks is limited to 10.000 entries
+* wallabag-checklinks now uses pagination to fetch all entries without limit (previously limited to 10,000 entries)
 * your weblinks will exposed if the Github repo is public, be careful. Use private repo or use wallabag-checklinks locally, look at the [release page](https://github.com/eumel8/wallabag-checklinks/releases) for binaries.
 
 ## credits

main.go

@@ -6,6 +6,7 @@ import (
     "log"
     "net/http"
     "os"
+    "strconv"
     "time"
 
     "github.com/go-resty/resty/v2"
@@ -24,6 +25,9 @@ type AuthResponse struct {
 }
 
 type EntryList struct {
+    Page  int `json:"page"`
+    Pages int `json:"pages"`
+    Total int `json:"total"`
     Embedded struct {
         Items []struct {
             ID   int      `json:"id"`
@@ -51,6 +55,27 @@ func getEnvOrFail(key string) string {
     return value
 }
 
+func getEnvOrDefault(key string, defaultValue string) string {
+    value := os.Getenv(key)
+    if value == "" {
+        return defaultValue
+    }
+    return value
+}
+
+func getEnvIntOrDefault(key string, defaultValue int) int {
+    value := os.Getenv(key)
+    if value == "" {
+        return defaultValue
+    }
+    intValue, err := strconv.Atoi(value)
+    if err != nil {
+        log.Printf("⚠️  Invalid value for %s, using default: %d", key, defaultValue)
+        return defaultValue
+    }
+    return intValue
+}
+
 func getAccessToken(client *resty.Client) (string, error) {
     resp, err := client.R().
         SetFormData(map[string]string{
@@ -77,41 +102,58 @@ func getAccessToken(client *resty.Client) (string, error) {
 }
 
 func getEntries(client *resty.Client, token string) ([]Entry, error) {
-    var entries EntryList
+    var result []Entry
+    page := 1
+    perPage := 100 // Reasonable page size
+
+    for {
+        var entries EntryList
+
+        _, err := client.R().
+            SetHeader("Authorization", "Bearer "+token).
+            SetQueryParams(map[string]string{
+                "page":    fmt.Sprintf("%d", page),
+                "perPage": fmt.Sprintf("%d", perPage),
+            }).
+            SetResult(&entries).
+            Get(wallabagURL + "/api/entries.json")
+
+        if err != nil {
+            return nil, err
+        }
 
-    _, err := client.R().
-        SetHeader("Authorization", "Bearer "+token).
-        SetQueryParams(map[string]string{
-            "perPage": "10000", // Adjust for more
-        }).
-        SetResult(&entries).
-        Get(wallabagURL + "/api/entries.json")
+        // Process entries from this page
+        for _, item := range entries.Embedded.Items {
+            tags := []string{}
+            for _, tag := range item.Tags {
+                tags = append(tags, tag.Label)
+            }
+            result = append(result, Entry{
+                ID:   item.ID,
+                URL:  item.URL,
+                Tags: tags,
+            })
+        }
 
-    if err != nil {
-        return nil, err
-    }
+        // Log progress
+        fmt.Printf("📥 Fetched page %d/%d (%d entries so far)\n", page, entries.Pages, len(result))
 
-    var result []Entry
-    for _, item := range entries.Embedded.Items {
-        tags := []string{}
-        for _, tag := range item.Tags {
-            tags = append(tags, tag.Label)
+        // Check if we've fetched all pages
+        if page >= entries.Pages {
+            break
         }
-        result = append(result, Entry{
-            ID:   item.ID,
-            URL:  item.URL,
-            Tags: tags,
-        })
+
+        page++
     }
 
     return result, nil
 }
 
-func checkURL(url string) int {
+func checkURL(url string, timeout int, tlsTimeout int) int {
     client := &http.Client{
-        Timeout: 15 * time.Second,
+        Timeout: time.Duration(timeout) * time.Second,
         Transport: &http.Transport{
-            TLSHandshakeTimeout: 10 * time.Second,
+            TLSHandshakeTimeout: time.Duration(tlsTimeout) * time.Second,
             IdleConnTimeout:     90 * time.Second,
         },
     }
@@ -215,8 +257,13 @@ func containsTag(tags []string, target string) bool {
 }
 
 func main() {
+    // Get configurable timeouts with defaults
+    apiTimeout := getEnvIntOrDefault("WALLABAG_API_TIMEOUT", 30)
+    httpTimeout := getEnvIntOrDefault("HTTP_CHECK_TIMEOUT", 15)
+    tlsTimeout := getEnvIntOrDefault("TLS_HANDSHAKE_TIMEOUT", 10)
+
     restyClient := resty.New().
-        SetTimeout(30 * time.Second).
+        SetTimeout(time.Duration(apiTimeout) * time.Second).
         SetRetryCount(3).
         SetRetryWaitTime(5 * time.Second)
 
@@ -233,7 +280,7 @@ func main() {
     fmt.Printf("🔎 Checking %d URLs...\n\n", len(entries))
 
     for _, entry := range entries {
-        status := checkURL(entry.URL)
+        status := checkURL(entry.URL, httpTimeout, tlsTimeout)
         hasDeadTag := containsTag(entry.Tags, "dead")
 
         if status == 0 {