fix: confluence page-body limit handling and selector warnings (#355)

What was wrong

- When a page body was bigger than --max-page-body-megabytes, the client
skipped it before the plugin saw it.
- Because of that, the plugin never marked the page ID / space ID as
“seen”.
- Result: valid page IDs/space IDs could show up in the “don’t exist /
no access” warning.

What changed

- Page-body size limit is now enforced in the plugin, not inside the
client visitor.
- We always mark the page as “seen” (returnedPageIDs / returnedSpaceIDs)
before deciding to skip its content.
- Warning about missing selectors now only reports IDs/keys that truly
returned no pages or were invalid.

Behaviour

- Big pages are skipped for content, but not treated as
missing/inaccessible.
- Total scan / per-response limits behaviour is unchanged.

Extra

- Updated sort parameter to use modified date instead of created date

Author: cx-leonardo-fontes

plugins/confluence.go

@@ -228,7 +228,8 @@ func (p *ConfluencePlugin) initialize(base, username, token string) error {
 		token,
 		WithMaxAPIResponseBytes(p.maxAPIResponseBytes),
 		WithMaxTotalScanBytes(p.maxTotalScanBytes),
-		WithMaxPageBodyBytes(p.maxPageBodyBytes),
+		// Page-body size limit is enforced in the plugin layer so that
+		// large-but-accessible pages still count as "seen".
 	)
 	if err != nil {
 		return err
@@ -383,6 +384,26 @@ func (p *ConfluencePlugin) emitInChunks(page *Page) error {
 	}
 }
 
+// shouldSkipPageBody applies the per-page body size limit (if configured).
+// It logs a warning when the limit is exceeded and returns true to indicate
+// that the page's content (and history) should be skipped.
+func (p *ConfluencePlugin) shouldSkipPageBody(page *Page) bool {
+	if p.maxPageBodyBytes == 0 || page == nil || page.Body.Storage == nil {
+		return false
+	}
+
+	bodySize := int64(len(page.Body.Storage.Value))
+	if bodySize <= p.maxPageBodyBytes {
+		return false
+	}
+
+	log.Warn().
+		Str("page_id", page.ID).
+		Int64("body_bytes", bodySize).
+		Msg("Skipping page content because the Confluence page body exceeded the configured size limit.")
+	return true
+}
+
 // emitUniquePage emits the current version of a page (and, if enabled, its historical versions)
 // ensuring each page ID is emitted only once. Also records the page SpaceID for selector warnings.
 func (p *ConfluencePlugin) emitUniquePage(ctx context.Context, page *Page) error {
@@ -395,6 +416,12 @@ func (p *ConfluencePlugin) emitUniquePage(ctx context.Context, page *Page) error
 		p.returnedSpaceIDs[page.SpaceID] = struct{}{}
 	}
 
+	// Enforce page body size limit after marking the page as seen, so that
+	// large-but-accessible pages are not misreported as missing or unauthorized.
+	if p.shouldSkipPageBody(page) {
+		return nil
+	}
+
 	// current version
 	if err := p.emitInChunks(page); err != nil {
 		return err
@@ -420,6 +447,9 @@ func (p *ConfluencePlugin) emitHistory(ctx context.Context, page *Page) error {
 		if err != nil {
 			return err
 		}
+		if p.shouldSkipPageBody(versionedPage) {
+			return nil
+		}
 		return p.emitInChunks(versionedPage)
 	})
 }

plugins/confluence_client.go

@@ -12,8 +12,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -72,7 +70,6 @@ type httpConfluenceClient struct {
 	// Optional limits (0 means "no limit").
 	maxAPIResponseBytes int64
 	maxTotalScanBytes   int64
-	maxPageBodyBytes    int64
 
 	totalScanBytes int64
 
@@ -107,18 +104,6 @@ func WithMaxTotalScanBytes(n int64) ConfluenceClientOption {
 	}
 }
 
-// WithMaxPageBodyBytes sets an optional limit in bytes for individual page bodies.
-// Pages whose storage body exceeds this limit are skipped and logged as warnings.
-// A value of 0 disables the limit. Negative values are treated as 0.
-func WithMaxPageBodyBytes(n int64) ConfluenceClientOption {
-	if n < 0 {
-		n = 0
-	}
-	return func(c *httpConfluenceClient) {
-		c.maxPageBodyBytes = n
-	}
-}
-
 // NewConfluenceClient constructs a ConfluenceClient for the given base input URL.
 // Behavior:
 //   - Normalizes base wiki URL to "https://{host}/wiki".
@@ -243,7 +228,7 @@ func (c *httpConfluenceClient) walkPagesWithFilter(
 	q := apiURL.Query()
 	q.Set("limit", strconv.Itoa(limit))
 	q.Set("body-format", "storage")
-	q.Set("sort", "-created-date") // Newest created date to oldest
+	q.Set("sort", "-modified-date") // Newest modified date to oldest
 
 	if filterKey != "" && len(filterValues) > 0 {
 		q.Set(filterKey, strings.Join(filterValues, ","))
@@ -383,9 +368,8 @@ func (c *httpConfluenceClient) walkPagesPaginated(
 		linkNext := nextURLFromLinkHeader(headers)
 
 		reader := c.wrapWithCountingReader(rc)
-		limitedVisit := c.wrapWithPageBodyLimit(visit)
 
-		bodyNext, decodeErr := streamPagesFromBody(reader, limitedVisit)
+		bodyNext, decodeErr := streamPagesFromBody(reader, visit)
 		closeErr := rc.Close()
 
 		if decodeErr != nil {
@@ -435,30 +419,6 @@ func (c *httpConfluenceClient) wrapWithCountingReader(rc io.ReadCloser) io.Reade
 	}
 }
 
-// wrapWithPageBodyLimit decorates the visit callback to skip pages whose
-// storage body exceeds maxPageBodyBytes, logging a warning when that happens.
-func (c *httpConfluenceClient) wrapWithPageBodyLimit(
-	visit func(*Page) error,
-) func(*Page) error {
-	if c.maxPageBodyBytes == 0 {
-		return visit
-	}
-
-	return func(p *Page) error {
-		if p.Body.Storage != nil {
-			bodySize := int64(len(p.Body.Storage.Value))
-			if bodySize > c.maxPageBodyBytes {
-				log.Warn().
-					Str("page_id", p.ID).
-					Int64("body_bytes", bodySize).
-					Msg("Skipping page content because the Confluence page body exceeded the configured size limit.")
-				return nil
-			}
-		}
-		return visit(p)
-	}
-}
-
 // handleStreamError centralizes handling for decode/close errors and decides
 // whether to stop, continue with a new cursor, or return an error.
 func (c *httpConfluenceClient) handleStreamError(
@@ -637,14 +597,14 @@ type PageBody struct {
 
 // Page represents a Confluence page.
 type Page struct {
-	ID        string            `json:"id"`
-	Status    string            `json:"status"`
-	Title     string            `json:"title"`
-	SpaceID   string            `json:"spaceId"`
-	Type      string            `json:"type"`
-	Body      PageBody          `json:"body"`
-	Links     map[string]string `json:"_links"`
-	Version   PageVersion       `json:"version"`
+	ID      string            `json:"id"`
+	Status  string            `json:"status"`
+	Title   string            `json:"title"`
+	SpaceID string            `json:"spaceId"`
+	Type    string            `json:"type"`
+	Body    PageBody          `json:"body"`
+	Links   map[string]string `json:"_links"`
+	Version PageVersion       `json:"version"`
 }
 
 // Space represents a Confluence space.

plugins/confluence_client_test.go

@@ -9,7 +9,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -1480,39 +1479,6 @@ func TestWithMaxTotalScanBytes(t *testing.T) {
 	}
 }
 
-func TestWithMaxPageBodyBytes(t *testing.T) {
-	tests := []struct {
-		name  string
-		input int64
-		want  int64
-	}{
-		{
-			name:  "zero disables limit",
-			input: 0,
-			want:  0,
-		},
-		{
-			name:  "positive sets limit",
-			input: 1024,
-			want:  1024,
-		},
-		{
-			name:  "negative treated as zero",
-			input: -42,
-			want:  0,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			c := &httpConfluenceClient{}
-			WithMaxPageBodyBytes(tt.input)(c)
-
-			assert.Equal(t, tt.want, c.maxPageBodyBytes)
-		})
-	}
-}
-
 func TestCountingReader(t *testing.T) {
 	tests := []struct {
 		name            string
@@ -1615,109 +1581,6 @@ func TestWrapWithCountingReader(t *testing.T) {
 	}
 }
 
-func TestWrapWithPageBodyLimit(t *testing.T) {
-	mkStorage := func(content string) *struct {
-		Value string `json:"value"`
-	} {
-		return &struct {
-			Value string `json:"value"`
-		}{Value: content}
-	}
-
-	tests := []struct {
-		name              string
-		maxPageBodyBytes  int64
-		page              *Page
-		visitErr          error
-		expectVisitCalled bool
-		expectedErr       error
-		expectSameFunc    bool
-	}{
-		{
-			name:             "no limit uses original visitor",
-			maxPageBodyBytes: 0,
-			page: &Page{
-				ID: "P1",
-				Body: PageBody{
-					Storage: mkStorage("some content that can be any size"),
-				},
-			},
-			visitErr:          assert.AnError,
-			expectVisitCalled: true,
-			expectedErr:       assert.AnError,
-			expectSameFunc:    true,
-		},
-		{
-			name:             "body below limit calls visitor",
-			maxPageBodyBytes: 10,
-			page: &Page{
-				ID: "P2",
-				Body: PageBody{
-					Storage: mkStorage("small"), // len = 5
-				},
-			},
-			visitErr:          assert.AnError,
-			expectVisitCalled: true,
-			expectedErr:       assert.AnError,
-			expectSameFunc:    false,
-		},
-		{
-			name:             "body above limit skips visitor",
-			maxPageBodyBytes: 5,
-			page: &Page{
-				ID: "P3",
-				Body: PageBody{
-					Storage: mkStorage("too-large-body"), // len > 5
-				},
-			},
-			visitErr:          assert.AnError, // should never be returned
-			expectVisitCalled: false,
-			expectedErr:       nil,
-			expectSameFunc:    false,
-		},
-		{
-			name:             "nil storage calls visitor",
-			maxPageBodyBytes: 5,
-			page: &Page{
-				ID: "P4",
-				Body: PageBody{
-					Storage: nil,
-				},
-			},
-			visitErr:          assert.AnError,
-			expectVisitCalled: true,
-			expectedErr:       assert.AnError,
-			expectSameFunc:    false,
-		},
-	}
-
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			c := &httpConfluenceClient{
-				maxPageBodyBytes: tc.maxPageBodyBytes,
-			}
-
-			visited := false
-			visitErr := tc.visitErr
-
-			visit := func(p *Page) error {
-				visited = true
-				return visitErr
-			}
-
-			wrapped := c.wrapWithPageBodyLimit(visit)
-
-			sameFunc := reflect.ValueOf(visit).Pointer() == reflect.ValueOf(wrapped).Pointer()
-
-			err := wrapped(tc.page)
-
-			assert.Equal(t, tc.expectSameFunc, sameFunc)
-			assert.Equal(t, tc.expectVisitCalled, visited)
-			assert.ErrorIs(t, err, tc.expectedErr)
-		})
-	}
-}
-
 func TestHandleStreamError(t *testing.T) {
 	base, _ := url.Parse("https://x.test/wiki/api/v2/pages?limit=10")