From 42cb6fa56017918812b755bacb391c43cf7c4fe6 Mon Sep 17 00:00:00 2001 From: arreyder Date: Mon, 22 Dec 2025 21:09:43 -0600 Subject: [PATCH 1/2] cache getFinishedSync result to eliminate N+1 query pattern --- pkg/dotc1z/c1file.go | 5 +++++ pkg/dotc1z/sql_helpers.go | 12 ++---------- pkg/dotc1z/sync_runs.go | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/pkg/dotc1z/c1file.go b/pkg/dotc1z/c1file.go index a29d0b449..5de571943 100644 --- a/pkg/dotc1z/c1file.go +++ b/pkg/dotc1z/c1file.go @@ -46,6 +46,11 @@ type C1File struct { readOnly bool encoderConcurrency int + // Cached sync run for listConnectorObjects (avoids N+1 queries) + cachedViewSyncRun *syncRun + cachedViewSyncOnce sync.Once + cachedViewSyncErr error + // Slow query tracking slowQueryLogTimes map[string]time.Time slowQueryLogTimesMu sync.Mutex diff --git a/pkg/dotc1z/sql_helpers.go b/pkg/dotc1z/sql_helpers.go index 8b6e0bfc9..9c3efa344 100644 --- a/pkg/dotc1z/sql_helpers.go +++ b/pkg/dotc1z/sql_helpers.go @@ -194,20 +194,12 @@ func listConnectorObjects[T proto.Message](ctx context.Context, c *C1File, table case reqSyncID != "": q = q.Where(goqu.C("sync_id").Eq(reqSyncID)) default: - var latestSyncRun *syncRun - var err error - latestSyncRun, err = c.getFinishedSync(ctx, 0, connectorstore.SyncTypeFull) + // Use cached sync run to avoid N+1 queries during pagination + latestSyncRun, err := c.getCachedViewSyncRun(ctx) if err != nil { return nil, "", err } - if latestSyncRun == nil { - latestSyncRun, err = c.getLatestUnfinishedSync(ctx, connectorstore.SyncTypeAny) - if err != nil { - return nil, "", err - } - } - if latestSyncRun != nil { q = q.Where(goqu.C("sync_id").Eq(latestSyncRun.ID)) } diff --git a/pkg/dotc1z/sync_runs.go b/pkg/dotc1z/sync_runs.go index cadf9a50a..75da3a1bb 100644 --- a/pkg/dotc1z/sync_runs.go +++ b/pkg/dotc1z/sync_runs.go @@ -95,6 +95,26 @@ type syncRun struct { ParentSyncID string } +// getCachedViewSyncRun returns the cached sync run for read operations. +// This avoids N+1 queries when paginating through listConnectorObjects. +// The result is computed once and cached for the lifetime of the C1File. +func (c *C1File) getCachedViewSyncRun(ctx context.Context) (*syncRun, error) { + c.cachedViewSyncOnce.Do(func() { + // First try to get a finished full sync + c.cachedViewSyncRun, c.cachedViewSyncErr = c.getFinishedSync(ctx, 0, connectorstore.SyncTypeFull) + if c.cachedViewSyncErr != nil { + return + } + + // If no finished sync, try to get an unfinished one + if c.cachedViewSyncRun == nil { + c.cachedViewSyncRun, c.cachedViewSyncErr = c.getLatestUnfinishedSync(ctx, connectorstore.SyncTypeAny) + } + }) + + return c.cachedViewSyncRun, c.cachedViewSyncErr +} + func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connectorstore.SyncType) (*syncRun, error) { ctx, span := tracer.Start(ctx, "C1File.getLatestUnfinishedSync") defer span.End() From a87aecaff2502f1ddeec5ff8415118a38c69cc0a Mon Sep 17 00:00:00 2001 From: arreyder Date: Mon, 22 Dec 2025 21:26:29 -0600 Subject: [PATCH 2/2] add tracing --- pkg/dotc1z/sync_runs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/dotc1z/sync_runs.go b/pkg/dotc1z/sync_runs.go index 75da3a1bb..c8a7b11a6 100644 --- a/pkg/dotc1z/sync_runs.go +++ b/pkg/dotc1z/sync_runs.go @@ -99,6 +99,9 @@ type syncRun struct { // This avoids N+1 queries when paginating through listConnectorObjects. // The result is computed once and cached for the lifetime of the C1File. func (c *C1File) getCachedViewSyncRun(ctx context.Context) (*syncRun, error) { + ctx, span := tracer.Start(ctx, "C1File.getCachedViewSyncRun") + defer span.End() + c.cachedViewSyncOnce.Do(func() { // First try to get a finished full sync c.cachedViewSyncRun, c.cachedViewSyncErr = c.getFinishedSync(ctx, 0, connectorstore.SyncTypeFull)