Create a "diff" sync entry for the items added between syncs (#366)

* Working on diff

* Getting there

* Nearly there?

* I think it'll work

* I think e2e but unsure

* Feels like it should be working but it's not saving to disk and idk why

* twerks

* Renaming some things

* Diffs should be considered partial syncs

* Comments

* Export target none

* silly wabbit coding is for engineers

* DB updated at the end of the function

* Rebase and fix compiler error.

---------

Co-authored-by: Geoff Greer <geoff@greer.fm>

Author: loganintech

pb/c1/connectorapi/baton/v1/baton.pb.go

@@ -224,6 +224,15 @@ func MakeMainCommand[T field.Configurable](
 					connectorrunner.WithTargetedSyncResourceIDs(v.GetStringSlice("sync-resources")),
 					connectorrunner.WithOnDemandSync(v.GetString("file")),
 				)
+			case v.GetBool("diff-syncs"):
+				opts = append(opts,
+					connectorrunner.WithDiffSyncs(
+						v.GetString("file"),
+						v.GetString("base-sync-id"),
+						v.GetString("applied-sync-id"),
+					),
+				)
+
 			default:
 				opts = append(opts, connectorrunner.WithOnDemandSync(v.GetString("file")))
 			}

pb/c1/connectorapi/baton/v1/baton.pb.validate.go

@@ -282,6 +282,11 @@ type rotateCredentialsConfig struct {
 type eventStreamConfig struct {
 }
 
+type syncDifferConfig struct {
+	baseSyncID    string
+	appliedSyncID string
+}
+
 type runnerConfig struct {
 	rlCfg                               *ratelimitV1.RateLimiterConfig
 	rlDescriptors                       []*ratelimitV1.RateLimitDescriptors_Entry
@@ -303,6 +308,7 @@ type runnerConfig struct {
 	bulkCreateTicketConfig              *bulkCreateTicketConfig
 	listTicketSchemasConfig             *listTicketSchemasConfig
 	getTicketConfig                     *getTicketConfig
+	syncDifferConfig                    *syncDifferConfig
 	skipFullSync                        bool
 	targetedSyncResourceIDs             []string
 	externalResourceC1Z                 string
@@ -555,6 +561,18 @@ func WithExternalResourceEntitlementFilter(entitlementId string) Option {
 	}
 }
 
+func WithDiffSyncs(c1zPath string, baseSyncID string, newSyncID string) Option {
+	return func(ctx context.Context, cfg *runnerConfig) error {
+		cfg.onDemand = true
+		cfg.c1zPath = c1zPath
+		cfg.syncDifferConfig = &syncDifferConfig{
+			baseSyncID:    baseSyncID,
+			appliedSyncID: newSyncID,
+		}
+		return nil
+	}
+}
+
 // NewConnectorRunner creates a new connector runner.
 func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Option) (*connectorRunner, error) {
 	runner := &connectorRunner{}
@@ -635,6 +653,8 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op
 			tm = local.NewGetTicket(ctx, cfg.getTicketConfig.ticketID)
 		case cfg.bulkCreateTicketConfig != nil:
 			tm = local.NewBulkTicket(ctx, cfg.bulkCreateTicketConfig.templatePath)
+		case cfg.syncDifferConfig != nil:
+			tm = local.NewDiffer(ctx, cfg.c1zPath, cfg.syncDifferConfig.baseSyncID, cfg.syncDifferConfig.appliedSyncID)
 		default:
 			tm, err = local.NewSyncer(ctx, cfg.c1zPath,
 				local.WithTmpDir(cfg.tempDir),

pkg/cli/commands.go

@@ -0,0 +1,113 @@
+package dotc1z
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/doug-martin/goqu/v9"
+	"github.com/segmentio/ksuid"
+)
+
+func (c *C1File) GenerateSyncDiff(ctx context.Context, baseSyncID string, appliedSyncID string) (string, error) {
+	// Validate that both sync runs exist
+	baseSync, err := c.getSync(ctx, baseSyncID)
+	if err != nil {
+		return "", err
+	}
+	if baseSync == nil {
+		return "", fmt.Errorf("generate-diff: base sync not found")
+	}
+
+	newSync, err := c.getSync(ctx, appliedSyncID)
+	if err != nil {
+		return "", err
+	}
+	if newSync == nil {
+		return "", fmt.Errorf("generate-diff: new sync not found")
+	}
+
+	// Generate a new unique ID for the diff sync
+	diffSyncID := ksuid.New().String()
+
+	if err := c.insertSyncRun(ctx, diffSyncID, SyncTypePartial, baseSyncID); err != nil {
+		return "", err
+	}
+
+	for _, t := range allTableDescriptors {
+		if strings.Contains(t.Name(), syncRunsTableName) {
+			continue
+		}
+
+		q, args, err := c.diffTableQuery(t, baseSyncID, appliedSyncID, diffSyncID)
+		if err != nil {
+			return "", err
+		}
+		_, err = c.db.ExecContext(ctx, q, args...)
+		if err != nil {
+			return "", err
+		}
+		c.dbUpdated = true
+	}
+
+	if err := c.endSyncRun(ctx, diffSyncID); err != nil {
+		return "", err
+	}
+
+	return diffSyncID, nil
+}
+
+func (c *C1File) diffTableQuery(table tableDescriptor, baseSyncID, appliedSyncID, newSyncID string) (string, []any, error) {
+	// Define the columns to select based on the table name
+	columns := []interface{}{
+		"external_id",
+		"data",
+		"sync_id",
+		"discovered_at",
+	}
+
+	tableName := table.Name()
+	// Add table-specific columns
+	switch {
+	case strings.Contains(tableName, resourcesTableName):
+		columns = append(columns, "resource_type_id", "parent_resource_type_id", "parent_resource_id")
+	case strings.Contains(tableName, resourceTypesTableName):
+		// Nothing new to add here
+	case strings.Contains(tableName, grantsTableName):
+		columns = append(columns, "resource_type_id", "resource_id", "entitlement_id", "principal_resource_type_id", "principal_resource_id")
+	case strings.Contains(tableName, entitlementsTableName):
+		columns = append(columns, "resource_type_id", "resource_id")
+	case strings.Contains(tableName, assetsTableName):
+		columns = append(columns, "content_type")
+	}
+
+	// Build the subquery to find external_ids in the base sync
+	subquery := c.db.Select("external_id").
+		From(tableName).
+		Where(goqu.C("sync_id").Eq(baseSyncID))
+
+	queryColumns := []interface{}{}
+	for _, col := range columns {
+		if col == "sync_id" {
+			queryColumns = append(queryColumns, goqu.L(fmt.Sprintf("'%s' as sync_id", newSyncID)))
+			continue
+		}
+		queryColumns = append(queryColumns, col)
+	}
+
+	// Build the main query to select records from newSyncID that don't exist in baseSyncID
+	query := c.db.Insert(tableName).
+		Cols(columns...).
+		Prepared(true).
+		FromQuery(
+			c.db.Select(queryColumns...).
+				From(tableName).
+				Where(
+					goqu.C("sync_id").Eq(appliedSyncID),
+					goqu.C("external_id").NotIn(subquery),
+				),
+		)
+
+	// Generate the SQL and args
+	return query.ToSQL()
+}