feat(sync): complete incremental sync protocol with storage-pull, state management, and tests (#799)

- Add storage-pull strategy: rclone-based sync from remote storage with
  overlay and clean modes, URI parsing (storage://remote:bucket/path),
  transfer parallelism, and automatic rclone availability checking
- Add SyncStateManager: persistent state load/save with configurable
  paths, workspace hash calculation via SHA-256 of key project files,
  and drift detection for external modification awareness
- Add action.yml inputs: syncStrategy, syncInputRef, syncStorageRemote,
  syncRevertAfter, syncStatePath with sensible defaults
- Wire sync into Input (5 getters), BuildParameters (5 fields), index.ts
  (local build path), and RemoteClient (orchestrator path) with post-job
  overlay revert when syncRevertAfter is true
- Add 42 unit tests covering all strategies, URI parsing, state
  management, hash calculation, drift detection, error handling, and
  edge cases (missing rclone, invalid URIs, absent state, empty diffs)
- Add root:true to eslintrc to prevent plugin resolution conflicts

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: frostebite

.eslintrc.json

@@ -1,4 +1,5 @@
 {
+  "root": true,
   "plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
   "extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
   "parser": "@typescript-eslint/parser",

action.yml

@@ -182,8 +182,8 @@ inputs:
     required: false
     default: ''
     description:
-      '[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with the
-      keys image, secrets (name, value object array), command line string)'
+      '[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with
+      the keys image, secrets (name, value object array), command line string)'
   awsStackName:
     default: 'game-ci'
     required: false
@@ -279,6 +279,24 @@ inputs:
     description:
       '[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
       fixes.'
+  syncStrategy:
+    description: 'Workspace sync strategy: full, git-delta, direct-input, storage-pull'
+    required: false
+    default: 'full'
+  syncInputRef:
+    description: 'URI for direct-input or storage-pull content (storage://remote/path or file path)'
+    required: false
+  syncStorageRemote:
+    description: 'rclone remote name for storage-backed inputs (defaults to rcloneRemote)'
+    required: false
+  syncRevertAfter:
+    description: 'Revert overlaid changes after job completion'
+    required: false
+    default: 'true'
+  syncStatePath:
+    description: 'Path to sync state file for delta tracking'
+    required: false
+    default: '.game-ci/sync-state.json'
 
 outputs:
   volume:

dist/index.js

@@ -3,6 +3,8 @@ import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output
 import { Cli } from './model/cli/cli';
 import MacBuilder from './model/mac-builder';
 import PlatformSetup from './model/platform-setup';
+import { IncrementalSyncService } from './model/orchestrator/services/sync';
+import { SyncStrategy } from './model/orchestrator/services/sync/sync-state';
 
 async function runMain() {
   try {
@@ -23,6 +25,14 @@ async function runMain() {
 
     if (buildParameters.providerStrategy === 'local') {
       core.info('Building locally');
+
+      // Apply incremental sync strategy before build
+      const syncStrategy = buildParameters.syncStrategy as SyncStrategy;
+      if (syncStrategy !== 'full') {
+        core.info(`[Sync] Applying sync strategy: ${syncStrategy}`);
+        await applySyncStrategy(buildParameters, workspace);
+      }
+
       await PlatformSetup.setup(buildParameters, actionFolder);
       exitCode =
         process.platform === 'darwin'
@@ -32,6 +42,16 @@ async function runMain() {
               actionFolder,
               ...buildParameters,
             });
+
+      // Revert overlays after job completion if configured
+      if (buildParameters.syncRevertAfter && syncStrategy !== 'full') {
+        core.info('[Sync] Reverting overlay changes after job completion');
+        try {
+          await IncrementalSyncService.revertOverlays(workspace, buildParameters.syncStatePath);
+        } catch (revertError) {
+          core.warning(`[Sync] Overlay revert failed: ${(revertError as Error).message}`);
+        }
+      }
     } else {
       await Orchestrator.run(buildParameters, baseImage.toString());
       exitCode = 0;
@@ -50,4 +70,58 @@ async function runMain() {
   }
 }
 
+/**
+ * Apply the configured sync strategy to the workspace before build.
+ */
+async function applySyncStrategy(buildParameters: BuildParameters, workspace: string): Promise<void> {
+  const strategy = buildParameters.syncStrategy as SyncStrategy;
+  const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspace, buildParameters.syncStatePath);
+
+  if (resolvedStrategy === 'full') {
+    core.info('[Sync] Resolved to full sync (no incremental state available)');
+
+    return;
+  }
+
+  switch (resolvedStrategy) {
+    case 'git-delta': {
+      const targetReference = buildParameters.gitSha || buildParameters.branch;
+      const changedFiles = await IncrementalSyncService.syncGitDelta(
+        workspace,
+        targetReference,
+        buildParameters.syncStatePath,
+      );
+      core.info(`[Sync] Git delta sync applied: ${changedFiles} file(s) changed`);
+      break;
+    }
+    case 'direct-input': {
+      if (!buildParameters.syncInputRef) {
+        throw new Error('[Sync] direct-input strategy requires syncInputRef to be set');
+      }
+      const overlays = await IncrementalSyncService.applyDirectInput(
+        workspace,
+        buildParameters.syncInputRef,
+        buildParameters.syncStorageRemote || undefined,
+        buildParameters.syncStatePath,
+      );
+      core.info(`[Sync] Direct input applied: ${overlays.length} overlay(s)`);
+      break;
+    }
+    case 'storage-pull': {
+      if (!buildParameters.syncInputRef) {
+        throw new Error('[Sync] storage-pull strategy requires syncInputRef to be set');
+      }
+      const pulledFiles = await IncrementalSyncService.syncStoragePull(workspace, buildParameters.syncInputRef, {
+        rcloneRemote: buildParameters.syncStorageRemote || undefined,
+        syncRevertAfter: buildParameters.syncRevertAfter,
+        statePath: buildParameters.syncStatePath,
+      });
+      core.info(`[Sync] Storage pull complete: ${pulledFiles.length} file(s)`);
+      break;
+    }
+    default:
+      core.warning(`[Sync] Unknown sync strategy: ${resolvedStrategy}`);
+  }
+}
+
 runMain();

dist/index.js.map

@@ -106,6 +106,11 @@ class BuildParameters {
   public cacheUnityInstallationOnMac!: boolean;
   public unityHubVersionOnMac!: string;
   public dockerWorkspacePath!: string;
+  public syncStrategy!: string;
+  public syncInputRef!: string;
+  public syncStorageRemote!: string;
+  public syncRevertAfter!: boolean;
+  public syncStatePath!: string;
 
   public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
     return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
@@ -242,6 +247,11 @@ class BuildParameters {
       cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
       unityHubVersionOnMac: Input.unityHubVersionOnMac,
       dockerWorkspacePath: Input.dockerWorkspacePath,
+      syncStrategy: Input.syncStrategy,
+      syncInputRef: Input.syncInputRef,
+      syncStorageRemote: Input.syncStorageRemote,
+      syncRevertAfter: Input.syncRevertAfter,
+      syncStatePath: Input.syncStatePath,
     };
   }
 

src/index.ts

@@ -241,6 +241,28 @@ class Input {
     return Input.getInput('dockerWorkspacePath') ?? '/github/workspace';
   }
 
+  static get syncStrategy(): string {
+    return Input.getInput('syncStrategy') ?? 'full';
+  }
+
+  static get syncInputRef(): string {
+    return Input.getInput('syncInputRef') ?? '';
+  }
+
+  static get syncStorageRemote(): string {
+    return Input.getInput('syncStorageRemote') ?? '';
+  }
+
+  static get syncRevertAfter(): boolean {
+    const input = Input.getInput('syncRevertAfter') ?? 'true';
+
+    return input === 'true';
+  }
+
+  static get syncStatePath(): string {
+    return Input.getInput('syncStatePath') ?? '.game-ci/sync-state.json';
+  }
+
   static get dockerCpuLimit(): string {
     return Input.getInput('dockerCpuLimit') ?? os.cpus().length.toString();
   }

src/model/build-parameters.ts

@@ -15,15 +15,24 @@ import BuildParameters from '../../build-parameters';
 import { Cli } from '../../cli/cli';
 import OrchestratorOptions from '../options/orchestrator-options';
 import ResourceTracking from '../services/core/resource-tracking';
+import { IncrementalSyncService } from '../services/sync';
+import { SyncStrategy } from '../services/sync/sync-state';
 
 export class RemoteClient {
   @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
   static async setupRemoteClient() {
     OrchestratorLogger.log(`bootstrap game ci orchestrator...`);
     await ResourceTracking.logDiskUsageSnapshot('remote-cli-pre-build (start)');
-    if (!(await RemoteClient.handleRetainedWorkspace())) {
+
+    const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
+
+    if (syncStrategy !== 'full') {
+      OrchestratorLogger.log(`[Sync] Using incremental sync strategy: ${syncStrategy}`);
+      await RemoteClient.handleIncrementalSync(syncStrategy);
+    } else if (!(await RemoteClient.handleRetainedWorkspace())) {
       await RemoteClient.bootstrapRepository();
     }
+
     await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
     await RemoteClient.runCustomHookFiles(`before-build`);
   }
@@ -157,6 +166,20 @@ export class RemoteClient {
 
       await RemoteClient.runCustomHookFiles(`after-build`);
 
+      // Revert sync overlays if configured
+      const syncStrategy = (Orchestrator.buildParameters.syncStrategy || 'full') as SyncStrategy;
+      if (Orchestrator.buildParameters.syncRevertAfter && syncStrategy !== 'full') {
+        try {
+          OrchestratorLogger.log('[Sync] Reverting overlay changes after job completion');
+          await IncrementalSyncService.revertOverlays(
+            OrchestratorFolders.repoPathAbsolute,
+            Orchestrator.buildParameters.syncStatePath,
+          );
+        } catch (revertError: any) {
+          RemoteClientLogger.logWarning(`[Sync] Overlay revert failed: ${revertError.message}`);
+        }
+      }
+
       // WIP - need to give the pod permissions to create config map
       await RemoteClientLogger.handleLogManagementPostJob();
     } catch (error: any) {
@@ -229,6 +252,78 @@ export class RemoteClient {
       RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
     }
   }
+
+  /**
+   * Handle incremental sync strategies (git-delta, direct-input, storage-pull).
+   *
+   * For git-delta: requires an existing workspace with sync state; fetches and applies
+   * only changed files.
+   *
+   * For direct-input and storage-pull: requires an existing workspace; applies overlay
+   * content on top.
+   *
+   * Falls back to full bootstrapRepository() if incremental sync cannot proceed.
+   */
+  private static async handleIncrementalSync(strategy: SyncStrategy): Promise<void> {
+    const buildParameters = Orchestrator.buildParameters;
+    const workspacePath = OrchestratorFolders.repoPathAbsolute;
+    const statePath = buildParameters.syncStatePath;
+
+    // Resolve strategy — may fall back to 'full' if no state exists
+    const resolvedStrategy = IncrementalSyncService.resolveStrategy(strategy, workspacePath, statePath);
+
+    if (resolvedStrategy === 'full') {
+      OrchestratorLogger.log('[Sync] Falling back to full bootstrap');
+      if (!(await RemoteClient.handleRetainedWorkspace())) {
+        await RemoteClient.bootstrapRepository();
+      }
+
+      return;
+    }
+
+    switch (resolvedStrategy) {
+      case 'git-delta': {
+        const targetReference = buildParameters.gitSha || buildParameters.branch;
+        OrchestratorLogger.log(`[Sync] Git delta sync to ${targetReference}`);
+        const changedFiles = await IncrementalSyncService.syncGitDelta(workspacePath, targetReference, statePath);
+        OrchestratorLogger.log(`[Sync] Git delta complete: ${changedFiles} file(s) updated`);
+        break;
+      }
+      case 'direct-input': {
+        const inputReference = buildParameters.syncInputRef;
+        if (!inputReference) {
+          throw new Error('[Sync] direct-input strategy requires syncInputRef');
+        }
+        OrchestratorLogger.log(`[Sync] Applying direct input: ${inputReference}`);
+        await IncrementalSyncService.applyDirectInput(
+          workspacePath,
+          inputReference,
+          buildParameters.syncStorageRemote || undefined,
+          statePath,
+        );
+        break;
+      }
+      case 'storage-pull': {
+        const storageUri = buildParameters.syncInputRef;
+        if (!storageUri) {
+          throw new Error('[Sync] storage-pull strategy requires syncInputRef');
+        }
+        OrchestratorLogger.log(`[Sync] Storage pull from: ${storageUri}`);
+        await IncrementalSyncService.syncStoragePull(workspacePath, storageUri, {
+          rcloneRemote: buildParameters.syncStorageRemote || undefined,
+          syncRevertAfter: buildParameters.syncRevertAfter,
+          statePath,
+        });
+        break;
+      }
+      default:
+        OrchestratorLogger.logWarning(`[Sync] Unknown strategy: ${resolvedStrategy}, falling back to full`);
+        if (!(await RemoteClient.handleRetainedWorkspace())) {
+          await RemoteClient.bootstrapRepository();
+        }
+    }
+  }
+
   public static async bootstrapRepository() {
     await OrchestratorSystem.Run(
       `mkdir -p ${OrchestratorFolders.ToLinuxFolder(OrchestratorFolders.uniqueOrchestratorJobFolderAbsolute)}`,