feat: start using the content aware hash for downloading artifacts (flutter#171927)

towards flutter#171790 

Use the content aware hash when downloading engine artifacts. These are
currently produced when changes to DEPs, engine/, or the release file
are changed in a not-seen-before way.

We can eventually remove `engine.version` being tracked in release
branches as an optimization.

`FLUTTER_PREBUILT_ENGINE_VERSION` will stay for overriding the
engine.version for testing. Though Cocoon does not need to actually set
it for framework only PRs anymore.

Author: jtmcdole

bin/internal/update_engine_version.ps1

@@ -57,15 +57,7 @@ if (![string]::IsNullOrEmpty($env:FLUTTER_PREBUILT_ENGINE_VERSION)) {
 # the current branch is forked from, which would be the last version of the
 # engine artifacts built from CI.
 } else {
-  $ErrorActionPreference = "Continue"
-  git -C "$flutterRoot" remote get-url upstream *> $null
-  $exitCode = $?
-  $ErrorActionPreference = "Stop"
-  if ($exitCode) {
-    $engineVersion = (git -C "$flutterRoot"  merge-base HEAD upstream/master)
-  } else {
-    $engineVersion = (git -C "$flutterRoot"  merge-base HEAD origin/master)
-  }
+  $engineVersion = & "$flutterRoot/bin/internal/content_aware_hash.ps1"
 }
 
 # Write the engine version out so downstream tools know what to look for.

bin/internal/update_engine_version.sh

@@ -57,17 +57,7 @@ elif [ -n "$(git -C "$FLUTTER_ROOT" ls-files bin/internal/engine.version)" ]; th
 # the current branch is forked from, which would be the last version of the
 # engine artifacts built from CI.
 else
-  set +e
-  # We fallback to origin/master if upstream is not detected.
-  git -C "$FLUTTER_ROOT" remote get-url upstream >/dev/null 2>&1
-  exit_code=$?
-  set -e
-
-  if [[ $exit_code -eq 0 ]]; then
-    ENGINE_VERSION=$(git -C "$FLUTTER_ROOT" merge-base HEAD upstream/master)
-  else
-    ENGINE_VERSION=$(git -C "$FLUTTER_ROOT" merge-base HEAD origin/master)
-  fi
+  ENGINE_VERSION=$("$FLUTTER_ROOT/bin/internal/content_aware_hash.sh")
 fi
 
 # Write the engine version out so downstream tools know what to look for.

dev/tools/test/update_engine_version_test.dart

@@ -105,6 +105,11 @@ void main() {
       testRoot.binInternalUpdateEngineVersion.path,
     );
 
+    // Copy the content_aware_hash script and create a rough directory structure.
+    flutterRoot.binInternalContentAwareHash.copySyncRecursive(
+      testRoot.binInternalContentAwareHash.path,
+    );
+
     // Regression test for https://github.com/flutter/flutter/pull/164396;
     // on a fresh checkout bin/cache does not exist, so avoid trying to create
     // this folder.
@@ -132,6 +137,11 @@ void main() {
         'internal',
         localFs.path.basename(testRoot.binInternalUpdateEngineVersion.path),
       ),
+      localFs.path.join(
+        'bin',
+        'internal',
+        localFs.path.basename(testRoot.binInternalContentAwareHash.path),
+      ),
       localFs.path.join('bin', 'internal', 'engine.version'),
       localFs.path.join('engine', 'src', '.gn'),
       'DEPS',
@@ -230,13 +240,22 @@ void main() {
     run('git', <String>['fetch', remote], workingPath: rootPath);
   }
 
-  /// Returns the SHA computed by `merge-base HEAD {{ref}}/master`.
-  String gitMergeBase({required String ref}) {
-    final io.ProcessResult mergeBaseHeadOrigin = run('git', <String>[
-      'merge-base',
-      'HEAD',
-      '$ref/master',
-    ]);
+  /// Returns the SHA computed by `content_aware_hash`.
+  String gitContentHash({required _FlutterRootUnderTest fileSystem}) {
+    final String executable;
+    final List<String> args;
+    final String script = fileSystem.binInternalContentAwareHash.path;
+    if (const LocalPlatform().isWindows) {
+      executable = 'powershell';
+      args = <String>[script];
+    } else if (usePowershellOnPosix) {
+      executable = 'pwsh';
+      args = <String>[script];
+    } else {
+      executable = script;
+      args = <String>[];
+    }
+    final io.ProcessResult mergeBaseHeadOrigin = run(executable, args);
     return mergeBaseHeadOrigin.stdout as String;
   }
 
@@ -312,7 +331,10 @@ void main() {
       pinEngineVersionForReleaseBranch(engineHash: 'abc123', gitTrack: false);
       runUpdateEngineVersion();
 
-      expect(testRoot.binCacheEngineStamp, _hasFileContentsMatching(gitMergeBase(ref: 'upstream')));
+      expect(
+        testRoot.binCacheEngineStamp,
+        _hasFileContentsMatching(gitContentHash(fileSystem: testRoot)),
+      );
     });
   });
 
@@ -325,14 +347,20 @@ void main() {
       setupRemote(remote: 'upstream');
       runUpdateEngineVersion();
 
-      expect(testRoot.binCacheEngineStamp, _hasFileContentsMatching(gitMergeBase(ref: 'upstream')));
+      expect(
+        testRoot.binCacheEngineStamp,
+        _hasFileContentsMatching(gitContentHash(fileSystem: testRoot)),
+      );
     });
 
     test('fallsback to origin/master', () async {
       setupRemote(remote: 'origin');
       runUpdateEngineVersion();
 
-      expect(testRoot.binCacheEngineStamp, _hasFileContentsMatching(gitMergeBase(ref: 'origin')));
+      expect(
+        testRoot.binCacheEngineStamp,
+        _hasFileContentsMatching(gitContentHash(fileSystem: testRoot)),
+      );
     });
   });
 
@@ -392,6 +420,13 @@ final class _FlutterRootUnderTest {
           'update_engine_version.${platform.isWindows || forcePowershell ? 'ps1' : 'sh'}',
         ),
       ),
+      binInternalContentAwareHash: root.childFile(
+        fileSystem.path.join(
+          'bin',
+          'internal',
+          'content_aware_hash.${platform.isWindows || forcePowershell ? 'ps1' : 'sh'}',
+        ),
+      ),
     );
   }
 
@@ -417,6 +452,7 @@ final class _FlutterRootUnderTest {
     required this.binInternalEngineVersion,
     required this.binCacheEngineRealm,
     required this.binInternalUpdateEngineVersion,
+    required this.binInternalContentAwareHash,
   });
 
   final Directory root;
@@ -447,6 +483,11 @@ final class _FlutterRootUnderTest {
   /// - [binInternalEngineVersion]
   /// - [binInternalEngineRealm]
   final File binInternalUpdateEngineVersion;
+
+  /// `bin/internal/content_aware_hash.{sh|ps1}`.
+  ///
+  /// This file contains a shell script that computes the content hash
+  final File binInternalContentAwareHash;
 }
 
 extension on File {

docs/tool/Engine-artifacts.md

@@ -13,18 +13,19 @@ for Android, iOS, and so-on).
 
 When using a _released_ version of Flutter, i.e. from a channel such as `stable`,
 [`bin/internal/engine.version`](../../bin/internal/engine.version) is set to the
-git commit SHA for a merged commit in `https://github.com/flutter/flutter`, where
+content hash SHA for a merged commit in `https://github.com/flutter/flutter`, where
 the engine artifacts have already been pre-built and uploaded.
 
 When using the `master` channel, or _contributing_ to Flutter (which is typically
-as a fork of Flutter's `master` channel), the git commit SHA is _computed_ by
-using `git merge-base HEAD upstream/master` (falling back to `git merge-base HEAD origin/master`
-to support direct forks or `flutter/flutter`).
+as a fork of Flutter's `master` channel), the engine SHA is _computed_ by
+generating a content-aware hash of files that affect the engine build (such as
+`DEPS` and the `engine` directory itself).
 
 For _advanced_ use-cases, such as on CI platforms, or for custom 1-off testing
 using a pre-built Flutter engine (to use a _locally_ built Flutter engine see
 [locally built engines](../contributing/testing/Running-and-writing-tests.md#locally-built-engines)), the environment variable `FLUTTER_PREBUILT_ENGINE_VERSION` can be set,
-again to a git commit SHA for a merged commit in `flutter/flutter`:
+again to a engine SHA for a merged commit in `flutter/flutter`. This is only needed
+if different artifacts from the content sha are desired:
 
 ```sh
 $ FLUTTER_PREBUILT_ENGINE_VERSION=abc123 flutter --version
@@ -40,33 +41,33 @@ stateDiagram-v2
     UseEnvVar: Use <code>FLUTTER_PREBUILT_ENGINE_VERSION</code>
     CheckReleaseFile: <code>bin/internal/engine.version</code> exists?
     UseReleaseFile: Use <code>bin/internal/engine.version</code>
-    UseMergeBase: <code>git merge-base HEAD upstream/master</code>
+    UseContentAwareHash: Compute content-aware hash
 
     CheckEnvVar --> UseEnvVar: Yes
     CheckEnvVar --> CheckReleaseFile: No
     UseEnvVar --> [*]: Done
     CheckReleaseFile --> UseReleaseFile: Yes
-    CheckReleaseFile --> UseMergeBase: No
+    CheckReleaseFile --> UseContentAwareHash: No
     UseReleaseFile --> [*]: Done
-    UseMergeBase --> [*]: Done
+    UseContentAwareHash --> [*]: Done
 ```
 
 ## Flutter CI/CD Testing
 
 On Cocoon (Flutter's internal CI/CD) we _often_ set
 `FLUTTER_PREBUILT_ENGINE_VERSION` to the following:
 
-| Branch                    | Presubmit    | Merge Queue        | Postsubmit                         |
-| ------------------------- | ------------ | ------------------ | ---------------------------------- |
-| `main`                    | `commit.sha` | _Uses normal flow_ | _Uses normal flow_                 |
-| `flutter-x.x-candidate.x` | `commit.sha` | N/A[^1]            | _Uses normal flow_                 |
-| `stable` or `beta`        | N/A[^3]      | N/A[^1]            | N/A[^3]                            |
-| _anything else_[^2]       | `commit.sha` | _Uses normal flow_ | _Uses postsubmit engine artifacts_ |
+| Branch                    | Presubmit     | Merge Queue   | Postsubmit    |
+| ------------------------- | ------------- | ------------- | ------------- |
+| `main`                    | `content.sha` | `content.sha` | `content.sha` |
+| `flutter-x.x-candidate.x` | `content.sha` | N/A[^1]       | `content.sha` |
+| `stable` or `beta`        | N/A[^3]       | N/A[^1]       | N/A[^3]       |
+| _anything else_[^2]       | `content.sha` | `content.sh`  | `content.sha` |
 
 > To generate a new `engine.version`:
 >
 > ```sh
-> ./bin/internal/last_engine_commit.sh > ./bin/internal/engine.version
+> ./bin/internal/content_aware_hash.sh > ./bin/internal/engine.version
 > ```
 >
 > As of [`b0ccfb53801abc9b0aa93e7cca3a3841513c3086`](https://flutter.googlesource.com/recipes/+/b0ccfb53801abc9b0aa93e7cca3a3841513c3086) (May 6 2025), the packaging release process will refuse to let you publish a

packages/flutter_tools/lib/src/flutter_cache.dart

@@ -606,7 +606,10 @@ class FlutterRunnerSDKArtifacts extends CachedArtifact {
     if (!_platform.isLinux && !_platform.isMacOS) {
       return;
     }
-    final url = '${cache.cipdBaseUrl}/flutter/fuchsia/+/git_revision:$version';
+    // Keep in sync with
+    //   engine/src/flutter/tools/fuchsia/build_fuchsia_artifacts.py
+    //   engine/src/flutter/tools/fuchsia/merge_and_upload_debug_symbols.py
+    final url = '${cache.cipdBaseUrl}/flutter/fuchsia/+/content_aware_hash:$version';
     await artifactUpdater.downloadZipArchive(
       'Downloading package flutter runner...',
       Uri.parse(url),
@@ -633,7 +636,10 @@ class CipdArchiveResolver extends VersionedPackageResolver {
 
   @override
   String resolveUrl(String packageName, String version) {
-    return '${cache.cipdBaseUrl}/flutter/$packageName/+/git_revision:$version';
+    // Keep in sync with
+    //   engine/src/flutter/tools/fuchsia/build_fuchsia_artifacts.py
+    //   engine/src/flutter/tools/fuchsia/merge_and_upload_debug_symbols.py
+    return '${cache.cipdBaseUrl}/flutter/$packageName/+/content_aware_hash:$version';
   }
 }