feat: Tag Fuchsia artifacts by content hash (flutter#172132)

After the fuchsia build process, be sure to tag uploaded artifacts with
`content_aware_hash:` if configured.

fixes flutter#171985

This was a bit of a rabbit hole, with some "magic parameters" that
control uploading / tagging. I'm not sure cipd supports two tags with
the say keys; but I wouldn't want the content hash having the key
"git_revision".

Pre-submits: Not affected since --engine-version is always '' which is
read as "do not upload"
Post-submits: Reads the `linux_fuchsia.json` builder to look for flags.

> Note: This needs to land before the tool can be updated to download
the content_aware_hash tag.

Tested: locally, on linux, after building all the x64 targets. With and
without engine-version, with and without linux_fuchsia.json flags.

```shell
$ src/flutter/tools/fuchsia/merge_and_upload_debug_symbols.py --target-arch x64 --engine 'abcd' --upload --out-dir tmp  --symbol-dirs out/ci/fuchsia_debug_x64/.build-id out/ci/fuchsia_release_x64/.build-id out/ci/fuchsia_profile_x64/.build-id

Using content hash 2201006225127f112f6576fcf73dd00671b2012e for engine version
['cipd', 'create', '-pkg-def', '/usr/local/google/home/codefu/src/flutter/engine/tmp/debug_symbols.cipd.yaml', '-ref', 'latest', '-tag', 'git_revision:abcd', '-verification-timeout', '10m0s', '-tag', 'content_aware_hash:2201006225127f112f6576fcf73dd00671b2012e']
```

```
$ python3 src/flutter/tools/fuchsia/build_fuchsia_artifacts.py --archs x64 --engine 'abc' --cipd-dry-run --upload

Using content hash 2201006225127f112f6576fcf73dd00671b2012e for engine version
codefu: ['cipd', 'create', '-pkg-def', 'fuchsia.cipd.yaml', '-ref', 'latest', '-tag', 'git_revision:abc', '-tag', 'content_aware_hash:2201006225127f112f6576fcf73dd00671b2012e']
```

Author: jtmcdole

engine/src/flutter/tools/fuchsia/build_fuchsia_artifacts.py

@@ -16,6 +16,7 @@
 import subprocess
 import sys
 import tempfile
+from get_content_hash import get_content_hash
 
 _script_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..'))
 _src_root_dir = os.path.join(_script_dir, '..', '..', '..')
@@ -240,7 +241,7 @@ def RunCIPDCommandWithRetries(command):
         raise
 
 
-def ProcessCIPDPackage(upload, engine_version):
+def ProcessCIPDPackage(upload, engine_version, content_hash):
   if not upload or not IsLinux():
     RunCIPDCommandWithRetries([
         'cipd', 'pkg-build', '-pkg-def', 'fuchsia.cipd.yaml', '-out',
@@ -256,22 +257,32 @@ def ProcessCIPDPackage(upload, engine_version):
     print('--upload requires --engine-version to be specified.')
     return
 
-  tag = 'git_revision:%s' % engine_version
-  already_exists = CheckCIPDPackageExists('flutter/fuchsia', tag)
+  git_tag = 'git_revision:%s' % engine_version
+  already_exists = CheckCIPDPackageExists('flutter/fuchsia', git_tag)
   if already_exists:
-    print('CIPD package flutter/fuchsia tag %s already exists!' % tag)
+    print('CIPD package flutter/fuchsia tag %s already exists!' % git_tag)
     return
 
-  RunCIPDCommandWithRetries([
+  command = [
       'cipd',
       'create',
       '-pkg-def',
       'fuchsia.cipd.yaml',
       '-ref',
       'latest',
       '-tag',
-      tag,
-  ])
+      git_tag,
+  ]
+
+  content_tag = 'content_aware_hash:%s' % content_hash
+  already_exists = CheckCIPDPackageExists('flutter/fuchsia', content_tag)
+  if already_exists:
+    print('CIPD package flutter/fuchsia tag %s already exists!' % content_tag)
+    # do not return; content hash can match multiple PRs (reverts, framework only)
+  else:
+    command.extend(['-tag', content_tag])
+
+  RunCIPDCommandWithRetries(command)
 
 
 def main():
@@ -400,13 +411,18 @@ def main():
   # presubmit workflows.
   should_upload = args.upload
   engine_version = args.engine_version
-  if not engine_version:
+  content_hash = ''
+  if engine_version:
+    # When content hashing is enabled, the engine version will be a content
+    # hash instead of a git revision.
+    content_hash = get_content_hash()
+  else:
     engine_version = 'HEAD'
     should_upload = False
 
   # Create and optionally upload CIPD package
   if args.cipd_dry_run or args.upload:
-    ProcessCIPDPackage(should_upload, engine_version)
+    ProcessCIPDPackage(should_upload, engine_version, content_hash)
 
   return 0
 

engine/src/flutter/tools/fuchsia/get_content_hash.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" Shared code for handling content_aware_hashing for fuchsia.
+"""
+import json
+import os
+import subprocess
+
+_script_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..'))
+_src_root_dir = os.path.join(_script_dir, '..', '..', '..')
+
+
+def get_content_hash():
+  ci_config_path = os.path.join(_src_root_dir, 'flutter', 'ci', 'builders', 'linux_fuchsia.json')
+  upload_content_hash = False
+  if os.path.exists(ci_config_path):
+    with open(ci_config_path, 'r') as f:
+      ci_config = json.load(f)
+      upload_content_hash = ci_config.get('luci_flags', {}).get('upload_content_hash', False)
+  if upload_content_hash:
+    script_path = os.path.join(
+        _src_root_dir, '..', '..', 'bin', 'internal', 'content_aware_hash.sh'
+    )
+    if os.path.exists(script_path):
+      command = [script_path]
+      try:
+        content_hash = subprocess.check_output(command, text=True).strip()
+        print('Using content hash %s for engine version' % content_hash)
+        return content_hash
+      except subprocess.CalledProcessError as e:
+        print('Error getting content hash, falling back to git hash: %s' % e)
+    else:
+      print('Could not find content_aware_hash.sh at %s' % script_path)
+  return ''

engine/src/flutter/tools/fuchsia/merge_and_upload_debug_symbols.py

@@ -18,6 +18,7 @@
 import sys
 import tarfile
 import tempfile
+from get_content_hash import get_content_hash
 
 # Path to the engine root checkout. This is used to calculate absolute
 # paths if relative ones are passed to the script.
@@ -77,13 +78,23 @@ def CheckCIPDPackageExists(package_name, tag):
     return True
 
 
-def ProcessCIPDPackage(upload, cipd_yaml, engine_version, out_dir, target_arch):
+def ProcessCIPDPackage(upload, cipd_yaml, engine_version, content_hash, out_dir, target_arch):
   _packaging_dir = GetPackagingDir(out_dir)
-  tag = 'git_revision:%s' % engine_version
   package_name = 'flutter/fuchsia-debug-symbols-%s' % target_arch
-  already_exists = CheckCIPDPackageExists(package_name, tag)
+
+  git_tag = 'git_revision:%s' % engine_version
+  already_exists = CheckCIPDPackageExists(package_name, git_tag)
   if already_exists:
-    print('CIPD package %s tag %s already exists!' % (package_name, tag))
+    print('CIPD package %s tag %s already exists!' % (package_name, git_tag))
+
+  content_tag = ''
+  if content_hash:
+    content_tag = 'content_aware_hash:%s' % content_hash
+    content_already_exists = CheckCIPDPackageExists(package_name, content_tag)
+    if content_already_exists:
+      print('CIPD package %s tag %s already exists!' % (package_name, content_tag))
+      content_tag = ''
+      # do not return; content hash can match multiple PRs (reverts, framework only)
 
   if upload and IsLinux() and not already_exists:
     command = [
@@ -94,10 +105,12 @@ def ProcessCIPDPackage(upload, cipd_yaml, engine_version, out_dir, target_arch):
         '-ref',
         'latest',
         '-tag',
-        tag,
+        git_tag,
         '-verification-timeout',
         '10m0s',
     ]
+    if content_tag:
+      command.extend(['-tag', content_tag])
   else:
     command = [
         'cipd', 'pkg-build', '-pkg-def', cipd_yaml, '-out',
@@ -210,11 +223,16 @@ def main():
   # on presubmit.
   should_upload = args.upload
   engine_version = args.engine_version
-  if not engine_version:
+  content_hash = ''
+  if engine_version:
+    # When content hashing is enabled, the engine version will be a content
+    # hash instead of a git revision.
+    content_hash = get_content_hash()
+  else:
     engine_version = 'HEAD'
     should_upload = False
 
-  ProcessCIPDPackage(should_upload, cipd_def, engine_version, out_dir, arch)
+  ProcessCIPDPackage(should_upload, cipd_def, engine_version, content_hash, out_dir, arch)
   return 0