feat: Plumbs content hash to post-submit LUCI builds (#4795)

This change plumbs the content-aware hash from the scheduler to the LUCI
build service for post-submit builds. This will allow LUCI/recipes to
use the content hash in post-submit builds (windows arm, ddm, etc).

The following changes are included:
- The `ContentAwareHashService` is updated to include a
`getHashByCommitSha` method.
- The `Scheduler` now retrieves the content hash on addCommit() and
passes it to `LuciBuildService`.
- `LuciBuildService` plumbs the `contentHash` through to buildbucket
properties.
- Tests have been added to verify that the content hash is correctly
passed to the `LuciBuildService`.

Extension of flutter/flutter#167779

Author: jtmcdole

app_dart/lib/src/service/content_aware_hash_service.dart

@@ -336,6 +336,31 @@ interface class ContentAwareHashService {
     );
     return contentHash.waitingShas;
   }
+
+  /// Looks up a content hash via its [commitSha] and returns the hash if
+  /// the document is found.
+  ///
+  /// Note: This only returns the content hash if the commitSha was used to
+  /// build the engine artifacts. It is not a generic "what is the hash" of
+  /// this git commitSha.
+  Future<String?> getHashByCommitSha(String commitSha) async {
+    final docs = await _firestore.query(
+      ContentAwareHashBuilds.metadata.collectionId,
+      {'${ContentAwareHashBuilds.fieldCommitSha} =': commitSha},
+      orderMap: {
+        ContentAwareHashBuilds.fieldCreateTimestamp: kQueryOrderDescending,
+      },
+    );
+    if (docs.isEmpty) {
+      return null;
+    }
+    if (docs.length > 1) {
+      log.warn(
+        'CAHS(commitSha: $commitSha): getHashByCommitSha - multiple hashes found; using latest',
+      );
+    }
+    return ContentAwareHashBuilds.fromDocument(docs.first).contentHash;
+  }
 }
 
 enum MergeQueueHashStatus { wait, build, complete, ignoreJob, error }

app_dart/lib/src/service/luci_build_service.dart

@@ -607,6 +607,7 @@ class LuciBuildService {
   Future<List<PendingTask>> schedulePostsubmitBuilds({
     required CommitRef commit,
     required List<PendingTask> toBeScheduled,
+    String? contentHash,
   }) async {
     if (toBeScheduled.isEmpty) {
       log.debug(
@@ -639,12 +640,16 @@ class LuciBuildService {
       log.info(
         'create postsubmit schedule request for target: ${pending.target} in commit ${commit.sha}',
       );
+      final properties = <String, Object>{
+        if (contentHash != null) 'content_hash': contentHash,
+      };
       final scheduleBuildRequest = await _createPostsubmitScheduleBuild(
         commit: commit,
         target: pending.target,
         taskName: pending.taskName,
         priority: pending.priority,
         currentAttempt: pending.currentAttempt,
+        properties: properties,
       );
       buildRequests.add(
         bbv2.BatchRequest_Request(scheduleBuild: scheduleBuildRequest),
@@ -703,10 +708,13 @@ class LuciBuildService {
         'create postsubmit schedule request for target: $target in commit ${commit.sha}',
       );
 
+      final properties = <String, Object>{
+        if (contentHash != null) 'content_hash': contentHash,
+      };
       final scheduleBuildRequest = await _createMergeGroupScheduleBuild(
         commit: commit,
         target: target,
-        extraProperties: {if (contentHash != null) 'content_hash': contentHash},
+        properties: properties,
       );
       buildRequests.add(
         bbv2.BatchRequest_Request(scheduleBuild: scheduleBuildRequest),
@@ -850,7 +858,7 @@ class LuciBuildService {
     );
 
     final processedProperties = target.getProperties().cast<String, Object?>();
-    processedProperties.addAll(properties ?? <String, Object?>{});
+    if (properties != null) processedProperties.addAll(properties);
     processedProperties['git_branch'] = commit.branch;
     processedProperties['git_repo'] = commit.slug.name;
 
@@ -915,7 +923,7 @@ class LuciBuildService {
     required CommitRef commit,
     required Target target,
     int priority = kDefaultPriority,
-    Map<String, String> extraProperties = const {},
+    Map<String, Object?>? properties,
   }) async {
     log.info(
       'Creating merge group schedule builder for ${target.name} on commit ${commit.sha}',
@@ -938,7 +946,7 @@ class LuciBuildService {
     processedProperties['is_fusion'] = 'true';
     processedProperties[kMergeQueueKey] = true;
     processedProperties['git_repo'] = commit.slug.name;
-    processedProperties.addAll(extraProperties);
+    if (properties != null) processedProperties.addAll(properties);
 
     final propertiesStruct =
         bbv2.Struct()..mergeFromProto3Json(processedProperties);

app_dart/lib/src/service/scheduler.dart

@@ -149,6 +149,7 @@ class Scheduler {
       log.debug('Skipping ${commit.sha} as repo is not supported');
       return;
     }
+    final contentHash = await _contentAwareHash.getHashByCommitSha(commit.sha);
 
     final _TaskCommitScheduling scheduling;
     if (Config.defaultBranch(commit.slug) == commit.branch) {
@@ -216,7 +217,11 @@ class Scheduler {
       'Immediately scheduled tasks for $commit: '
       '${toBeScheduled.map((t) => '"${t.taskName}"').join(', ')}',
     );
-    await _batchScheduleBuilds(commit.toRef(), toBeScheduled);
+    await _batchScheduleBuilds(
+      commit.toRef(),
+      toBeScheduled,
+      contentHash: contentHash,
+    );
     await _uploadToBigQuery(commit);
   }
 
@@ -234,8 +239,9 @@ class Scheduler {
   /// Each batch request contains [Config.batchSize] builds to be scheduled.
   Future<void> _batchScheduleBuilds(
     CommitRef commit,
-    List<PendingTask> toBeScheduled,
-  ) async {
+    List<PendingTask> toBeScheduled, {
+    String? contentHash,
+  }) async {
     final batchLog = StringBuffer(
       'Scheduling ${toBeScheduled.length} tasks in batches for ${commit.sha} as follows:\n',
     );
@@ -250,6 +256,7 @@ class Scheduler {
         _luciBuildService.schedulePostsubmitBuilds(
           commit: commit,
           toBeScheduled: batch,
+          contentHash: contentHash,
         ),
       );
     }

app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart

@@ -520,6 +520,7 @@ final class _FakeLuciBuildService extends Fake implements LuciBuildService {
   Future<List<PendingTask>> schedulePostsubmitBuilds({
     required CommitRef commit,
     required List<PendingTask> toBeScheduled,
+    String? contentHash,
   }) async {
     scheduledPostsubmitBuilds.addAll(toBeScheduled);
     return [];

app_dart/test/service/content_aware_hash_service_test.dart

@@ -379,6 +379,27 @@ void main() {
       );
     });
   });
+
+  group('getHashByCommitSha', () {
+    test('returns the hash when the document is found', () async {
+      firestoreService.putDocument(
+        ContentAwareHashBuilds(
+          createdOn: DateTime.now(),
+          contentHash: '1' * 40,
+          commitSha: 'a' * 40,
+          buildStatus: BuildStatus.inProgress,
+          waitingShas: [],
+        ),
+      );
+      final hash = await cahs.getHashByCommitSha('a' * 40);
+      expect(hash, '1' * 40);
+    });
+
+    test('returns null when the document is not found', () async {
+      final hash = await cahs.getHashByCommitSha('a' * 40);
+      expect(hash, isNull);
+    });
+  });
 }
 
 extension on String {

app_dart/test/service/luci_build_service/schedule_prod_builds_test.dart

@@ -661,6 +661,65 @@ void main() {
     );
   });
 
+  test('schedules a post-submit build with content hash', () async {
+    final commit = generateFirestoreCommit(
+      1,
+      branch: 'master',
+      repo: 'flutter',
+    );
+
+    when(mockBuildBucketClient.listBuilders(any)).thenAnswer((_) async {
+      return bbv2.ListBuildersResponse(
+        builders: [
+          bbv2.BuilderItem(
+            id: bbv2.BuilderID(
+              bucket: 'prod',
+              project: 'flutter',
+              builder: 'Linux 1',
+            ),
+          ),
+        ],
+      );
+    });
+
+    await expectLater(
+      luci.schedulePostsubmitBuilds(
+        commit: commit.toRef(),
+        toBeScheduled: [
+          PendingTask(
+            target: generateTarget(
+              1,
+              properties: {
+                'recipe': 'devicelab/devicelab',
+                'os': 'debian-10.12',
+              },
+              slug: Config.flutterSlug,
+            ),
+            taskName: generateFirestoreTask(1, commitSha: commit.sha).taskName,
+            priority: LuciBuildService.kDefaultPriority,
+            currentAttempt: 1,
+          ),
+        ],
+        contentHash: 'abc',
+      ),
+      completion(isEmpty),
+    );
+
+    final bbv2.ScheduleBuildRequest scheduleBuild;
+    {
+      final batchRequest = bbv2.BatchRequest().createEmptyInstance();
+      batchRequest.mergeFromProto3Json(pubSub.messages.single);
+
+      expect(batchRequest.requests, hasLength(1));
+      scheduleBuild = batchRequest.requests.single.scheduleBuild;
+    }
+
+    expect(
+      scheduleBuild.properties.fields,
+      containsPair('content_hash', bbv2.Value(stringValue: 'abc')),
+    );
+  });
+
   test('does not run a non-existent builder', () async {
     final commit = generateFirestoreCommit(1, branch: 'main', repo: 'flutter');
 

app_dart/test/service/scheduler_test.dart

@@ -281,6 +281,45 @@ void main() {
         );
       });
 
+      test('schedules cocoon based targets with content hash', () async {
+        final luciBuildService = MockLuciBuildService();
+        const contentHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
+        fakeContentAwareHash.hashByCommit['1'] = contentHash;
+        when(
+          luciBuildService.schedulePostsubmitBuilds(
+            commit: anyNamed('commit'),
+            toBeScheduled: anyNamed('toBeScheduled'),
+            contentHash: contentHash,
+          ),
+        ).thenAnswer((_) async => []);
+        scheduler = Scheduler(
+          cache: cache,
+          config: config,
+          githubChecksService: GithubChecksService(
+            config,
+            githubChecksUtil: mockGithubChecksUtil,
+          ),
+          getFilesChanged: getFilesChanged,
+          ciYamlFetcher: ciYamlFetcher,
+          luciBuildService: luciBuildService,
+          contentAwareHash: fakeContentAwareHash,
+          firestore: firestore,
+          bigQuery: bigQuery,
+        );
+
+        // This test is testing `GuaranteedPolicy` get scheduled - there's only one now.
+        await scheduler.addCommits(
+          createCommitList(<String>['1'], branch: 'main', repo: 'packages'),
+        );
+        verify(
+          luciBuildService.schedulePostsubmitBuilds(
+            commit: anyNamed('commit'),
+            toBeScheduled: anyNamed('toBeScheduled'),
+            contentHash: contentHash,
+          ),
+        ).called(1);
+      });
+
       test(
         'schedules cocoon based targets - multiple batch requests',
         () async {

app_dart/test/src/service/fake_content_aware_hash_service.dart

@@ -73,4 +73,10 @@ class FakeContentAwareHashService implements ContentAwareHashService {
     nextCompletedShas = null;
     return shas;
   }
+
+  final hashByCommit = <String, String>{};
+  @override
+  Future<String?> getHashByCommitSha(String commitSha) async {
+    return hashByCommit[commitSha];
+  }
 }