Recover from an ActionResult json print error (buildfarm#2390)

Clients will upload results that contain execution_metadata that may
contain unknown protobuf message types in the auxiliary_metadata
repeated Any field. Purge unparseable objects and print warnings
indicating what happened.

Author: werkt

src/main/java/build/buildfarm/instance/shard/RedisShardBackplane.java

@@ -22,6 +22,7 @@
 
 import build.bazel.remote.execution.v2.ActionResult;
 import build.bazel.remote.execution.v2.ExecuteOperationMetadata;
+import build.bazel.remote.execution.v2.ExecutedActionMetadata;
 import build.bazel.remote.execution.v2.ExecutionStage;
 import build.bazel.remote.execution.v2.Platform;
 import build.bazel.remote.execution.v2.RequestMetadata;
@@ -72,6 +73,7 @@
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.longrunning.Operation;
+import com.google.protobuf.Any;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.Timestamp;
 import com.google.protobuf.util.JsonFormat;
@@ -940,10 +942,45 @@ public void blacklistAction(String actionId) throws IOException {
                 jedis, actionId, "", configs.getBackplane().getActionBlacklistExpire()));
   }
 
+  private String printActionResult(ActionResult actionResult)
+      throws InvalidProtocolBufferException {
+    InvalidProtocolBufferException cause;
+    try {
+      return actionResultPrinter.print(actionResult);
+    } catch (InvalidProtocolBufferException e) {
+      // can happen with unknown types in auxiliary_metadata
+      // this is extremely brittle and will have issues with any novel introduction of Any in remote
+      // apis releases
+      cause = e;
+    }
+
+    ActionResult.Builder builder = actionResult.toBuilder();
+    ExecutedActionMetadata.Builder metadata =
+        builder.getExecutionMetadataBuilder().clearAuxiliaryMetadata();
+    for (Any auxiliaryMetadata : actionResult.getExecutionMetadata().getAuxiliaryMetadataList()) {
+      try {
+        // test the serialization capacity of this any
+        actionResultPrinter.print(auxiliaryMetadata);
+        // serialization passed, re-add it
+        metadata.addAuxiliaryMetadata(auxiliaryMetadata);
+      } catch (InvalidProtocolBufferException e) {
+        // ignore
+      }
+    }
+
+    String json = actionResultPrinter.print(builder.build());
+    // purge must have succeeded, indicate as much to the server log
+    log.log(
+        Level.WARNING,
+        "error printing auxiliary_metadata for key %s, unrecognized content purged",
+        cause);
+    return json;
+  }
+
   @SuppressWarnings("ConstantConditions")
   @Override
   public void putActionResult(ActionKey actionKey, ActionResult actionResult) throws IOException {
-    String json = actionResultPrinter.print(actionResult);
+    String json = printActionResult(actionResult);
     client.run(
         jedis ->
             state.actionCache.insert(

src/test/java/build/buildfarm/instance/shard/RedisShardBackplaneTest.java

@@ -25,8 +25,12 @@
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
+import build.bazel.remote.execution.v2.Action;
+import build.bazel.remote.execution.v2.ActionResult;
+import build.bazel.remote.execution.v2.ExecuteOperationMetadata;
 import build.bazel.remote.execution.v2.RequestMetadata;
 import build.buildfarm.common.DigestUtil;
+import build.buildfarm.common.DigestUtil.ActionKey;
 import build.buildfarm.common.DigestUtil.HashFunction;
 import build.buildfarm.common.config.BuildfarmConfigs;
 import build.buildfarm.common.config.Queue;
@@ -44,11 +48,13 @@
 import build.buildfarm.v1test.QueueEntry;
 import build.buildfarm.v1test.ShardWorker;
 import build.buildfarm.v1test.WorkerChange;
+import build.buildfarm.v1test.WorkerExecutedMetadata;
 import build.buildfarm.worker.resources.LocalResourceSet;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.longrunning.Operation;
+import com.google.protobuf.Any;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.util.JsonFormat;
 import java.io.IOException;
@@ -502,4 +508,34 @@ public void testAddWorker() throws IOException {
             JsonFormat.printer().print(shardWorker));
     verify(jedis, times(1)).publish(anyString(), anyString());
   }
+
+  @Test
+  public void putActionResultPurgesUnknownAuxiliaryMetadatas() throws Exception {
+    UnifiedJedis jedis = mock(UnifiedJedis.class);
+    RedisClient client = new RedisClient(jedis);
+    DistributedState state = new DistributedState();
+    state.actionCache = mock(RedisMap.class);
+    RedisShardBackplane backplane = createBackplane("put-action-result-purges-test");
+    backplane.start(client, state, "putActionResultPurges/test:0000", name -> {});
+    DigestUtil digestUtil = new DigestUtil(HashFunction.SHA256);
+    ActionKey actionKey = digestUtil.computeActionKey(Action.getDefaultInstance());
+
+    ActionResult.Builder actionResult = ActionResult.newBuilder();
+    // action results cannot currently parse ExecuteOperationMetadata, ensure this continues to be
+    // true.
+    actionResult
+        .getExecutionMetadataBuilder()
+        .addAuxiliaryMetadata(Any.pack(ExecuteOperationMetadata.getDefaultInstance()))
+        .addAuxiliaryMetadata(Any.pack(WorkerExecutedMetadata.getDefaultInstance()));
+
+    backplane.putActionResult(actionKey, actionResult.build());
+
+    ArgumentCaptor<String> resultCaptor = ArgumentCaptor.forClass(String.class);
+    verify(state.actionCache, times(1))
+        .insert(eq(jedis), eq(actionKey.toString()), resultCaptor.capture(), any(Integer.class));
+    verifyNoMoreInteractions(state.actionCache);
+    String json = resultCaptor.getValue();
+    assertThat(
+        backplane.parseActionResult(json).getExecutionMetadata().getAuxiliaryMetadataCount() == 1);
+  }
 }