feat(flexible-outcalls): support flexible responses in CanisterHttpPayload (#9159)

Adds a `flexible_responses: Vec<FlexibleCanisterHttpResponses>` field to
the `CanisterHttpPayload` incl. protobuf serialization support.

This is a pure type addition with no behavioral changes; construction
and validation logic will follow in a subsequent PR.

Author: fspreiss

rs/https_outcalls/consensus/src/payload_builder.rs

@@ -401,6 +401,8 @@ impl CanisterHttpPayloadBuilderImpl {
                 .collect(),
             timeouts,
             divergence_responses,
+            // TODO(flexible-http-outcalls): implement flexible responses
+            flexible_responses: vec![],
         }
     }
 
@@ -416,6 +418,11 @@ impl CanisterHttpPayloadBuilderImpl {
             return Ok(());
         }
 
+        // Flexible responses are not yet supported
+        if !payload.flexible_responses.is_empty() {
+            return invalid_artifact(InvalidCanisterHttpPayloadReason::FlexibleResponsesNotEmpty);
+        }
+
         // Check whether feature is enabled and reject if it isn't.
         // NOTE: All payloads that are processed at this point are non-empty
         if !self.is_enabled(validation_context).map_err(|err| {

rs/https_outcalls/consensus/src/payload_builder/parse.rs

@@ -7,7 +7,9 @@ use ic_protobuf::{
 };
 use ic_types::{
     NumBytes,
-    batch::{CanisterHttpPayload, iterator_to_bytes, slice_to_messages},
+    batch::{
+        CanisterHttpPayload, FlexibleCanisterHttpResponses, iterator_to_bytes, slice_to_messages,
+    },
     messages::CallbackId,
 };
 use std::collections::HashSet;
@@ -24,6 +26,9 @@ pub(crate) fn bytes_to_payload(data: &[u8]) -> Result<CanisterHttpPayload, Proxy
             Some(MessageType::DivergenceResponse(response)) => {
                 payload.divergence_responses.push(response.try_into()?)
             }
+            Some(MessageType::FlexibleResponses(flex_responses)) => payload
+                .flexible_responses
+                .push(FlexibleCanisterHttpResponses::try_from(flex_responses)?),
             None => return Err(ProxyDecodeError::MissingField("message_type")),
         }
     }
@@ -55,7 +60,14 @@ pub(crate) fn payload_to_bytes(payload: &CanisterHttpPayload, max_size: NumBytes
                             pb::CanisterHttpResponseWithConsensus::from(response),
                         )),
                     }),
-            );
+            )
+            .chain(payload.flexible_responses.iter().map(|flex_responses| {
+                CanisterHttpResponseMessage {
+                    message_type: Some(MessageType::FlexibleResponses(
+                        pb::FlexibleCanisterHttpResponses::from(flex_responses),
+                    )),
+                }
+            }));
 
     iterator_to_bytes(message_iterator, max_size)
 }
@@ -94,6 +106,7 @@ fn get_id_from_message(message: CanisterHttpResponseMessage) -> Option<u64> {
             .shares
             .first()
             .and_then(|share| share.metadata.as_ref().map(|md| md.id)),
+        Some(MessageType::FlexibleResponses(flex_responses)) => Some(flex_responses.callback_id),
         Some(MessageType::Timeout(id)) => Some(id),
         None => None,
     }

rs/https_outcalls/consensus/src/payload_builder/tests.rs

@@ -217,6 +217,7 @@ fn multiple_payload_test() {
                 }],
                 timeouts: vec![],
                 divergence_responses: vec![],
+                flexible_responses: vec![],
             };
             let past_payload = payload_to_bytes(&past_payload, NumBytes::new(4 * 1024 * 1024));
 
@@ -644,6 +645,7 @@ fn duplicate_validation() {
             responses: vec![response_and_metadata_to_proof(&response, &metadata)],
             timeouts: vec![],
             divergence_responses: vec![],
+            flexible_responses: vec![],
         };
         let payload = payload_to_bytes(&payload, NumBytes::new(4 * 1024 * 1024));
         let past_payloads = vec![PastPayload {
@@ -697,6 +699,7 @@ fn divergence_response_validation_test() {
                         }))
                         .collect(),
                 }],
+                flexible_responses: vec![],
             };
             let payload = payload_to_bytes(&payload, NumBytes::new(4 * 1024 * 1024));
 
@@ -717,6 +720,7 @@ fn divergence_response_validation_test() {
                         .map(|node_id| metadata_to_share(node_id.try_into().unwrap(), &metadata))
                         .collect(),
                 }],
+                flexible_responses: vec![],
             };
             let payload = payload_to_bytes(&payload, NumBytes::new(4 * 1024 * 1024));
 
@@ -754,6 +758,7 @@ fn divergence_response_validation_test() {
                         }))
                         .collect(),
                 }],
+                flexible_responses: vec![],
             };
             let payload = payload_to_bytes(&payload, NumBytes::new(4 * 1024 * 1024));
 
@@ -1790,6 +1795,7 @@ where
             responses: vec![response_and_metadata_to_proof(&response, &metadata)],
             timeouts: vec![],
             divergence_responses: vec![],
+            flexible_responses: vec![],
         };
 
         let payload = payload_to_bytes(&payload, NumBytes::new(4 * 1024 * 1024));

rs/https_outcalls/consensus/src/test_utils.rs

@@ -36,6 +36,7 @@ impl BatchPayloadBuilder for FakeCanisterHttpPayloadBuilder {
             responses: self.0.clone(),
             timeouts: vec![],
             divergence_responses: vec![],
+            flexible_responses: vec![],
         };
         payload_to_bytes(&payload, max_size)
     }

rs/interfaces/src/canister_http.rs

@@ -70,6 +70,8 @@ pub enum InvalidCanisterHttpPayloadReason {
     DuplicateResponse(CallbackId),
     DivergenceProofContainsMultipleCallbackIds,
     DivergenceProofDoesNotMeetDivergenceCriteria,
+    /// The payload contains flexible responses, which are not yet supported
+    FlexibleResponsesNotEmpty,
     /// The payload could not be deserialized
     DecodeError(ProxyDecodeError),
 }

rs/protobuf/def/types/v1/canister_http.proto

@@ -71,11 +71,22 @@ message CanisterHttpResponseDivergence {
   repeated CanisterHttpShare shares = 1;
 }
 
+message FlexibleCanisterHttpResponseWithProof {
+  CanisterHttpResponse response = 1;
+  CanisterHttpShare proof = 2;
+}
+
+message FlexibleCanisterHttpResponses {
+  uint64 callback_id = 1;
+  repeated FlexibleCanisterHttpResponseWithProof responses = 2;
+}
+
 message CanisterHttpResponseMessage {
   oneof message_type {
     CanisterHttpResponseWithConsensus response = 1;
     uint64 timeout = 2;
     CanisterHttpResponseDivergence divergence_response = 3;
+    FlexibleCanisterHttpResponses flexible_responses = 4;
   }
 }
 

rs/protobuf/src/gen/types/types.v1.rs

@@ -612,10 +612,24 @@ pub struct CanisterHttpResponseDivergence {
     pub shares: ::prost::alloc::vec::Vec<CanisterHttpShare>,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct FlexibleCanisterHttpResponseWithProof {
+    #[prost(message, optional, tag = "1")]
+    pub response: ::core::option::Option<CanisterHttpResponse>,
+    #[prost(message, optional, tag = "2")]
+    pub proof: ::core::option::Option<CanisterHttpShare>,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct FlexibleCanisterHttpResponses {
+    #[prost(uint64, tag = "1")]
+    pub callback_id: u64,
+    #[prost(message, repeated, tag = "2")]
+    pub responses: ::prost::alloc::vec::Vec<FlexibleCanisterHttpResponseWithProof>,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct CanisterHttpResponseMessage {
     #[prost(
         oneof = "canister_http_response_message::MessageType",
-        tags = "1, 2, 3"
+        tags = "1, 2, 3, 4"
     )]
     pub message_type: ::core::option::Option<canister_http_response_message::MessageType>,
 }
@@ -629,6 +643,8 @@ pub mod canister_http_response_message {
         Timeout(u64),
         #[prost(message, tag = "3")]
         DivergenceResponse(super::CanisterHttpResponseDivergence),
+        #[prost(message, tag = "4")]
+        FlexibleResponses(super::FlexibleCanisterHttpResponses),
     }
 }
 #[derive(Clone, PartialEq, ::prost::Message)]

rs/types/types/src/batch.rs

@@ -9,7 +9,10 @@ mod vetkd;
 mod xnet;
 
 pub use self::{
-    canister_http::{CanisterHttpPayload, MAX_CANISTER_HTTP_PAYLOAD_SIZE},
+    canister_http::{
+        CanisterHttpPayload, FlexibleCanisterHttpResponseWithProof, FlexibleCanisterHttpResponses,
+        MAX_CANISTER_HTTP_PAYLOAD_SIZE,
+    },
     execution_environment::{
         CanisterCyclesCostSchedule, CanisterQueryStats, LocalQueryStats, QueryStats,
         QueryStatsPayload, RawQueryStats, TotalQueryStats,