MB-42256: Handle seqno ack responses

We may receive a seqno ack response if a vBucket is deleted at the
"wrong" time. The consumer node must be able to handle this instead of
taking down the DCP connection or it can cause rebalances to fail that
should not

Change-Id: I26c4ca5301a64cffad8bcc8a5c4f547b8fd9d86f
Reviewed-on: http://review.couchbase.org/c/kv_engine/+/139108
Well-Formed: Build Bot <[email protected]>
Tested-by: Ben Huddleston <[email protected]>
Reviewed-by: Dave Rigby <[email protected]>

Author: BenHuddleston

daemon/mcbp_executors.cc

@@ -654,6 +654,8 @@ void initialize_mbcp_lookup_map() {
                            process_bin_dcp_response);
     setup_response_handler(cb::mcbp::ClientOpcode::DcpAbort,
                            process_bin_dcp_response);
+    setup_response_handler(cb::mcbp::ClientOpcode::DcpSeqnoAcknowledged,
+                           process_bin_dcp_response);
     setup_response_handler(cb::mcbp::ClientOpcode::GetErrorMap,
                            process_bin_dcp_response);
 

engines/ep/src/dcp/consumer.cc

@@ -1035,6 +1035,12 @@ bool DcpConsumer::handleResponse(const protocol_binary_response_header* resp) {
         producerIsVersion5orHigher = status != cb::mcbp::Status::UnknownCommand;
         getErrorMapState = GetErrorMapState::Skip;
         return true;
+    } else if (opcode == cb::mcbp::ClientOpcode::DcpSeqnoAcknowledged) {
+        // Seqno ack might respond in a non-success case if the vBucket has gone
+        // away on the producer. We don't really care if this happens, the
+        // stream has probably already gone away, but we don't want to take down
+        // the connection (return false) as it might cause a rebalance to fail.
+        return true;
     }
 
     logger->warn("Trying to handle an unknown response {}, disconnecting",

engines/ep/tests/module_tests/dcp_test.cc

@@ -2988,6 +2988,22 @@ TEST_P(ConnectionTest, ReplicateJustBeforeThrottleThreshold) {
     sendConsumerMutationsNearThreshold(false);
 }
 
+TEST_P(ConnectionTest, ConsumerHandlesSeqnoAckResponse) {
+    const void* cookie = create_mock_cookie();
+
+    /* Set up a consumer connection */
+    auto consumer =
+            std::make_shared<MockDcpConsumer>(*engine, cookie, "test_consumer");
+
+    protocol_binary_response_header resp{};
+    resp.response.setMagic(cb::mcbp::Magic::AltClientResponse);
+    resp.response.setOpcode(cb::mcbp::ClientOpcode::DcpSeqnoAcknowledged);
+    resp.response.setStatus(cb::mcbp::Status::NotMyVbucket);
+    EXPECT_TRUE(consumer->handleResponse(&resp));
+
+    destroy_mock_cookie(cookie);
+}
+
 void ConnectionTest::processConsumerMutationsNearThreshold(
         bool beyondThreshold) {
     const void* cookie = create_mock_cookie();

tests/testapp/testapp_durability.cc

@@ -168,6 +168,54 @@ TEST_P(DurabilityTest, GetAndTouchNotSupported) {
     executeTouchOrGatCommand(ClientOpcode::Gat);
 }
 
+TEST_P(DurabilityTest, AckResponseHandled) {
+    if (mcd_env->getTestBucket().getName() != "default_engine") {
+        // Will fail in EP engine if the DCP consumer isn't present
+        return;
+    }
+    // Send our response to see if the executor accepts it. If not, it would
+    // disconnect the connection
+    std::vector<uint8_t> rspBuffer(1024);
+    ResponseBuilder rspBuilder({rspBuffer.data(), rspBuffer.size()});
+    rspBuilder.setOpcode(cb::mcbp::ClientOpcode::DcpSeqnoAcknowledged);
+    rspBuilder.setMagic(Magic::AltClientResponse);
+    rspBuilder.setFramingExtras(encode(cb::durability::Requirements()));
+    rspBuilder.setKey(
+            {reinterpret_cast<const uint8_t*>(name.data()), name.size()});
+    rspBuffer.resize(rspBuilder.getFrame()->getFrame().size());
+
+    Frame rspFrame;
+    rspFrame.payload = std::move(rspBuffer);
+
+    auto& conn = getConnection();
+    conn.sendFrame(rspFrame);
+
+    // Send something else, a GAT in this case, to test that the connection is
+    // still up
+    std::vector<uint8_t> reqBuffer(1024);
+    RequestBuilder reqBuilder({reqBuffer.data(), reqBuffer.size()});
+    std::string value = "";
+    reqBuilder.setOpcode(ClientOpcode::Gat);
+    reqBuilder.setMagic(Magic::AltClientRequest);
+    reqBuilder.setFramingExtras(encode(cb::durability::Requirements()));
+    reqBuilder.setExtras(request::GatPayload().getBuffer());
+    reqBuilder.setKey(
+            {reinterpret_cast<const uint8_t*>(name.data()), name.size()});
+    reqBuilder.setValue(
+            {reinterpret_cast<const uint8_t*>(value.data()), value.size()});
+    reqBuffer.resize(reqBuilder.getFrame()->getFrame().size());
+
+    Frame reqFrame;
+    reqFrame.payload = std::move(reqBuffer);
+    conn.sendFrame(reqFrame);
+
+    BinprotResponse resp;
+    conn.recvResponse(resp);
+
+    EXPECT_EQ(Status::NotSupported, resp.getStatus());
+    EXPECT_NE(0xdeadbeef, ntohll(resp.getCas()));
+}
+
 class SubdocDurabilityTest : public DurabilityTest {
 protected:
     void SetUp() override {