Self healing open

.github/workflows/ci-verification.yml

@@ -233,3 +233,25 @@ jobs:
           name: tlc-trace-validation-consensus
           path: |
             tla/traces/*
+
+  model-checking-self-healing-open:
+    name: Model Checking - Self-Healing Open
+    runs-on: [self-hosted, 1ES.Pool=gha-vmss-d16av5-ci]
+    container:
+      image: mcr.microsoft.com/azurelinux/base/core:3.0
+      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
+
+    steps:
+      - name: "Checkout dependencies"
+        shell: bash
+        run: |
+          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+          tdnf -y update
+          tdnf -y install ca-certificates git
+
+      - uses: actions/checkout@v5
+      - name: Install Stateright dependencies
+        run: |
+          tdnf install -y cargo
+
+      - run: cd tla/disaster-recovery && cargo run check

CMakeLists.txt

@@ -380,6 +380,7 @@ endif()
 set(CCF_IMPL_SOURCE
     ${CCF_DIR}/src/enclave/main.cpp ${CCF_DIR}/src/enclave/thread_local.cpp
     ${CCF_DIR}/src/node/quote.cpp ${CCF_DIR}/src/node/uvm_endorsements.cpp
+    ${CCF_DIR}/src/node/self_healing_open_impl.cpp
 )
 
 add_ccf_static_library(
@@ -688,11 +689,20 @@ if(BUILD_TESTS)
     add_unit_test(
       frontend_test
       ${CMAKE_CURRENT_SOURCE_DIR}/src/node/rpc/test/frontend_test.cpp
-      ${CCF_DIR}/src/node/quote.cpp ${CCF_DIR}/src/node/uvm_endorsements.cpp
+      ${CCF_DIR}/src/node/quote.cpp
+      ${CCF_DIR}/src/node/uvm_endorsements.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/node/self_healing_open_impl.cpp
     )
     target_link_libraries(
-      frontend_test PRIVATE ${CMAKE_THREAD_LIBS_INIT} http_parser ccf_js
-                            ccf_endpoints ccfcrypto ccf_kv
+      frontend_test
+      PRIVATE ${CMAKE_THREAD_LIBS_INIT}
+              http_parser
+              ccf_js
+              ccf_endpoints
+              ccfcrypto
+              ccf_kv
+              uv
+              curl
     )
 
     add_unit_test(
@@ -718,11 +728,20 @@ if(BUILD_TESTS)
     add_unit_test(
       node_frontend_test
       ${CMAKE_CURRENT_SOURCE_DIR}/src/node/rpc/test/node_frontend_test.cpp
-      ${CCF_DIR}/src/node/quote.cpp ${CCF_DIR}/src/node/uvm_endorsements.cpp
+      ${CCF_DIR}/src/node/quote.cpp
+      ${CCF_DIR}/src/node/uvm_endorsements.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/node/self_healing_open_impl.cpp
     )
     target_link_libraries(
-      node_frontend_test PRIVATE ${CMAKE_THREAD_LIBS_INIT} http_parser ccf_js
-                                 ccf_endpoints ccfcrypto ccf_kv
+      node_frontend_test
+      PRIVATE ${CMAKE_THREAD_LIBS_INIT}
+              http_parser
+              ccf_js
+              ccf_endpoints
+              ccfcrypto
+              ccf_kv
+              uv
+              curl
     )
 
     add_unit_test(
@@ -1190,15 +1209,16 @@ if(BUILD_TESTS)
       10000
       --use-jwt
   )
-  add_test_bin(
-    curl_test ${CMAKE_CURRENT_SOURCE_DIR}/src/http/test/curl_test.cpp
-  )
-  target_link_libraries(curl_test PRIVATE curl uv http_parser)
 
   if(LONG_TESTS)
     add_e2e_test(
       NAME e2e_curl PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_curl.py
     )
+
+    add_test_bin(
+      curl_test ${CMAKE_CURRENT_SOURCE_DIR}/src/http/test/curl_test.cpp
+    )
+    target_link_libraries(curl_test PRIVATE curl uv http_parser)
   endif()
 endif()
 

doc/audit/builtin_maps.rst

@@ -571,4 +571,51 @@ While the contents themselves are encrypted, the table is public so as to be acc
 **Value** The mechanism by which the ledger secret was recovered.
 
 .. doxygenenum:: ccf::RecoveryType
-   :project: CCF
+   :project: CCF
+
+``self_healing_open.nodes``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Key** Intrinsic node ID: A string which is unique to a particular node role within a cluster. Currently its IP and port.
+
+**Value** 
+
+.. doxygenstruct:: SelfHealingOpenNodeInfo
+   :project: CCF
+   :members:
+
+``self_healing_open.gossip``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Key** Intrinsic node ID of the source of the gossip message.
+
+**Value**
+
+.. doxygenstruct:: ccf::self_healing_open::GossipRequest
+   :project: CCF
+   :members:
+
+``self_healing_open.chosen_node``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Value** The intrinsic node ID of the chosen node. This will either be the node this node voted for, or the node that is has received an `IAmOpen` message from.
+
+``self_healing_open.votes``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Key** Intrinsic node ID of the node which has voted for this node to be opened.
+
+``selfhealingopen.sm_state``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Value** State machine state of the self-healing open protocol.
+
+``selfhealingopen.timeout_sm_state``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Value** Timeout state machine state of the self-healing open protocol. Ticks based on `failover_timeout` and advances `selfhealingopen.sm_state` if it falls behind.
+
+``selfhealingopen.failover_open``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Value** Boolean flag indicating whether the latest self-healing-open recovery opened using a failover timeout.

doc/host_config_schema/cchost_config.json

@@ -428,6 +428,28 @@
                   "previous_sealed_ledger_secret_location": {
                     "type": ["string"],
                     "description": "Path to the sealed ledger secret folder, the ledger secrets for the recovered service will be unsealed from here instead of reconstructed from recovery shares."
+                  },
+                  "self_healing_open": {
+                    "type": "object",
+                    "properties": {
+                      "addresses": {
+                        "type": "array",
+                        "items": {
+                          "type": "string"
+                        },
+                        "description": "List of addresses (host:port) of the cluster that should open via self-healing-open"
+                      },
+                      "retry_timeout": {
+                        "type": "string",
+                        "default": "100ms",
+                        "description": "Interval (time string) at which the node re-sends self-healing-open messages. This should be significantly less than 'failover_timeout'"
+                      },
+                      "failover_timeout": {
+                        "type": "string",
+                        "default": "2000ms",
+                        "description": "Interval (time string) after which the node forcibly advances to the next phase of the self-healing-open protocol"
+                      }
+                    }
                   }
                 },
                 "required": ["previous_service_identity_file"],

doc/operations/recovery.rst

@@ -145,6 +145,94 @@ Which of these two paths is taken is noted in the `public:ccf.internal.last_reco
       ...
     $ /opt/ccf/bin/js_generic --config /path/to/config/file
 
+Self-Healing-Open recovery
+--------------------------
+
+In environments with limited orchestration or limited operator access, it is desirable to allow an automated disaster recovery without operator intervention.
+At a high level, Self-Healing-Open recovery allows recovering replicas to discover which node has the most up-to-date ledger and automatically recover the network using that ledger.
+
+There are two paths, a election path, and a very-high-availablity failover path.
+The election path ensures that if: all nodes restart and have full network connectivity, a majority of nodes' on-disk ledger contains every committed transaction, and no timeouts trigger; then there will be only one recovered network, then all committed transaction will be persisted.
+However, the election path can become stuck, in which case the failover path is designed to ensure progress.
+
+In the election path, nodes first gossip with each other, learning of the ledgers of other nodes.
+Once they have heard from every node they vote for the node with the best ledger.
+If a node receives votes from a majority of nodes, it invokes `transition-to-open` and notifies the other nodes to restart and join it.
+This path is illustrated below, and is guaranteed to succeed if all nodes can communicate and no timeouts trigger.
+
+.. mermaid::
+
+    sequenceDiagram
+      participant N1
+      participant N2
+      participant N3
+
+      Note over N1, N3: Gossip
+
+      N1 ->> N2: Gossip(Tx=1)
+      N1 ->> N3: Gossip(Tx=1)
+      N2 ->> N3: Gossip(Tx=2)
+      N3 ->> N2: Gossip(Tx=3)
+
+      Note over N1, N3: Vote
+      N2 ->> N3: Vote
+      N3 ->> N3: Vote
+
+      Note over N1, N3: Open/Join
+      N3 ->> N1: IAmOpen
+      N3 ->> N2: IAmOpen
+
+      Note over N1, N2: Restart
+
+      Note over N3: Transition-to-open
+
+      Note over N3: Local unsealing
+
+      Note over N3: Open
+
+      N1 ->> N3: Join
+      N2 ->> N3: Join
+
+In the failover path, each phase has a timeout to skip to the next phase if a failure has occurred.
+For example, the election path requires all nodes to communicate to advance from the gossip phase to the vote phase.
+However, if any node fails to recover, the election path is stuck.
+In this case, after a timeout, nodes will advance to the vote phase regardless of whether they have heard from all nodes, and vote for the best ledger they have heard of at that point.
+
+Unfortunately, this can lead to multiple forks of the service if different nodes cannot communicate with each other and timeout.
+Hence, we recommend setting the timeout substantially higher than the highest expected recovery time, to minimise the chance of this happening.
+To audit if timeouts were used to open the service, the `public:ccf.gov.selfhealingopen.failover_open` table tracks this.
+
+This failover path is illustrated below.
+
+.. mermaid::
+
+    sequenceDiagram
+      participant N1
+      participant N2
+      participant N3
+
+      Note over N1, N3: Gossip
+
+      N2 ->> N3: Gossip(Tx=2)
+      N3 ->> N2: Gossip(Tx=3)
+
+      Note over N1: Timeout
+      Note over N3: Timeout
+
+      Note over N1, N3: Vote
+
+      N1 ->> N1: Vote
+      N3 ->> N3: Vote
+      N2 ->> N3: Vote
+
+      Note over N1, N3: Open/Join
+
+      Note over N1: Transition-to-open
+      Note over N3: Transition-to-open
+
+
+If the network fails during reconfiguration, each node will use its latest known configuration to recover. Since reconfiguration requires votes from a majority of nodes, the latest configuration should recover using the election path, however nodes in the previous configuration may recover using the election path.
+
 Notes
 -----
 

include/ccf/node/startup_config.h

@@ -102,6 +102,14 @@ namespace ccf
     Snapshots snapshots = {};
   };
 
+  struct SelfHealingOpenConfig
+  {
+    std::vector<std::string> addresses;
+    ccf::ds::TimeString retry_timeout = {"100ms"};
+    ccf::ds::TimeString failover_timeout = {"2000ms"};
+    bool operator==(const SelfHealingOpenConfig&) const = default;
+  };
+
   struct StartupConfig : CCFConfig
   {
     StartupConfig() = default;
@@ -146,6 +154,7 @@ namespace ccf
         std::nullopt;
       std::optional<std::string> previous_sealed_ledger_secret_location =
         std::nullopt;
+      std::optional<SelfHealingOpenConfig> self_healing_open = std::nullopt;
     };
     Recover recover = {};
   };

include/ccf/service/tables/self_healing_open.h

@@ -0,0 +1,76 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ccf/ds/enum_formatter.h"
+#include "ccf/ds/json.h"
+#include "ccf/ds/quote_info.h"
+#include "ccf/service/map.h"
+
+using IntrinsicIdentifier = std::string;
+
+struct SelfHealingOpenNodeInfo
+{
+  ccf::QuoteInfo quote_info;
+  std::string published_network_address;
+  std::vector<uint8_t> cert_der;
+  std::string service_identity;
+  IntrinsicIdentifier intrinsic_id;
+};
+
+DECLARE_JSON_TYPE(SelfHealingOpenNodeInfo);
+DECLARE_JSON_REQUIRED_FIELDS(
+  SelfHealingOpenNodeInfo,
+  quote_info,
+  published_network_address,
+  cert_der,
+  service_identity,
+  intrinsic_id);
+
+enum class SelfHealingOpenSM
+{
+  GOSSIPING = 0,
+  VOTING,
+  OPENING, // by chosen node
+  JOINING, // by all other replicas
+  OPEN,
+};
+
+DECLARE_JSON_ENUM(
+  SelfHealingOpenSM,
+  {{SelfHealingOpenSM::GOSSIPING, "Gossiping"},
+   {SelfHealingOpenSM::VOTING, "Voting"},
+   {SelfHealingOpenSM::OPENING, "Opening"},
+   {SelfHealingOpenSM::JOINING, "Joining"},
+   {SelfHealingOpenSM::OPEN, "Open"}});
+
+namespace ccf
+{
+  using SelfHealingOpenNodeInfoMap =
+    ServiceMap<IntrinsicIdentifier, ::SelfHealingOpenNodeInfo>;
+  using SelfHealingOpenGossips =
+    ServiceMap<IntrinsicIdentifier, ccf::kv::Version>;
+  using SelfHealingOpenChosenNode = ServiceValue<IntrinsicIdentifier>;
+  using SelfHealingOpenVotes = ServiceSet<IntrinsicIdentifier>;
+  using SelfHealingOpenSMState = ServiceValue<SelfHealingOpenSM>;
+  using SelfHealingOpenTimeoutSMState = ServiceValue<SelfHealingOpenSM>;
+  using SelfHealingOpenFailoverFlag = ServiceValue<bool>;
+
+  namespace Tables
+  {
+    static constexpr auto SELF_HEALING_OPEN_NODES =
+      "public:ccf.gov.selfhealingopen.nodes";
+    static constexpr auto SELF_HEALING_OPEN_GOSSIPS =
+      "public:ccf.gov.selfhealingopen.gossip";
+    static constexpr auto SELF_HEALING_OPEN_CHOSEN_NODE =
+      "public:ccf.gov.selfhealingopen.chosen_node";
+    static constexpr auto SELF_HEALING_OPEN_VOTES =
+      "public:ccf.gov.selfhealingopen.votes";
+    static constexpr auto SELF_HEALING_OPEN_SM_STATE =
+      "public:ccf.gov.selfhealingopen.sm_state";
+    static constexpr auto SELF_HEALING_OPEN_TIMEOUT_SM_STATE =
+      "public:ccf.gov.selfhealingopen.timeout_sm_state";
+    static constexpr auto SELF_HEALING_OPEN_FAILOVER_FLAG =
+      "public:ccf.gov.selfhealingopen.failover_open";
+  }
+}