Merge pull request #710 from LIT-Protocol/LIT-4016-epoch-change-events

fix(lit-core): LIT-4016 - Enhance error handling during epoch changes

Author: Ansonhkg

local-tests/setup/tinny-environment.ts

@@ -277,6 +277,12 @@ export class TinnyEnvironment {
       );
     }
 
+    this.litNodeClient.on('connected', () => {
+      console.log(
+        'Received `connected` event from `litNodeClient. Ready to go!'
+      );
+    });
+
     await this.litNodeClient.connect();
 
     if (!this.litNodeClient.ready) {

package.json

@@ -60,6 +60,7 @@
     "date-and-time": "^2.4.1",
     "depd": "^2.0.0",
     "ethers": "^5.7.1",
+    "eventemitter3": "^5.0.1",
     "jose": "^4.14.4",
     "micromodal": "^0.4.10",
     "multiformats": "^9.7.1",

packages/core/src/lib/lit-core.ts

@@ -1,4 +1,5 @@
 import { ethers } from 'ethers';
+import { EventEmitter } from 'eventemitter3';
 
 import {
   canonicalAccessControlConditionFormatter,
@@ -140,7 +141,7 @@ const FALLBACK_RPC_URLS = [
   'https://eth.llamarpc.com',
 ];
 
-export class LitCore {
+export class LitCore extends EventEmitter {
   config: LitNodeClientConfigWithDefaults = {
     alertWhenUnauthorized: false,
     debug: true,
@@ -173,6 +174,8 @@ export class LitCore {
 
   // ========== Constructor ==========
   constructor(config: LitNodeClientConfig | CustomNetwork) {
+    super();
+
     if (!(config.litNetwork in LIT_NETWORKS)) {
       const validNetworks = Object.keys(LIT_NETWORKS).join(', ');
       throw new InvalidParamType(
@@ -300,20 +303,16 @@ export class LitCore {
   private async _handleStakingContractStateChange(
     state: STAKING_STATES_VALUES
   ) {
-    log(`New state detected: "${state}"`);
-
-    const validatorData = await this._getValidatorData();
-
-    if (state === STAKING_STATES.Active) {
-      // We always want to track the most recent epoch number on _all_ networks
+    try {
+      log(`New state detected: "${state}"`);
 
-      this._epochState = await this._fetchCurrentEpochState(
-        validatorData.epochInfo
-      );
+      if (state === STAKING_STATES.Active) {
+        const validatorData = await this._getValidatorData();
 
-      if (CENTRALISATION_BY_NETWORK[this.config.litNetwork] !== 'centralised') {
         // We don't need to handle node urls changing on centralised networks, since their validator sets are static
-        try {
+        if (
+          CENTRALISATION_BY_NETWORK[this.config.litNetwork] !== 'centralised'
+        ) {
           log(
             'State found to be new validator set locked, checking validator set'
           );
@@ -322,39 +321,54 @@ export class LitCore {
           const delta: string[] = validatorData.bootstrapUrls.filter((item) =>
             existingNodeUrls.includes(item)
           );
-          // if the sets differ we reconnect.
+
+          // check if the node sets are non-matching and re-connect if they do not.
           if (delta.length > 1) {
-            // check if the node sets are non-matching and re-connect if they do not.
             /*
-                TODO: This covers *most* cases where a node may come in or out of the active
-                set which we will need to re attest to the execution environments.
-                However, the sdk currently does not know if there is an active network operation pending.
-                Such that the state when the request was sent will now mutate when the response is sent back.
-                The sdk should be able to understand its current execution environment and wait on an active
-                network request to the previous epoch's node set before changing over.
-              */
+                  TODO: This covers *most* cases where a node may come in or out of the active
+                  set which we will need to re attest to the execution environments.
+                  However, the sdk currently does not know if there is an active network operation pending.
+                  Such that the state when the request was sent will now mutate when the response is sent back.
+                  The sdk should be able to understand its current execution environment and wait on an active
+                  network request to the previous epoch's node set before changing over.
+                */
             log(
               'Active validator sets changed, new validators ',
               delta,
               'starting node connection'
             );
+            await this.connect(); // Will update `epochInfo`
           }
-
-          await this.connect();
-        } catch (err: unknown) {
-          // FIXME: We should emit an error event so that consumers know that we are de-synced and can connect() again
-          // But for now, our every-30-second network sync will fix things in at most 30s from now.
-          // this.ready = false; Should we assume core is invalid if we encountered errors refreshing from an epoch change?
-          const { message = '' } = err as
-            | Error
-            | NodeClientErrorV0
-            | NodeClientErrorV1;
-          logError(
-            'Error while attempting to reconnect to nodes after epoch transition:',
-            message
+        } else {
+          // In case of centralised networks, we don't run `connect()` flow, so we will manually update epochInfo here
+          this._epochState = await this._fetchCurrentEpochState(
+            validatorData.epochInfo
           );
         }
       }
+    } catch (err: unknown) {
+      // Ensure that any methods that check `this.ready` throw errors to the caller, and any consumers can check appropriately
+      this.ready = false;
+
+      const { message = '' } = err as
+        | Error
+        | NodeClientErrorV0
+        | NodeClientErrorV1;
+      logError(
+        'Error while attempting to reconnect to nodes after epoch transition:',
+        message
+      );
+
+      const handshakeError = new Error(
+        'Error while attempting to reconnect to nodes after epoch transition:' +
+          message
+      );
+
+      // Signal to any listeners that we've encountered a fatal error
+      this.emit('error', handshakeError);
+
+      // Signal to any listeners that we're 'disconnected' from LIT network
+      this.emit('disconnected', { reason: 'error', error: handshakeError });
     }
   }
 
@@ -399,6 +413,7 @@ export class LitCore {
     this._stopListeningForNewEpoch();
     // this._stopNetworkPolling();
     setMiscLitConfig(undefined);
+    this.emit('disconnected', { reason: 'disconnect' });
   }
 
   // _stopNetworkPolling() {
@@ -477,11 +492,6 @@ export class LitCore {
   }
 
   private async _connect() {
-    // Ensure an ill-timed epoch change event doesn't trigger concurrent config changes while we're already doing that
-    this._stopListeningForNewEpoch();
-    // Ensure we don't fire an existing network sync poll handler while we're in the midst of connecting anyway
-    // this._stopNetworkPolling();
-
     // Initialize a contractContext if we were not given one; this allows interaction against the staking contract
     // to be handled locally from then on
     if (!this.config.contractContext) {
@@ -523,7 +533,6 @@ export class LitCore {
     }
 
     // Re-use staking contract instance from previous connect() executions that succeeded to improve performance
-    // noinspection ES6MissingAwait - intentionally not `awaiting` so we can run this in parallel below
     const validatorData = await this._getValidatorData();
 
     this._stakingContract = validatorData.stakingContract;
@@ -554,6 +563,8 @@ export class LitCore {
       latestBlockhash: this.latestBlockhash,
     });
 
+    this.emit('connected', true);
+
     // browser only
     if (isBrowser()) {
       document.dispatchEvent(new Event('lit-ready'));

packages/lit-node-client-nodejs/src/lib/lit-node-client-nodejs.ts

@@ -874,7 +874,7 @@ export class LitNodeClientNodeJs
     // ========== Validate Params ==========
     if (!this.ready) {
       const message =
-        '[executeJs] LitNodeClient is not ready.  Please call await litNodeClient.connect() first.';
+        '[executeJs] LitNodeClient is not ready. Please call await litNodeClient.connect() first.';
 
       throw new LitNodeClientNotReadyError({}, message);
     }
@@ -1988,6 +1988,14 @@ export class LitNodeClientNodeJs
 
     const signatures: SessionSigsMap = {};
 
+    if (!this.ready) {
+      // If the client isn't ready, `connectedNodes` may be out-of-date, and we should throw an error immediately
+      const message =
+        '[executeJs] LitNodeClient is not ready. Please call await litNodeClient.connect() first.';
+
+      throw new LitNodeClientNotReadyError({}, message);
+    }
+
     this.connectedNodes.forEach((nodeAddress: string) => {
       const toSign: SessionSigningTemplate = {
         ...signingTemplate,