Merge branch 'main' into dependabot/npm_and_yarn/dashboard/secp256k1-5.0.1

Author: paulo-ocean

.env.example

@@ -27,7 +27,7 @@ export INDEXER_INTERVAL=
 export ALLOWED_ADMINS=
 export DASHBOARD=true
 export RATE_DENY_LIST=
-export MAX_REQ_PER_SECOND=
+export MAX_REQ_PER_MINUTE=
 export MAX_CHECKSUM_LENGTH=
 export LOG_LEVEL=
 export HTTP_API_PORT=

.github/workflows/ci.yml

@@ -132,15 +132,6 @@ jobs:
       - run: docker image ls
       - name: Delete default runner images
         run: |
-          docker image rm node:20
-          docker image rm node:20-alpine
-          docker image rm node:18
-          docker image rm node:18-alpine
-          docker image rm debian:10
-          docker image rm debian:11
-          docker image rm ubuntu:22.04
-          docker image rm ubuntu:20.04
-          docker image rm moby/buildkit:latest
           rm -rf /usr/share/swift/
       - name: Wait for contracts deployment and C2D cluster to be ready
         working-directory: ${{ github.workspace }}/barge
@@ -226,15 +217,6 @@ jobs:
       - run: docker image ls
       - name: Delete default runner images
         run: |
-          docker image rm node:20
-          docker image rm node:20-alpine
-          docker image rm node:18
-          docker image rm node:18-alpine
-          docker image rm debian:10
-          docker image rm debian:11
-          docker image rm ubuntu:22.04
-          docker image rm ubuntu:20.04
-          docker image rm moby/buildkit:latest
           rm -rf /usr/share/swift/
 
       - name: Wait for contracts deployment and C2D cluster to be ready
@@ -278,6 +260,8 @@ jobs:
           P2P_ENABLE_AUTONAT: 'false'
           ALLOWED_ADMINS: '["0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"]'
           DB_TYPE: 'elasticsearch'
+          MAX_REQ_PER_MINUTE: 320
+          MAX_CONNECTIONS_PER_MINUTE: 320
       - name: Check Ocean Node is running
         run: |
           for i in $(seq 1 90); do

docs/API.md

@@ -449,6 +449,38 @@ returns P2P peer
 
 ---
 
+## find peer multiaddress
+
+### `HTTP` GET /findPeer/?
+
+#### Description
+
+returns P2P peer multiaddresses if found in DHT
+
+#### Query Parameters
+
+| name    | type   | required | description |
+| ------- | ------ | -------- | ----------- |
+| peerId  | string | v        | peer id     |
+| timeout | int    | optional | timeout     |
+
+#### Response
+
+```
+{
+    "id": "16Uiu2HAmLhRDqfufZiQnxvQs2XHhd6hwkLSPfjAQg1gH8wgRixiP",
+    "multiaddrs": [
+        "/ip4/127.0.0.1/tcp/9000",
+        "/ip4/127.0.0.1/tcp/9001/ws",
+        "/ip4/172.18.0.2/tcp/9000",
+        "/ip4/172.18.0.2/tcp/9001/ws",
+        "/ip6/::1/tcp/9002"
+    ]
+}
+```
+
+---
+
 ## Get P2P Peers
 
 ### `HTTP` GET /getP2PPeers
@@ -501,13 +533,14 @@ returns an empty object if it is valid otherwise an array with error
 
 #### Parameters
 
-| name       | type   | required | description                                       |
-| ---------- | ------ | -------- | ------------------------------------------------- |
-| command    | string | v        | command name                                      |
-| node       | string |          | if not present it means current node              |
-| id         | string | v        | document id or did                                |
-| chainId    | number | v        | chain id of network on which document is provided |
-| nftAddress | string | v        | address of nft token                              |
+| name       | type     | required | description                                       |
+| ---------- | -------- | -------- | ------------------------------------------------- |
+| command    | string   | v        | command name                                      |
+| node       | string   |          | if not present it means current node              |
+| multiAddrs | string[] |          | if passed, use this instead of peerStore & DHT     |
+| id         | string   | v        | document id or did                                |
+| chainId    | number   | v        | chain id of network on which document is provided |
+| nftAddress | string   | v        | address of nft token                              |
 
 #### Request
 

docs/dockerDeployment.md

@@ -99,7 +99,7 @@ services:
       #      INDEXER_INTERVAL: ''
       DASHBOARD: 'true'
       #      RATE_DENY_LIST: ''
-      #      MAX_REQ_PER_SECOND: ''
+      #      MAX_REQ_PER_MINUTE: ''
       #      MAX_CHECKSUM_LENGTH: ''
       #      LOG_LEVEL: ''
       HTTP_API_PORT: '8000'

docs/env.md

@@ -23,7 +23,8 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/
 - `ALLOWED_ADMINS`: Sets the public address of accounts which have access to admin endpoints e.g. shutting down the node. Example: `"[\"0x967da4048cD07aB37855c090aAF366e4ce1b9F48\",\"0x388C818CA8B9251b393131C08a736A67ccB19297\"]"`
 - `DASHBOARD`: If `false` the dashboard will not run. If not set or `true` the dashboard will start with the node. Example: `false`
 - `RATE_DENY_LIST`: Blocked list of IPs and peer IDs. Example: `"{ \"peers\": [\"16Uiu2HAkuYfgjXoGcSSLSpRPD6XtUgV71t5RqmTmcqdbmrWY9MJo\"], \"ips\": [\"127.0.0.1\"] }"`
-- `MAX_REQ_PER_SECOND`: Number of requests per second allowed by the same client. Example: `3`
+- `MAX_REQ_PER_MINUTE`: Number of requests per minute allowed by the same client (IP or Peer id). Example: `30`
+- `MAX_CONNECTIONS_PER_MINUTE`: Max number of requests allowed per minute (all clients). Example: `120`
 - `MAX_CHECKSUM_LENGTH`: Define the maximum length for a file if checksum is required (Mb). Example: `10`
 - `IS_BOOTSTRAP`: Is this node to be used as bootstrap node or not. Default is `false`.
 
@@ -54,7 +55,7 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/
 - `P2P_pubsubPeerDiscoveryInterval`: Interval (in ms) for discovery using pubsub. Defaults to `10000` (three seconds). Example: `10000`
 - `P2P_dhtMaxInboundStreams`: Maximum number of DHT inbound streams. Defaults to `500`. Example: `500`
 - `P2P_dhtMaxOutboundStreams`: Maximum number of DHT outbound streams. Defaults to `500`. Example: `500`
-- `P2P_ENABLE_DHT_SERVER`: Enable DHT server mode. This should be enabled for bootstrapers & well established nodes. Default: `false`
+- `P2P_DHT_FILTER`: Filter address in DHT. 0 = (Default) No filter 1. Filter private ddresses. 2. Filter public addresses
 - `P2P_mDNSInterval`: Interval (in ms) for discovery using mDNS. Defaults to `20000` (20 seconds). Example: `20000`
 - `P2P_connectionsMaxParallelDials`: Maximum number of parallel dials. Defaults to `150`. Example: `150`
 - `P2P_connectionsDialTimeout`: Timeout for dial commands. Defaults to `10000` (10 seconds). Example: `10000`

scripts/ocean-node-quickstart.sh

@@ -155,7 +155,8 @@ services:
 #      INDEXER_INTERVAL: ''
       DASHBOARD: 'true'
 #      RATE_DENY_LIST: ''
-#      MAX_REQ_PER_SECOND: ''
+#      MAX_REQ_PER_MINUTE: ''
+#      MAX_CONNECTIONS_PER_MINUTE: ''
 #      MAX_CHECKSUM_LENGTH: ''
 #      LOG_LEVEL: ''
       HTTP_API_PORT: '$HTTP_API_PORT'

src/@types/OceanNode.ts

@@ -23,7 +23,12 @@ export interface OceanNodeKeys {
   privateKey: any
   ethAddress: string
 }
-
+/* eslint-disable no-unused-vars */
+export enum dhtFilterMethod {
+  filterPrivate = 'filterPrivate', // default, remove all private addresses from DHT
+  filterPublic = 'filterPublic', // remove all public addresses from DHT
+  filterNone = 'filterNone' // do not remove all any addresses from DHT
+}
 export interface OceanNodeP2PConfig {
   bootstrapNodes: string[]
   bootstrapTimeout: number
@@ -41,7 +46,7 @@ export interface OceanNodeP2PConfig {
   pubsubPeerDiscoveryInterval: number
   dhtMaxInboundStreams: number
   dhtMaxOutboundStreams: number
-  enableDHTServer: boolean
+  dhtFilter: dhtFilterMethod
   mDNSInterval: number
   connectionsMaxParallelDials: number
   connectionsDialTimeout: number
@@ -91,7 +96,8 @@ export interface OceanNodeConfig {
   assetPurgatoryUrl: string
   allowedAdmins?: string[]
   codeHash?: string
-  rateLimit?: number
+  rateLimit?: number // per request ip or peer
+  maxConnections?: number // global, regardless of client address(es)
   denyList?: DenyList
   unsafeURLs?: string[]
   isBootstrap?: boolean

src/OceanNode.ts

@@ -11,6 +11,17 @@ import { pipe } from 'it-pipe'
 import { GENERIC_EMOJIS, LOG_LEVELS_STR } from './utils/logging/Logger.js'
 import { Handler } from './components/core/handler/handler.js'
 import { C2DEngines } from './components/c2d/compute_engines.js'
+
+export interface RequestLimiter {
+  requester: string | string[] // IP address or peer ID
+  lastRequestTime: number // time of the last request done (in miliseconds)
+  numRequests: number // number of requests done in the specific time period
+}
+
+export interface RequestDataCheck {
+  valid: boolean
+  updatedRequestData: RequestLimiter
+}
 export class OceanNode {
   // eslint-disable-next-line no-use-before-define
   private static instance: OceanNode
@@ -20,6 +31,7 @@ export class OceanNode {
   private c2dEngines: C2DEngines
   // requester
   private remoteCaller: string | string[]
+  private requestMap: Map<string, RequestLimiter>
   // eslint-disable-next-line no-useless-constructor
   private constructor(
     private db?: Database,
@@ -28,6 +40,7 @@ export class OceanNode {
     private indexer?: OceanIndexer
   ) {
     this.coreHandlers = CoreHandlersRegistry.getInstance(this)
+    this.requestMap = new Map<string, RequestLimiter>()
     if (node) {
       node.setCoreHandlers(this.coreHandlers)
     }
@@ -95,6 +108,14 @@ export class OceanNode {
     return this.remoteCaller
   }
 
+  public getRequestMapSize(): number {
+    return this.requestMap.size
+  }
+
+  public getRequestMap(): Map<string, RequestLimiter> {
+    return this.requestMap
+  }
+
   /**
    * Use this method to direct calls to the node as node cannot dial into itself
    * @param message command message

src/components/P2P/handleProtocolCommands.ts

@@ -9,6 +9,16 @@ import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../utils/logging/Logger.js'
 import StreamConcat from 'stream-concat'
 import { Handler } from '../core/handler/handler.js'
 import { getConfiguration } from '../../utils/index.js'
+import { checkConnectionsRateLimit } from '../httpRoutes/requestValidator.js'
+import { CONNECTIONS_RATE_INTERVAL } from '../../utils/constants.js'
+import { RequestLimiter } from '../../OceanNode.js'
+
+// hold data about last request made
+const connectionsData: RequestLimiter = {
+  lastRequestTime: Date.now(),
+  requester: '',
+  numRequests: 0
+}
 
 export class ReadableString extends Readable {
   private sent = false
@@ -60,10 +70,14 @@ export async function handleProtocolCommands(otherPeerConnection: any) {
     return status
   }
 
-  const denyList = await (await getConfiguration()).denyList
+  const configuration = await getConfiguration()
+  // check deny list configs
+  const { denyList } = configuration
   if (denyList.peers.length > 0) {
     if (denyList.peers.includes(remotePeer.toString())) {
-      P2P_LOGGER.error(`Incoming request denied to peer: ${remotePeer}`)
+      P2P_LOGGER.warn(
+        `Incoming request denied to peer: ${remotePeer} (peer its on deny list)`
+      )
 
       if (connectionStatus === 'open') {
         statusStream = new ReadableString(
@@ -79,6 +93,37 @@ export async function handleProtocolCommands(otherPeerConnection: any) {
       return
     }
   }
+  // check connections rate limit
+  const requestTime = Date.now()
+  if (requestTime - connectionsData.lastRequestTime > CONNECTIONS_RATE_INTERVAL) {
+    // last one was more than 1 minute ago? reset counter
+    connectionsData.numRequests = 0
+  }
+  // always increment counter
+  connectionsData.numRequests += 1
+  // update time and requester information
+  connectionsData.lastRequestTime = requestTime
+  connectionsData.requester = remoteAddr
+
+  // check global rate limits (not ip related)
+  const requestRateValidation = checkConnectionsRateLimit(configuration, connectionsData)
+  if (!requestRateValidation.valid) {
+    P2P_LOGGER.warn(
+      `Incoming request denied to peer: ${remotePeer} (rate limit exceeded)`
+    )
+    if (connectionStatus === 'open') {
+      statusStream = new ReadableString(
+        JSON.stringify(buildWrongCommandStatus(403, 'Rate limit exceeded'))
+      )
+      try {
+        await pipe(statusStream, otherPeerConnection.stream.sink)
+      } catch (e) {
+        P2P_LOGGER.error(e)
+      }
+    }
+    await closeStreamConnection(otherPeerConnection.connection, remotePeer)
+    return
+  }
 
   try {
     // eslint-disable-next-line no-unreachable-loop