feat: HTTP external functions SDK support (#33)

* feat: add registerHttpFunction API for HTTP external functions

- Node: registerHttpFunction(id, config), HttpInvocationConfig types, integration test
- Rust: register_http_function, HttpInvocationConfig, HttpFunctionRef, unit tests
- Python: register_http_function, HttpInvocationConfig, unit + integration tests
- Add HttpFunctionsModule to config-test.yaml for integration tests

* feat(tests): enhance HTTP external functions tests for registration, event delivery, and unregistration

- Added tests for registering and unregistering HTTP functions, ensuring proper function lifecycle management.
- Implemented tests for delivering events with custom headers and to multiple external functions across different topics.
- Included a test to verify that events stop being delivered after unregistration.
- Enhanced the WebhookProbe class to capture headers from incoming requests for more comprehensive testing.
- Updated integration tests to cover all configuration options for HTTP functions, including authentication and custom headers.

* feat: add validation for duplicate function IDs in registration

- Implemented checks in Node, Python, and Rust SDKs to prevent registration of functions with duplicate IDs, enhancing error handling and ensuring unique function identifiers.
- Updated tests to verify the new validation logic for HTTP external functions and standard functions, ensuring robustness in function registration processes.

* fix: improve enum variant handling and clean up serialization code

- Added a Clippy lint allowance for large enum variants in the `Outbound` enum to enhance code quality.
- Simplified the serialization call in the tests by removing unnecessary method chaining, improving readability and maintainability.

* feat: update test configuration for HTTP functions and API module

- Added a new port configuration for the HTTP functions module.
- Updated the port for the Rest API module to 3199.
- Introduced security settings for the HTTP functions module, including URL allowlist and HTTPS requirements.

Author: ytallo

.github/engine-config/test-config.yml

@@ -1,3 +1,4 @@
+port: 49134
 modules:
   - class: modules::stream::StreamModule
     config:
@@ -19,7 +20,7 @@ modules:
 
   - class: modules::api::RestApiModule
     config:
-      port: 3111
+      port: 3199
       host: 127.0.0.1
       default_timeout: 30000
       concurrency_request_limit: 1024
@@ -59,6 +60,15 @@ modules:
       adapter:
         class: modules::queue::BuiltinQueueAdapter
 
+  - class: modules::http_functions::HttpFunctionsModule
+    config:
+      security:
+        url_allowlist:
+          - localhost
+          - 127.0.0.1
+        block_private_ips: false
+        require_https: false
+
   - class: modules::pubsub::PubSubModule
     config:
       adapter:

packages/node/iii/src/iii-types.ts

@@ -53,6 +53,19 @@ export type RegisterServiceMessage = {
   parent_service_id?: string
 }
 
+export type HttpAuthConfig =
+  | { type: 'hmac'; secret_key: string }
+  | { type: 'bearer'; token_key: string }
+  | { type: 'api_key'; header: string; value_key: string }
+
+export type HttpInvocationConfig = {
+  url: string
+  method?: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE'
+  timeout_ms?: number
+  headers?: Record<string, string>
+  auth?: HttpAuthConfig
+}
+
 export type RegisterFunctionFormat = {
   name: string
   /**
@@ -80,7 +93,7 @@ export type RegisterFunctionFormat = {
 export type RegisterFunctionMessage = {
   message_type: MessageType.RegisterFunction
   /**
-   * The path of the function
+   * The path of the function (use :: for namespacing, e.g. external::my_lambda)
    */
   id: string
   /**
@@ -96,6 +109,10 @@ export type RegisterFunctionMessage = {
    */
   response_format?: RegisterFunctionFormat
   metadata?: Record<string, unknown>
+  /**
+   * HTTP invocation config for external HTTP functions (Lambda, Cloudflare Workers, etc.)
+   */
+  invocation?: HttpInvocationConfig
 }
 
 export type InvokeFunctionMessage = {

packages/node/iii/src/iii.ts

@@ -14,6 +14,7 @@ import {
 import {
   type IIIMessage,
   type FunctionInfo,
+  type HttpInvocationConfig,
   type InvocationResultMessage,
   type InvokeFunctionMessage,
   MessageType,
@@ -101,6 +102,7 @@ export type InitOptions = {
 class Sdk implements ISdk {
   private ws?: WebSocket
   private functions = new Map<string, RemoteFunctionData>()
+  private httpFunctions = new Map<string, RegisterFunctionMessage>()
   private services = new Map<string, Omit<RegisterServiceMessage, 'functions'>>()
   private invocations = new Map<string, Invocation & { timeout?: NodeJS.Timeout }>()
   private triggers = new Map<string, RegisterTriggerMessage>()
@@ -190,6 +192,9 @@ class Sdk implements ISdk {
     if (!message.id || message.id.trim() === '') {
       throw new Error('id is required')
     }
+    if (this.httpFunctions.has(message.id)) {
+      throw new Error(`function id already registered: ${message.id}`)
+    }
 
     this.sendMessage(MessageType.RegisterFunction, message, true)
     this.functions.set(message.id, {
@@ -230,6 +235,38 @@ class Sdk implements ISdk {
     }
   }
 
+  registerHttpFunction = (id: string, config: HttpInvocationConfig): FunctionRef => {
+    if (!id || id.trim() === '') {
+      throw new Error('id is required')
+    }
+    if (this.functions.has(id) || this.httpFunctions.has(id)) {
+      throw new Error(`function id already registered: ${id}`)
+    }
+
+    const message: RegisterFunctionMessage = {
+      message_type: MessageType.RegisterFunction,
+      id,
+      invocation: {
+        url: config.url,
+        method: config.method ?? 'POST',
+        timeout_ms: config.timeout_ms,
+        headers: config.headers,
+        auth: config.auth,
+      },
+    }
+
+    this.sendMessage(MessageType.RegisterFunction, message, true)
+    this.httpFunctions.set(id, message)
+
+    return {
+      id,
+      unregister: () => {
+        this.sendMessage(MessageType.UnregisterFunction, { id }, true)
+        this.httpFunctions.delete(id)
+      },
+    }
+  }
+
   registerService = (message: Omit<RegisterServiceMessage, 'message_type'>): void => {
     this.sendMessage(MessageType.RegisterService, message, true)
     this.services.set(message.id, { ...message, message_type: MessageType.RegisterService })
@@ -606,6 +643,9 @@ class Sdk implements ISdk {
     this.functions.forEach(({ message }) => {
       this.sendMessage(MessageType.RegisterFunction, message, true)
     })
+    this.httpFunctions.forEach(message => {
+      this.sendMessage(MessageType.RegisterFunction, message, true)
+    })
     this.triggers.forEach(trigger => {
       this.sendMessage(MessageType.RegisterTrigger, trigger, true)
     })

packages/node/iii/src/index.ts

@@ -5,6 +5,7 @@ export { Logger } from './logger'
 export type { ISdk } from './types'
 
 export type { ApiRequest, HttpRequest, HttpResponse, ApiResponse, Channel } from './types'
+export type { HttpInvocationConfig, HttpAuthConfig } from './iii-types'
 export type { StreamChannelRef } from './iii-types'
 export { ChannelWriter, ChannelReader } from './channels'
 

packages/node/iii/src/types.ts

@@ -1,5 +1,6 @@
 import type {
   FunctionInfo,
+  HttpInvocationConfig,
   RegisterFunctionMessage,
   RegisterServiceMessage,
   RegisterTriggerMessage,
@@ -105,6 +106,14 @@ export interface ISdk {
    */
   registerFunction(func: RegisterFunctionInput, handler: RemoteFunctionHandler): FunctionRef
 
+  /**
+   * Registers an HTTP external function (Lambda, Cloudflare Workers, etc.). The engine invokes the URL when triggered.
+   * @param id - Function path (use :: for namespacing, e.g. external::my_lambda)
+   * @param config - HTTP endpoint config (url, method, timeout_ms, headers, auth)
+   * @returns A function ref for unregistering
+   */
+  registerHttpFunction(id: string, config: HttpInvocationConfig): FunctionRef
+
   /**
    * Invokes a function.
    * @param function_id - The path to the function

packages/node/iii/tests/fixtures/config-test.yaml

@@ -60,6 +60,15 @@ modules:
       adapter:
         class: modules::queue::BuiltinQueueAdapter
 
+  - class: modules::http_functions::HttpFunctionsModule
+    config:
+      security:
+        url_allowlist:
+          - localhost
+          - 127.0.0.1
+        block_private_ips: false
+        require_https: false
+
   - class: modules::cron::CronModule
     config:
       adapter: