Add event-based offchain indexer with Runtime API and RPC for pallet-robonomics-cps

Cargo.toml

@@ -25,6 +25,8 @@ members = [
   "frame/digital-twin",
   "frame/xcm-info",
   "frame/cps",
+  "frame/cps/rpc",
+  "frame/cps/rpc/runtime-api",
   "frame/wrapped-asset",
   "runtime/robonomics",
   "tools/libcps",
@@ -245,6 +247,11 @@ pallet-robonomics-liability = { path = "./frame/liability", default-features = f
 pallet-robonomics-rws = { path = "./frame/rws", default-features = false }
 pallet-robonomics-cps = { path = "./frame/cps", default-features = false }
 pallet-wrapped-asset = { path = "./frame/wrapped-asset", default-features = false }
+pallet-xcm-info = { path = "./frame/xcm-info", default-features = false }
+
+# CPS indexer RPC
+pallet-robonomics-cps-rpc = { path = "./frame/cps/rpc" }
+pallet-robonomics-cps-rpc-runtime-api = { path = "./frame/cps/rpc/runtime-api", default-features = false }
 
 # Robonomics runtimes
 robonomics-runtime = { path = "./runtime/robonomics" }

bin/robonomics/Cargo.toml

@@ -23,3 +23,7 @@ assert_cmd.workspace = true
 
 [build-dependencies]
 substrate-build-script-utils.workspace = true
+
+[features]
+default = []
+cps-offchain-indexer = ["robonomics-runtime/cps-offchain-indexer"]

frame/cps/Cargo.toml

@@ -9,33 +9,46 @@ repository.workspace = true
 license = "Apache-2.0"
 
 [dependencies]
-scale-info = { workspace = true }
-parity-scale-codec = { workspace = true }
-sp-runtime = { workspace = true }
-sp-std = { workspace = true }
-sp-core = { workspace = true }
-frame-system = { workspace = true }
-frame-support = { workspace = true }
-frame-benchmarking = { workspace = true, optional = true }
+scale-info.workspace = true
+parity-scale-codec.workspace = true
+sp-runtime.workspace = true
+sp-std.workspace = true
+sp-core.workspace = true
+sp-io.workspace = true
+sp-tracing.workspace = true
+serde.workspace = true
+serde.optional = true
+frame-system.workspace = true
+frame-support.workspace = true
+frame-benchmarking.workspace = true
+frame-benchmarking.optional = true
 
 [dev-dependencies]
-sp-io = { workspace = true }
-pallet-balances = { workspace = true, features = ["std"] }
-pallet-proxy = { workspace = true, features = ["std"] }
+sp-io.workspace = true
+pallet-balances.workspace = true
+pallet-balances.features = ["std"]
+pallet-proxy.workspace = true
+pallet-proxy.features = ["std"]
+serde_json.workspace = true
 
 [features]
 default = ["std"]
 std = [
   "parity-scale-codec/std",
   "sp-runtime/std",
+  "sp-tracing/std",
   "sp-std/std",
   "sp-core/std",
+  "sp-io/std",
+  "serde?/std",
   "frame-system/std",
   "frame-support/std",
   "frame-benchmarking?/std",
   "scale-info/std",
 ]
 
+offchain-indexer = ["serde"]
+
 runtime-benchmarks = [
   "frame-benchmarking",
   "frame-support/runtime-benchmarks",

frame/cps/README.md

@@ -988,20 +988,193 @@ await api.tx.cps.moveNode(nodeId, newParentId).signAndSend(account);
 | **IPFS + DB** | Decentralized storage | No ownership enforcement | Content distribution |
 | **ERC-721 NFTs** | Standard, composable | Gas-expensive, limited structure | Digital collectibles |
 
+## Offchain Worker Indexer (Optional Feature)
+
+The CPS pallet includes an efficient offchain worker that collects historical data using an **event-based indexing pattern** and exposes it via RPC API.
+
+### Architecture
+
+**Event-Based Indexing Pattern:**
+
+The indexer uses a highly efficient single-write-per-block pattern:
+
+1. **During Block Execution**: As extrinsics execute, CPS events (node creation, updates, deletions) are tracked in temporary storage (`ModifiedNodesThisBlock`)
+2. **In `on_finalize` Hook**: All collected events are written **once** to offchain storage as a single event queue using `sp_io::offchain_index::set()`
+3. **Offchain Worker**: Reads the event queue and processes each event, indexing data into queryable storage structures
+
+**Benefits:**
+- ✅ **Single write per block** instead of multiple writes during execution
+- ✅ **Automatic** - no manual integration needed
+- ✅ **Efficient** - minimal overhead during block execution  
+- ✅ **Clean separation** - event collection happens in runtime, processing in offchain worker
+
+### Enabling the Feature
+
+Add the `offchain-indexer` feature flag to enable indexing:
+
+```toml
+pallet-robonomics-cps = { 
+    default-features = false, 
+    path = "../frame/cps",
+    features = ["offchain-indexer"]
+}
+```
+
+### Storage Structure
+
+The indexer uses a double-map storage structure with **node_id first**, then block number:
+
+1. **Meta Records**: `cps::meta::<node_id><block_number>` - Metadata updates
+2. **Payload Records**: `cps::payload::<node_id><block_number>` - Payload updates
+3. **Node Operations**: `cps::operations::<node_id><block_number>` - Node lifecycle events
+4. **Node Index**: `cps::node_index::<node_id>` - Tracks which nodes have indexed data
+5. **Event Queue**: `cps::event_queue::<block_number>` - Pending events for processing
+
+All records include:
+- Block number (as timestamp, u64)
+- Node ID for efficient filtering
+- Associated data (Vec<u8>, encoded)
+- Operation type (for node operations: Create/Move/Delete)
+
+### Runtime API Integration
+
+Implement the Runtime API in your runtime:
+
+```rust
+impl pallet_robonomics_cps_rpc_runtime_api::CpsIndexerApi<Block> for Runtime {
+    fn get_meta_records(
+        node_id: Option<NodeId>,
+        from: Option<u64>,
+        to: Option<u64>
+    ) -> Vec<MetaRecord> {
+        pallet_robonomics_cps::offchain::storage::get_meta_records(node_id, from, to)
+    }
+
+    fn get_payload_records(
+        node_id: Option<NodeId>,
+        from: Option<u64>,
+        to: Option<u64>
+    ) -> Vec<PayloadRecord> {
+        pallet_robonomics_cps::offchain::storage::get_payload_records(node_id, from, to)
+    }
+
+    fn get_node_operations(
+        node_id: Option<NodeId>,
+        from: Option<u64>,
+        to: Option<u64>
+    ) -> Vec<NodeOperation> {
+        pallet_robonomics_cps::offchain::storage::get_node_operations(node_id, from, to)
+    }
+}
+```
+
+Note: **No manual index_* calls needed!** Events are automatically tracked and indexed.
+
+### RPC Extension Setup
+
+Add the RPC extension to your node's RPC handler:
+
+```rust
+use pallet_robonomics_cps_rpc::{CpsIndexerRpc, CpsIndexerRpcApiServer};
+
+// In your RPC setup
+let cps_rpc = CpsIndexerRpc::new(client.clone());
+io.merge(cps_rpc.into_rpc())?;
+```
+
+### RPC Endpoints
+
+Query historical data via JSON-RPC:
+
+**Get meta records:**
+```bash
+curl -H "Content-Type: application/json" \
+     -d '{"id":1, "jsonrpc":"2.0", "method":"cps_getMetaRecords", "params":[1704067200, 1704153600]}' \
+     http://localhost:9933
+```
+
+**Get payload records:**
+```bash
+curl -H "Content-Type: application/json" \
+     -d '{"id":1, "jsonrpc":"2.0", "method":"cps_getPayloadRecords", "params":[1704067200, 1704153600]}' \
+     http://localhost:9933
+```
+
+**Get node operations:**
+```bash
+curl -H "Content-Type: application/json" \
+     -d '{"id":1, "jsonrpc":"2.0", "method":"cps_getNodeOperations", "params":[1704067200, 1704153600]}' \
+     http://localhost:9933
+```
+
+### Response Format
+
+All RPC methods return arrays of timestamped records:
+
+```json
+{
+  "jsonrpc": "2.0",
+  "result": [
+    {
+      "timestamp": 1704067200,
+      "data": "0x48656c6c6f20576f726c64"
+    },
+    {
+      "timestamp": 1704070800,
+      "data": "0x54657374204461746131"
+    }
+  ],
+  "id": 1
+}
+```
+
+Node operations additionally include an `operationType` field:
+
+```json
+{
+  "timestamp": 1704067200,
+  "operationType": "create",
+  "data": "0x..."
+}
+```
+
+### Benefits
+
+- **Historical Analysis**: Query past system states and changes
+- **Audit Trails**: Track when nodes were created/modified
+- **Monitoring**: Build dashboards showing system activity over time
+- **Debugging**: Investigate issues by reviewing historical events
+- **Compliance**: Maintain queryable records for regulatory requirements
+
+### Performance Considerations
+
+- Offchain storage is node-local (not replicated across the network)
+- Storage size grows with activity; implement pruning if needed
+- Queries are bounded by time range to prevent excessive iteration
+- RPC calls are non-blocking and don't affect chain performance
+- **Important**: Current implementation iterates through every timestamp in range. For production use with large time ranges, add bounds checking in the Runtime API implementation to prevent potential DoS via excessive queries.
+
+### Security Considerations
+
+- **Public RPC Endpoints**: The RPC endpoints are publicly accessible. Consider implementing rate limiting at the node level for production deployments.
+- **No Data Encryption**: Offchain storage is not encrypted, but this is acceptable since indexed data comes from public on-chain events.
+- **No Consensus Impact**: Offchain worker operations are isolated and don't affect chain consensus.
+- **Local Storage**: Data is stored locally per node, not broadcast to the network.
+
 ## Roadmap
 
 **Current (v1):**
 - ✅ Hierarchical tree with cycle prevention
 - ✅ Plain and encrypted data storage
 - ✅ O(1) operations via path storage
 - ✅ Compact encoding for efficiency
+- ✅ Offchain worker indexer with RPC API (optional feature)
 
 **Planned (v2):**
 - 🔮 Multi-owner nodes with role-based permissions
 - 🔮 Node templates for rapid deployment
 - 🔮 Batch operations for bulk updates
 - 🔮 Additional encryption algorithms (AES-GCM, ChaCha20)
-- 🔮 Off-chain worker integration for automated maintenance
 
 ## Technical Documentation
 

frame/cps/rpc/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "pallet-robonomics-cps-rpc"
+description = "RPC extension for Robonomics CPS Indexer"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license = "Apache-2.0"
+
+[dependencies]
+jsonrpsee = { workspace = true, features = ["client-core", "server", "macros"] }
+serde = { workspace = true, features = ["derive"] }
+hex = { workspace = true }
+parity-scale-codec = { workspace = true }
+sp-api = { workspace = true }
+sp-blockchain = { workspace = true }
+sp-runtime = { workspace = true }
+pallet-robonomics-cps = { path = "..", default-features = false, features = [
+  "offchain-indexer",
+] }
+pallet-robonomics-cps-rpc-runtime-api = { path = "./runtime-api", default-features = false }
+
+[features]
+default = ["std"]
+std = [
+  "parity-scale-codec/std",
+  "sp-api/std",
+  "sp-runtime/std",
+  "pallet-robonomics-cps/std",
+  "pallet-robonomics-cps-rpc-runtime-api/std",
+]
+offchain-indexer = ["pallet-robonomics-cps/offchain-indexer"]

frame/cps/rpc/runtime-api/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "pallet-robonomics-cps-rpc-runtime-api"
+description = "Runtime API for Robonomics CPS Indexer"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license = "Apache-2.0"
+
+[dependencies]
+parity-scale-codec = { workspace = true }
+sp-api = { workspace = true }
+sp-std = { workspace = true }
+pallet-robonomics-cps = { path = "../..", default-features = false }
+
+[features]
+default = ["std"]
+std = [
+  "parity-scale-codec/std",
+  "sp-api/std",
+  "sp-std/std",
+  "pallet-robonomics-cps/std",
+]