Merge pull request #127 from firstbatchxyz/erhant/subcrates

Workspaces

Author: erhant

.env.example

@@ -10,13 +10,16 @@ DKN_ADMIN_PUBLIC_KEY=0208ef5e65a9c656a6f92fb2c770d5d5e2ecffe02a6aade19207f75110b
 DKN_MODELS=
 
 ## DRIA (optional) ##
-# P2P address, you don't need to change this unless you really want this port.
+# P2P address, you don't need to change this unless this port is already in use.
 DKN_P2P_LISTEN_ADDR=/ip4/0.0.0.0/tcp/4001
 # Comma-separated static relay nodes
 DKN_RELAY_NODES=
 # Comma-separated static bootstrap nodes
 DKN_BOOTSTRAP_NODES=
- 
+# Set to a number of seconds to wait before exiting, only use in profiling build!
+# Otherwise, leave this empty.
+DKN_EXIT_TIMEOUT=
+
 ## Open AI (if used, required) ##
 OPENAI_API_KEY=
 

.github/workflows/build_dev_container.yml

@@ -3,9 +3,17 @@ on:
   push:
     branches: ["master"]
     paths:
-      - "src/**"
+      # Source files in each member
+      - "compute/src/**"
+      - "p2p/src/**"
+      - "workflows/src/**"
+      # Cargo in each member
+      - "compute/Cargo.toml"
+      - "p2p/Cargo.toml"
+      - "workflows/Cargo.toml"
+      # root-level changes
       - "Cargo.lock"
-      - "Cargo.toml"
+      - "Cross.toml"
       - "Dockerfile"
       - "compose.yml"
 
@@ -44,7 +52,7 @@ jobs:
 
       - name: Set Image Tag
         id: itag
-        run: echo "itag=${{ steps.branch.outputs.branch }}-${{ steps.sha.outputs.sha }}-${{ steps.timestamp.outputs.timestamp }}" >> $GITHUB_OUTPUT
+        run: echo "itag=${{ steps.sha.outputs.branch }}-${{ steps.sha.outputs.sha }}-${{ steps.timestamp.outputs.timestamp }}" >> $GITHUB_OUTPUT
 
       - name: Build and push
         uses: docker/build-push-action@v6

.github/workflows/tests.yml

@@ -1,10 +1,10 @@
 name: tests
 
 on:
-  # push:
-  workflow_dispatch:
+  push:
     branches:
       - master
+  workflow_dispatch:
 
 jobs:
   test:
@@ -18,4 +18,4 @@ jobs:
         uses: actions-rust-lang/setup-rust-toolchain@v1
 
       - name: Run tests
-        run: cargo test
+        run: cargo test --workspace

Cargo.lock

@@ -1,78 +1,44 @@
-[package]
-name = "dkn-compute"
-version = "0.2.10"
+[workspace]
+resolver = "2"
+members = ["compute", "p2p", "workflows"]
+# compute node is the default member, until Oracle comes in
+# then, a Launcher will be the default member
+default-members = ["compute"]
+
+[workspace.package]
 edition = "2021"
+version = "0.2.11"
 license = "Apache-2.0"
 readme = "README.md"
-authors = ["Erhan Tezcan <[email protected]>"]
 
 # profiling build for flamegraphs
 [profile.profiling]
 inherits = "release"
 debug = true
 
-[features]
-# used by flamegraphs & instruments
-profiling = []
 
-[dependencies]
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[workspace.dependencies]
+# async stuff
 tokio-util = { version = "0.7.10", features = ["rt"] }
 tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
-parking_lot = "0.12.2"
+async-trait = "0.1.81"
+
+# serialize & deserialize
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-async-trait = "0.1.81"
+
+# http client
 reqwest = "0.12.5"
 
-# utilities
+# env reading
 dotenvy = "0.15.7"
-base64 = "0.22.0"
-hex = "0.4.3"
-hex-literal = "0.4.1"
-url = "2.5.0"
-urlencoding = "2.1.3"
-uuid = { version = "1.8.0", features = ["v4"] }
+
+# randomization
 rand = "0.8.5"
 
 # logging & errors
 env_logger = "0.11.3"
 log = "0.4.21"
 eyre = "0.6.12"
-
-# encryption (ecies) & signatures (ecdsa) & hashing & bloom-filters
-ecies = { version = "0.2", default-features = false, features = ["pure"] }
-libsecp256k1 = "0.7.1"
-sha2 = "0.10.8"
-sha3 = "0.10.8"
-fastbloom-rs = "0.5.9"
-
-# workflows
-ollama-workflows = { git = "https://github.com/andthattoo/ollama-workflows" }
-
-# peer-to-peer
-libp2p = { git = "https://github.com/anilaltuner/rust-libp2p.git", rev = "7ce9f9e", features = [
-    # libp2p = { version = "0.54.1", features = [
-    "dcutr",
-    "ping",
-    "relay",
-    "autonat",
-    "identify",
-    "tokio",
-    "gossipsub",
-    "mdns",
-    "noise",
-    "macros",
-    "tcp",
-    "yamux",
-    "quic",
-    "kad",
-] }
-libp2p-identity = { version = "0.2.9", features = ["secp256k1"] }
-tracing = { version = "0.1.40" }
-tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
-port_check = "0.2.1"
-
-# Vendor OpenSSL so that its easier to build cross-platform packages
-[dependencies.openssl]
-version = "*"
-features = ["vendored"]

Cargo.toml

@@ -7,15 +7,15 @@ endif
 ###############################################################################
 .PHONY: launch #       | Run with INFO logs in release mode
 launch:
-		RUST_LOG=none,dkn_compute=info cargo run --release
+		RUST_LOG=none,dkn_compute=info,dkn_workflows=info,dkn_p2p=info cargo run --release
 
 .PHONY: run #          | Run with INFO logs
 run:
-		RUST_LOG=none,dkn_compute=info cargo run
+		RUST_LOG=none,dkn_compute=info,dkn_workflows=info,dkn_p2p=info cargo run
 
 .PHONY: debug #        | Run with DEBUG logs with INFO log-level workflows
 debug:
-		RUST_LOG=warn,dkn_compute=debug,ollama_workflows=info cargo run
+		RUST_LOG=warn,dkn_compute=debug,dkn_workflows=debug,dkn_p2p=debug,ollama_workflows=info cargo run
 
 .PHONY: trace #        | Run with TRACE logs
 trace:
@@ -27,21 +27,21 @@ build:
 
 .PHONY: profile-cpu #  | Profile CPU usage with flamegraph
 profile-cpu:
-	  cargo flamegraph --root --profile=profiling --features=profiling
+	  DKN_EXIT_TIMEOUT=120 cargo flamegraph --root --profile=profiling
 
 .PHONY: profile-mem #  | Profile memory usage with instruments
 profile-mem:
-	  cargo instruments --profile=profiling --features=profiling -t Allocations
+	  DKN_EXIT_TIMEOUT=120 cargo instruments --profile=profiling -t Allocations
 
 ###############################################################################
 .PHONY: test #         | Run tests
 test:
-		cargo test
+		cargo test --workspace
 
 ###############################################################################
 .PHONY: lint #         | Run linter (clippy)
 lint:
-		cargo clippy
+		cargo clippy --workspace
 
 .PHONY: format #       | Run formatter (cargo fmt)
 format:

Makefile

@@ -36,7 +36,7 @@ Compute nodes can technically do any arbitrary task, from computing the square r
 
 - **Ping/Pong**: Dria Admin Node broadcasts **ping** messages at a set interval, it is a required duty of the compute node to respond with a **pong** to these so that they can be included in the list of available nodes for task assignment. These tasks will respect the type of model provided within the pong message, e.g. if a task requires `gpt-4o` and you are running `phi3`, you won't be selected for that task.
 
-- **Workflows**: Each task is given in the form of a workflow, based on [Ollama Workflows](https://github.com/andthattoo/ollama-workflows) (see repository for more information). In simple terms, each workflow defines the agentic behavior of an LLM, all captured in a single JSON file, and can represent things ranging from simple LLM generations to iterative web searching.
+- **Workflows**: Each task is given in the form of a workflow, based on [Ollama Workflows](https://github.com/andthattoo/ollama-workflows). In simple terms, each workflow defines the agentic behavior of an LLM, all captured in a single JSON file, and can represent things ranging from simple LLM generations to iterative web searching.
 
 ## Node Running
 

README.md

@@ -0,0 +1,50 @@
+[package]
+name = "dkn-compute"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+readme = "README.md"
+authors = ["Erhan Tezcan <[email protected]>"]
+
+[dependencies]
+tokio-util = { version = "0.7.10", features = ["rt"] }
+tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+async-trait = "0.1.81"
+reqwest = "0.12.5"
+
+# utilities
+dotenvy.workspace = true
+base64 = "0.22.0"
+hex = "0.4.3"
+hex-literal = "0.4.1"
+url = "2.5.0"
+urlencoding = "2.1.3"
+uuid = { version = "1.8.0", features = ["v4"] }
+
+port_check = "0.2.1"
+
+# logging & errors
+rand.workspace = true
+env_logger.workspace = true
+log.workspace = true
+eyre.workspace = true
+tracing = { version = "0.1.40" }
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+
+# encryption (ecies) & signatures (ecdsa) & hashing & bloom-filters
+ecies = { version = "0.2", default-features = false, features = ["pure"] }
+libsecp256k1 = "0.7.1"
+sha2 = "0.10.8"
+sha3 = "0.10.8"
+fastbloom-rs = "0.5.9"
+
+# dria subcrates
+dkn-p2p = { path = "../p2p" }
+dkn-workflows = { path = "../workflows" }
+
+# Vendor OpenSSL so that its easier to build cross-platform packages
+[dependencies.openssl]
+version = "*"
+features = ["vendored"]