diff --git a/.env.example b/.env.example index c6c061b..3c71219 100644 --- a/.env.example +++ b/.env.example @@ -1,17 +1,37 @@ -# Droq Node Template Environment Variables -# Copy this file to .env and update with your values +# Environment Variable Naming Convention: +# NODE_ - Node configuration +# NATS_ - NATS configuration +# LOG_ - Logging configuration +# DB_ - Database configuration +# HTTP_ - HTTP client configuration +# SERVICE_ - Service-specific configuration +# METRICS_ - Metrics and monitoring # Node Configuration NODE_NAME=droq-node-template +NODE_PORT=8000 +NODE_EXTERNAL_PORT=8000 +NODE_HEALTH_CHECK_ENABLED=true + +# Logging LOG_LEVEL=INFO # NATS Configuration NATS_URL=nats://localhost:4222 +NATS_CLIENT_NAME=${NODE_NAME:-droq-node-template} STREAM_NAME=droq-stream -# HTTP Client Configuration (optional) -# BASE_URL=https://api.example.com +# Optional: NATS Authentication +# NATS_USERNAME=nats-user +# NATS_PASSWORD=nats-pass + +# Optional: HTTP Configuration +# HTTP_BASE_URL=https://api.example.com +# HTTP_API_KEY=your-api-key + +# Optional: Database Configuration +# DB_URL=postgresql://user:pass@localhost/dbname -# Add your custom environment variables below -# API_KEY=your-api-key-here -# DATABASE_URL=postgresql://user:pass@localhost/dbname +# Optional: Metrics Configuration +# METRICS_PORT=8081 +# METRICS_ENABLED=true \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 89a5e24..37baae2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,49 +1,28 @@ -# Dockerfile template for Droq nodes -# This is an agnostic template - customize as needed for your node - FROM python:3.11-slim -# Set working directory WORKDIR /app -# Install system dependencies -# Uncomment and add system packages as needed: -# RUN apt-get update && apt-get install -y \ -# gcc \ -# g++ \ -# make \ -# curl \ -# && rm -rf /var/lib/apt/lists/* - -# Install uv COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv -# Copy dependency files COPY pyproject.toml uv.lock* ./ -# Install dependencies using uv -# Install dependencies directly (not as editable package) RUN uv pip install --system nats-py aiohttp || \ (uv pip compile pyproject.toml -o requirements.txt && \ uv pip install --system -r requirements.txt) -# Copy source code COPY src/ ./src/ -# Create non-root user for security RUN useradd -m -u 1000 nodeuser && chown -R nodeuser:nodeuser /app USER nodeuser -# Set environment variables ENV PYTHONPATH=/app ENV PYTHONUNBUFFERED=1 +ENV NODE_PORT=8000 + +EXPOSE ${NODE_PORT} -# Optional: Health check -# Uncomment and customize as needed: -# HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ -# CMD python -c "import sys; sys.exit(0)" +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD python -c "import sys; sys.exit(0)" || exit 1 -# Run the node -# Update this command to match your entry point -CMD ["uv", "run", "python", "-m", "node.main"] +CMD ["sh", "-c", "exec uv run python -m node.main --port=${NODE_PORT:-8000}"] diff --git a/README.md b/README.md index 596bc2d..31697cd 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,36 @@ # Droq Node Template -A Python template for building Droq nodes. Works with any Python code - minimal updates needed. +A Python template for building Droqflow nodes. ## Quick Start ```bash -# 1. Clone and setup -git clone -cd droq-node-template-py +git clone git@github.com:droq-ai/dfx-base-node-template-py.git +cd dfx-base-node-template-py uv sync -# 2. Replace src/node/main.py with your code +# Replace src/node/main.py with your code +# Add dependencies: uv add your-package -# 3. Add dependencies -uv add your-package +# Configure node.json with your node information +# Configure environment variables -# 4. Test locally +# Test locally PYTHONPATH=src uv run python -m node.main -# or -docker compose up -# 5. Build -docker build -t your-node:latest . +# Run with Docker +docker compose up ``` -## Documentation +## Next Steps -- [Usage Guide](docs/usage.md) - How to use the template -- [NATS Examples](docs/nats.md) - NATS publishing and consuming examples +1. Complete your node development +2. Configure [node.json](docs/node-configuration.md) with your node metadata +3. Register your node [TBD] -## Development -```bash -# Run tests -PYTHONPATH=src uv run pytest -# Format code -uv run black src/ tests/ -uv run ruff check src/ tests/ -# Add dependencies -uv add package-name -``` ## Docker @@ -50,32 +39,34 @@ uv add package-name docker build -t your-node:latest . # Run -docker run --rm your-node:latest - -# Development (with hot reload) -docker compose up +docker run -p 8000:8000 \ + -e NODE_NAME=my-node \ + -e NATS_URL=nats://localhost:4222 \ + your-node:latest ``` -## Environment Variables - -Copy `.env.example` to `.env` and update with your values: +## Development ```bash -cp .env.example .env +# Run tests +PYTHONPATH=src uv run pytest + +# Format code +uv run black src/ tests/ +uv run ruff check src/ tests/ + +# Add dependencies +uv add package-name ``` -Or set in `compose.yml` or pass to Docker: -- `NATS_URL` - NATS server URL (default: `nats://localhost:4222`) -- `STREAM_NAME` - JetStream name (default: `droq-stream`) -- `NODE_NAME` - Node identifier -- `LOG_LEVEL` - Logging level -## Next Steps -1. Test locally -2. Build Docker image -3. Register metadata in `droq-node-registry` (separate repo) +## Documentation + +- [Usage Guide](docs/usage.md) +- [Configuration Guide](docs/node-configuration.md) +- [NATS Examples](docs/nats.md) ## License -Apache License 2.0 +Apache License 2.0 \ No newline at end of file diff --git a/compose.yml b/compose.yml index 69ed348..03e7617 100644 --- a/compose.yml +++ b/compose.yml @@ -6,33 +6,25 @@ services: container_name: droq-node-template environment: - NODE_NAME=droq-node-template + - NODE_PORT=8000 - LOG_LEVEL=INFO - NATS_URL=nats://nats:4222 + - NATS_CLIENT_NAME=${NODE_NAME:-droq-node-template} - STREAM_NAME=droq-stream - # Add your environment variables here - # - API_KEY=${API_KEY} - # - DATABASE_URL=${DATABASE_URL} - # - BASE_URL=https://api.example.com volumes: - # Mount source code for development (hot reload) - ./src:/app/src:ro - # Mount additional volumes as needed - # - ./data:/app/data networks: - droq-network restart: unless-stopped - # Uncomment to expose ports if your node needs them - # ports: - # - "8080:8080" + ports: + - "${NODE_EXTERNAL_PORT:-8000}:${NODE_PORT:-8000}" - # NATS server for local development and testing nats: image: nats:latest container_name: droq-nats ports: - - "4222:4222" # Client connections - - "8222:8222" # HTTP monitoring - - "6222:6222" # Cluster routing + - "4222:4222" + - "8222:8222" command: ["-js", "-m", "8222"] networks: - droq-network diff --git a/docs/node-configuration.md b/docs/node-configuration.md new file mode 100644 index 0000000..08805c8 --- /dev/null +++ b/docs/node-configuration.md @@ -0,0 +1,320 @@ +# Node Configuration Guide + +This guide explains how to configure `node.json` for your Droq nodes. All fields are required and serve specific purposes in the Droq ecosystem. + +## Overview + +`node.json` is a metadata file that describes your node's identity, capabilities, and deployment configuration. It's used for: + +- Node registry registration +- Orchestration and service discovery +- Runtime metadata management +- Component discovery and routing + +## Required Fields + +### Identity Fields + +#### `version` (String) +**Purpose**: Semantic version of your node +**Format**: MAJOR.MINOR.PATCH (e.g., "1.0.0") +**Example**: +```json +"version": "1.2.3" +``` + +#### `node_id` (String) +**Purpose**: Unique identifier matching your repository name +**Format**: Repository name without organization prefix +**Examples**: +```json +"node_id": "my-custom-node" +"node_id": "data-processor-service" +"node_id": "math-executor-node" +``` + +#### `name` (String) +**Purpose**: Human-readable display name +**Example**: +```json +"name": "My Data Processing Node" +``` + +#### `description` (String) +**Purpose**: Brief description of what your node does +**Example**: +```json +"description": "Processes incoming data streams and applies transformations" +``` + +### Runtime Fields (Dynamic) + +#### `api_url` (String) +**Purpose**: External API endpoint for your node +**Integration**: Should align with `NODE_PORT` environment variable +**Example**: +```json +"api_url": "http://localhost:8000" +``` + +#### `ip_address` (String) +**Purpose**: Network location where node is deployed +**Example**: +```json +"ip_address": "127.0.0.1" +``` + +#### `docker_image` (String) +**Purpose**: Container image reference for deployment +**Format**: `{registry}/{image}:{tag}` +**Examples**: +```json +"docker_image": "myorg/my-node:latest" +"docker_image": "my-node:1.2.3" +``` + +#### `deployment_location` (String) +**Purpose**: Geographic or logical deployment location +**Examples**: +```json +"deployment_location": "local" +"deployment_location": "on-premise" +``` + +#### `status` (String) +**Purpose**: Current operational status of the node +**Values**: `"active"`, `"inactive"`, `"maintenance"`, `"error"` +**Example**: +```json +"status": "active" +``` + +### Metadata Fields + +#### `author` (String) +**Purpose**: Creator or organization name +**Example**: +```json +"author": "Droq Team" +"author": "My Company" +"author": "John Doe" +``` + +#### `created_at` (String) +**Purpose**: Node creation timestamp +**Format**: ISO 8601 (YYYY-MM-DDTHH:MM:SSZ) +**Example**: +```json +"created_at": "2025-01-15T10:30:00Z" +``` + +#### `source_code_location` (String) +**Purpose**: Repository URL for source code +**Example**: +```json +"source_code_location": "https://github.com/myorg/my-node" +``` + +## Component Configuration + +Components define the functional capabilities of your node. Each component has a unique path and metadata. + +### Component Path Structure + + + +### Component Structure +```json +"components": { + "ComponentName": { + "path": "dfx.base-node-template-py.core.main", + "description": "What this component does", + "author": "Component author (optional)" + } +} +``` + +### Examples + +**For node_id: "base-node-template-py"**: +```json +"components": { + "MainComponent": { + "path": "myapp.src.core.main", + "description": "Main entry point for the node", + "author": "Droq Team" + }, + "DataProcessor": { + "path": "myapp.src.processing.transform", + "description": "Transforms incoming data", + "author": "Droq Team" + } +} +``` + +## Configuration Values + +Set your configuration values directly in `node.json`: + +```json +{ + "api_url": "http://localhost:8000", + "ip_address": "127.0.0.1", + "docker_image": "my-node:latest", + "deployment_location": "local" +} +``` + +## Complete Examples + +### Example 1: Simple Computational Node + +```json +{ + "version": "1.0.0", + "node_id": "math-calculator", + "name": "Math Calculator Node", + "description": "Performs mathematical operations on incoming data", + "author": "Data Team", + "api_url": "http://localhost:8000", + "created_at": "2025-01-15T10:30:00Z", + "ip_address": "127.0.0.1", + "status": "active", + "docker_image": "myorg/math-calculator:1.0.0", + "deployment_location": "local", + "source_code_location": "https://github.com/myorg/math-calculator", + "components": { + "Calculator": { + "path": "dfx.math.calculator.core.main", + "description": "Main calculator component", + "author": "Data Team" + } + } +} +``` + +### Example 2: Multi-Component Node + +```json +{ + "version": "1.0.0", + "node_id": "data-service", + "name": "Data Service Node", + "description": "Provides data processing and validation services", + "author": "My Team", + "api_url": "http://localhost:8000", + "created_at": "2025-01-15T10:30:00Z", + "ip_address": "127.0.0.1", + "status": "active", + "docker_image": "myorg/data-service:latest", + "deployment_location": "local", + "source_code_location": "https://github.com/myorg/data-service", + "components": { + "Processor": { + "path": "dfx.data.service.processing.transform", + "description": "Transforms incoming data", + "author": "My Team" + }, + "Validator": { + "path": "dfx.data.service.utils.validate", + "description": "Validates data format", + "author": "My Team" + } + } +} +``` + + +## Integration with Docker and Environment Variables + +### Port Alignment + +Ensure `api_url` matches your `NODE_PORT` environment variable: + +```bash +# .env file +NODE_PORT=8080 + +# node.json file +"api_url": "http://localhost:8080" +``` + +### Docker Image Reference + +The `docker_image` field should match your Docker publishing configuration: + +```json +"docker_image": "myorg/my-node:latest" +``` + +Corresponds to: +```yaml +# GitHub Actions workflow +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} # myorg/my-node +``` + +### Environment Variable Integration + +Common environment variables that relate to node.json fields: + +```bash +# Node identification +NODE_NAME=my-custom-node + +# Network configuration +NODE_PORT=8000 +API_URL=http://localhost:8000 + +# Docker configuration +DOCKER_IMAGE=my-custom-node:latest +DEPLOYMENT_LOCATION=local +``` + +## Validation Checklist + +Before deploying, verify your `node.json`: + +- [ ] All required fields are present +- [ ] `version` follows semantic versioning (X.Y.Z) +- [ ] `node_id` matches repository name +- [ ] `api_url` is valid URL format +- [ ] Component paths follow `dfx.{node_id}.{category}.{component}` format +- [ ] `created_at` is valid ISO 8601 timestamp +- [ ] `status` is one of: "active", "inactive", "maintenance", "error" +- [ ] Component paths match your actual Python module structure + +### JSON Validation + +You can validate your JSON using command line: + +```bash +# Using python +python -m json.tool node.json + +# Using jq +jq . node.json > /dev/null && echo "Valid JSON" || echo "Invalid JSON" +``` + +## Best Practices + +1. **Naming Conventions** + - Use kebab-case for `node_id` + - Use descriptive names for components + - Follow semantic versioning + +2. **Component Organization** + - Group related components by category + - Use clear, descriptive component names + - Document component responsibilities + +3. **Version Management** + - Update `version` when making changes + - Tag releases consistently + - Keep version in sync with Docker tags + +4. **Configuration** + - Validate JSON format before deployment + - Keep configuration in version control + - Test configuration changes before deployment \ No newline at end of file diff --git a/docs/node-registration.md b/docs/node-registration.md new file mode 100644 index 0000000..4d8fb56 --- /dev/null +++ b/docs/node-registration.md @@ -0,0 +1,325 @@ +# Node Registration Guide + +This guide explains how to register your Droq node in the official droq-node-registry so it can be discovered by other users on [directory.droq.ai](https://directory.droq.ai). + +## Overview + +The droq-node-registry uses Git submodules to organize and manage registered nodes. Each node is added as a submodule under the `/nodes` directory, making it easy to maintain version control and updates. + +## Prerequisites + +Before registering your node, ensure you have: + +### Development Requirements +- [x] Node development completed and tested +- [x] Docker image built and published to a registry +- [x] `node.json` file properly configured and validated +- [x] Source code pushed to a public GitHub repository + +### node.json Requirements +- All required fields completed +- Correct semantic version (e.g., "1.0.0") +- Accurate `node_id` matching your repository name +- Valid `api_url` and `docker_image` references +- Proper timestamp in `created_at` field +- Complete component definitions + +### Git Requirements +- GitHub account with access to your node repository +- Git installed locally +- Fork permissions on droq-node-registry repository + +## Registration Process + +### Step 1: Fork the Registry Repository + +1. Visit [https://github.com/droq-ai/droq-node-registry](https://github.com/droq-ai/droq-node-registry) +2. Click "Fork" to create your personal copy +3. Note your fork URL: `https://github.com/YOUR_USERNAME/droq-node-registry` + +### Step 2: Clone Your Fork + +```bash +# Clone your fork +git clone https://github.com/YOUR_USERNAME/droq-node-registry.git +cd droq-node-registry + +# Add upstream remote (optional, for keeping updated) +git remote add upstream https://github.com/droq-ai/droq-node-registry.git +``` + +### Step 3: Add Your Node as Submodule + +Navigate to the nodes directory and add your node as a submodule: + +```bash +cd nodes + +# Add your node repository as submodule +# Replace {your-node-id} with your actual node_id from node.json +git submodule add https://github.com/YOUR_USERNAME/YOUR_NODE_REPOSITORY.git {your-node-id} + +# Example: +# git submodule add https://github.com/johndoe/my-data-processor.git my-data-processor +``` + +### Step 4: Verify Submodule Configuration + +Check that the submodule was added correctly: + +```bash +# List submodules +git submodule status + +# Should show your node with commit hash +# a1b2c3d4 nodes/{your-node-id} (heads/main) +``` + +### Step 5: Commit and Push Changes + +```bash +# Go back to repository root +cd .. + +# Stage and commit the submodule addition +git add nodes/{your-node-id} .gitmodules +git commit -m "Add {node-name} node" + +# Push to your fork +git push origin main +``` + +### Step 6: Create Pull Request + +1. Visit your fork on GitHub +2. Click "Contribute" → "Open pull request" +3. Target branch: `main` (in droq-ai/droq-node-registry) +4. Fill in the pull request details (see PR Requirements below) + +## Pull Request Requirements + +### PR Title Format +``` +Add {node-name} node +``` + +### PR Description Template + +```markdown +## Node Information +- **Node ID**: {your-node-id} +- **Node Name**: {node-name from node.json} +- **Description**: {brief description from node.json} +- **Version**: {version from node.json} + +## Docker Image +- **Registry**: {docker registry (e.g., ghcr.io, docker.io)} +- **Image**: {full image name with tag} +- **Source**: Link to node repository + +## Components +{List main components from node.json} + +## Testing +- [ ] Node builds and runs successfully +- [ ] Docker image is accessible +- [ ] node.json is valid and complete +- [ ] All components are properly defined + +## Additional Notes +{Any additional information about the node} +``` + +## Submodule Management + +### Using Specific Version/Tag + +To register a specific version of your node instead of the latest commit: + +```bash +# Navigate to the submodule directory +cd nodes/{your-node-id} + +# Checkout specific tag or commit +git checkout v1.0.0 # or specific commit hash + +# Go back to registry root and commit the version lock +cd ../.. +git add nodes/{your-node-id} +git commit -m "Update {node-name} to v1.0.0" +``` + +### Updating Existing Node Registration + +When you release a new version: + +```bash +# Navigate to the submodule +cd nodes/{your-node-id} + +# Pull latest changes or checkout new tag +git pull origin main +# OR: git checkout v2.0.0 + +# Go back to registry root +cd .. + +# Stage and commit the update +git add nodes/{your-node-id} +git commit -m "Update {node-name} to {new-version}" +git push origin main + +# Create PR for the update +``` + +### Removing a Node (for maintenance) + +```bash +# Remove submodule +git submodule deinit -f nodes/{your-node-id} +git rm -f nodes/{your-node-id} +rm -rf .git/modules/nodes/{your-node-id} + +# Commit the removal +git commit -m "Remove {node-name} node" +``` + +## Review Process + +### Automated Checks +- Submodule points to valid repository +- node.json file exists and is valid JSON +- Basic field validation in node.json + +### Manual Review +The Droq team will review: +- Node functionality and purpose +- Code quality and documentation +- Security considerations +- Integration with Droq ecosystem + +### Approval and Merge +- Once approved, PR will be merged into main registry +- Your node will be automatically discovered and indexed +- Node appears on [directory.droq.ai](https://directory.droq.ai) after processing + +## [directory.droq.ai](https://directory.droq.ai) Integration + +After your node is registered and merged: + +### Discovery Process +- Registry is periodically scanned for new nodes +- Each node's `node.json` is parsed and indexed +- Components are extracted and categorized +- Docker images are validated for accessibility + +### What Users See +- **Node Name**: From `name` field in node.json +- **Description**: From `description` field +- **Author**: From `author` field +- **Version**: From `version` field +- **Components**: List of available components +- **Repository**: Link to source code +- **Docker Image**: Image reference for deployment + +### Updates and Maintenance +- Node information updates when registry submodule is updated +- Version changes are reflected automatically +- Component additions/updates appear after next scan + +## Troubleshooting + +### Common Issues + +**Submodule shows as empty:** +```bash +# Initialize and update submodules +git submodule update --init --recursive +``` + +**Wrong commit/branch in submodule:** +```bash +# Navigate to submodule and fix +cd nodes/{your-node-id} +git checkout main # or specific tag +cd ../.. +git add nodes/{your-node-id} +git commit -m "Fix submodule reference" +``` + +**PR not showing submodule changes:** +- Ensure `.gitmodules` file is staged +- Verify the submodule directory is tracked +- Check that changes are committed, not just staged + +**node.json validation errors:** +- Validate JSON format: `python -m json.tool node.json` +- Check all required fields are present +- Verify URL formats and timestamp format + +**Docker image not accessible:** +- Verify image exists in registry +- Check repository permissions (public vs private) +- Test image pull: `docker pull your-image:tag` + +### Getting Help + +- **Registry Issues**: Open issue in droq-node-registry repository +- **Node Development**: Refer to the node template documentation +- **Directory Listings**: Contact Droq team through official channels + +## Best Practices + +### Before Registration +- Test your node thoroughly +- Validate node.json format +- Ensure Docker image is accessible +- Document components clearly +- Use semantic versioning + +### Submodule Management +- Use specific tags for releases instead of latest commits +- Keep submodule references up to date +- Update node.json when changing versions +- Document breaking changes in releases + +### PR Process +- Provide clear, detailed PR descriptions +- Include testing status +- Respond to review feedback promptly +- Follow established naming conventions + +### Maintenance +- Update registration when releasing new versions +- Monitor for issues or questions on your repository +- Keep documentation current +- Participate in Droq community discussions + +## Example: Complete Registration Flow + +Here's a complete example for registering a node called "my-processor": + +```bash +# 1. Fork and clone registry +git clone https://github.com/johndoe/droq-node-registry.git +cd droq-node-registry + +# 2. Add node as submodule +cd nodes +git submodule add https://github.com/johndoe/my-processor.git my-processor + +# 3. Lock to specific version +cd my-processor +git checkout v1.0.0 +cd ../.. + +# 4. Commit changes +git add nodes/my-processor .gitmodules +git commit -m "Add My Processor node" +git push origin main + +# 5. Create PR on GitHub +# (Visit your fork and open pull request) +``` + +Your node will now be reviewed and, once approved, will appear on [directory.droq.ai](https://directory.droq.ai) for other users to discover and use. \ No newline at end of file diff --git a/node.json b/node.json new file mode 100644 index 0000000..74b803b --- /dev/null +++ b/node.json @@ -0,0 +1,21 @@ +{ + "version": "0.1.0", + "node_id": "dfx-base-node-template-py", + "name": "Droq Base Node Template", + "description": "A Python boilerplate template for creating new DFX nodes.", + "author": "Droq Team", + "api_url": "http://localhost:8003", + "created_at": "2025-11-17T18:44:00Z", + "ip_address": "127.0.0.1", + "status": "active", + "docker_image": "droqai/dfx-base-node-template:latest", + "deployment_location": "local", + "source_code_location": "https://github.com/droq-ai/dfx-base-node-template-py", + "components": { + "DFXExampleComponent": { + "path": "dfx.components.example", + "description": "This is a sample droq component", + "author": "Droq Team" + } + } +} \ No newline at end of file