Merge pull request #46 from bcgov/jdev

chore(sandbox): simplify the local dev workspace setup

Author: junminahn

docker-compose/docker-compose.yml

@@ -32,10 +32,10 @@ x-n8n-env: &n8n-env
   N8N_DISABLED_MODULES: "external-secrets,mcp,chat-hub"
   N8N_ENCRYPTION_KEY: ${N8N_ENCRYPTION_KEY}
   N8N_USER_MANAGEMENT_JWT_SECRET: ${N8N_USER_MANAGEMENT_JWT_SECRET}
-  N8N_CUSTOM_EXTENSIONS: /home/node/.n8n/nodes
   OLLAMA_HOST: ${OLLAMA_HOST:-ollama:11434}
   N8N_PORT: '5678'
   NODE_ENV: production
+  N8N_PROTOCOL: 'http'
   N8N_TRUST_PROXY: 'true'
   N8N_PROXY_HOPS: "1"
   # OIDC
@@ -47,8 +47,6 @@ x-n8n-env: &n8n-env
   OIDC_USERINFO_ENDPOINT: http://keycloak:8080/realms/starter/protocol/openid-connect/userinfo
   OIDC_REDIRECT_URI: http://localhost:5678/rest/auth/oidc/callback
   OIDC_ROLES_CLAIM: client_roles
-  EXTERNAL_HOOK_FILES: /home/node/.n8n/external-hooks/oidc.cjs:/home/node/.n8n/external-hooks/test.cjs:/home/node/.n8n/external-hooks/custom-api.cjs
-  EXTERNAL_FRONTEND_HOOKS_URLS: /assets/oidc-frontend-hook.js
   N8N_ADDITIONAL_NON_UI_ROUTES: auth
 
 services:
@@ -123,27 +121,33 @@ services:
       keycloak:
         condition: service_started
 
-  n8n-import:
+  n8n-provision:
     <<: *common-props
-    container_name: n8n-import
+    container_name: n8n-provision
     restart: 'no'
     build:
-      context: .
-      dockerfile: dockerfiles/n8n.Dockerfile
+      context: ..
+      dockerfile: Dockerfile
     environment: *n8n-env
     volumes:
       - n8n_storage:/home/node/.n8n
       - ./n8n/demo-data:/demo-data
-      - ../community-nodes:/community-nodes
     entrypoint: /bin/sh
     command: >
       -c "if [ -z \"$$(n8n list:workflow --onlyId)\" ]; then
             n8n import:credentials --separate --input=/demo-data/credentials &&
             n8n import:workflow --separate --input=/demo-data/workflows;
           fi;
 
-          cd "$N8N_CUSTOM_EXTENSIONS" &&
-          npm install /community-nodes"
+          echo 'Installing community nodes';
+          mkdir -p /home/node/.n8n/nodes;
+          cd /home/node/.n8n/nodes && npm install /community-nodes;
+
+          echo 'Installing external hooks';
+          cd /external-hooks && node migrate.cjs;
+
+          echo 'n8n provision completed!';
+          "
     depends_on:
       postgres:
         condition: service_healthy
@@ -152,54 +156,17 @@ services:
     <<: *common-props
     container_name: n8n
     build:
-      context: .
-      dockerfile: dockerfiles/n8n.Dockerfile
+      context: ..
+      dockerfile: Dockerfile
     environment: *n8n-env
     ports:
       - '5678:5678'
     volumes:
       - n8n_storage:/home/node/.n8n
-      - ./n8n/demo-data:/demo-data
-      - ../community-nodes:/community-nodes
-      - ../external-hooks/dist/oidc.cjs:/home/node/.n8n/external-hooks/oidc.cjs
-      - ../external-hooks/dist/test.cjs:/home/node/.n8n/external-hooks/test.cjs
-      - ../external-hooks/dist/custom-api.cjs:/home/node/.n8n/external-hooks/custom-api.cjs
-      - ../external-hooks/drizzle:/home/node/.n8n/external-hooks/drizzle
     depends_on:
       postgres:
         condition: service_healthy
       keycloak:
         condition: service_started
-      n8n-import:
+      n8n-provision:
         condition: service_completed_successfully
-
-  qdrant:
-    <<: *common-props
-    container_name: qdrant
-    image: qdrant/qdrant:latest
-    ports:
-      - '6333:6333'
-    volumes:
-      - qdrant_storage:/qdrant/storage
-
-  # OLLAMA CONFIGURATIONS (Profiles)
-  # ollama:
-  #   <<: *common-props
-  #   container_name: ollama
-  #   image: ollama/ollama:latest
-  #   volumes:
-  #     - ollama_storage:/root/.ollama
-  #   ports:
-  #     - '11434:11434'
-
-  # ollama-pull-llama:
-  #   <<: *common-props
-  #   container_name: ollama-pull-llama
-  #   restart: 'no'
-  #   image: ollama/ollama:latest
-  #   environment:
-  #     OLLAMA_HOST: ollama:11434
-  #   entrypoint: /bin/sh
-  #   command: -c "sleep 5; ollama pull llama3.2"
-  #   depends_on:
-  #     - ollama

docker-compose/dockerfiles/n8n.Dockerfile

@@ -28,60 +28,47 @@ If you are using the Docker CLI without the Desktop GUI, ensure your environment
 <br> `ln -s $HOMEBREW_PREFIX/lib/docker/cli-plugins/docker-compose ~/.docker/cli-plugins/docker-compose` |
 | **Windows** | Follow the [WSL2 + Native Docker Guide](https://gist.github.com/martinsam16/4492957e3bbea34046f2c8b49c3e5ac0) |
 
----
-
 ## Getting Started
 
-### 1. Environment Configuration
-
-Ensure you have a `.env` file in the root directory with the following variables defined (referencing the Compose file):
-
-- `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB`
-- `KEYCLOAK_ADMIN`, `KEYCLOAK_ADMIN_PASSWORD`
-- `OIDC_CLIENT_ID`, `OIDC_CLIENT_SECRET`
-- `N8N_ENCRYPTION_KEY`, `N8N_USER_MANAGEMENT_JWT_SECRET`
+### Spin Up the Sandbox
 
-### 2. Spin Up the Sandbox
+Launch the environment. The setup includes automated provisioning for Keycloak and n8n custom nodes and external endpoints.
 
-Launch the environment. The setup includes automated provisioning for Keycloak and n8n demo data.
+In `docker-compose` directory, run:
 
 ```bash
-docker-compose up --build -d
+docker compose --env-file .env.example up --build
 
 ```
 
-## ⚙️ Service Catalog
+## Service Catalog
 
 Once the containers are healthy, the following services are available:
 
-| Service      | Description                                   | Local Access                                                                   |
-| ------------ | --------------------------------------------- | ------------------------------------------------------------------------------ |
-| **Keycloak** | Identity & Access Management (Custom Build)   | [http://localhost:8080](https://www.google.com/search?q=http://localhost:8080) |
-| **n8n**      | Workflow Automation (with OIDC & AI enabled)  | [http://localhost:5678](https://www.google.com/search?q=http://localhost:5678) |
-| **Postgres** | Primary DB (v18.0) - Shared by n8n & Keycloak | `localhost:5432`                                                               |
-| **Qdrant**   | Vector Database for AI/RAG                    | [http://localhost:6333](https://www.google.com/search?q=http://localhost:6333) |
-| **Ollama**   | Local LLM Runner (Optional/Commented Out)     | `localhost:11434`                                                              |
+| Service      | Description                  | Local Access                                                                   |
+| ------------ | ---------------------------- | ------------------------------------------------------------------------------ |
+| **Keycloak** | Identity & Access Management | [http://localhost:8080](https://www.google.com/search?q=http://localhost:8080) |
+| **n8n**      | Workflow Automation          | [http://localhost:5678](https://www.google.com/search?q=http://localhost:5678) |
+| **Postgres** | Primary DB (v18.0)           | `localhost:5432`                                                               |
+| **Ollama**   | Local LLM Runner (Optional)  | `localhost:11434`                                                              |
+| **Qdrant**   | Vector Database (Optional)   | `localhost:6333`                                                               |
 
----
+## Automated Provisioning Details
 
-## 🛠️ Automated Provisioning
+The sandbox uses a "Dependency Chain" to ensure services configure themselves in the correct order:
 
-The sandbox includes "Run-Once" containers that handle the heavy lifting of environment setup:
+1. **Postgres (`initdb`)**: Creates the initial schemas. **Note:** Ensure your `./initdb` folder contains a script to create the `keycloak` database, as Postgres only creates the one defined in `POSTGRES_DB` by default.
+2. **Keycloak Provisioning**: The `keycloak-provision` service uses the Keycloak Admin CLI or API to create the `starter` realm and the OIDC client.
+3. **n8n Provisioning**:
 
-- **`keycloak-provision`**: Automatically configures the `starter` realm and OIDC clients after Keycloak starts.
-- **`n8n-import`**:
-- Checks if workflows exist; if not, imports credentials and workflows from `./n8n/demo-data`.
-- Installs community nodes from `../community-nodes`.
-
-- **`initdb`**: The Postgres service automatically executes scripts located in `./initdb` on first boot.
+- **Idempotency**: Checks if workflows exist before importing to avoid duplicates.
+- **Custom Nodes**: Injects community nodes into the `/home/node/.n8n/nodes` directory.
+- **Hooks**: Runs `migrate.cjs` to set up external hooks (e.g., custom logging or auditing).
 
 ## 💾 Persistence & Volumes
 
-Data is persisted across restarts using named Docker volumes:
-
-- `n8n_storage`: Config, nodes, and local binary data.
-- `postgres_storage`: All relational data (n8n & Keycloak tables).
-- `qdrant_storage`: Vector collections and indexes.
-- `ollama_storage`: Downloaded LLM models (e.g., Llama 3.2).
+Data is persisted across restarts using named Docker volumes. If you need to **wipe the environment**, run `docker-compose down -v`.
 
-> **Note on AI Features:** Ollama is currently commented out in the `docker-compose.yaml`. To use local AI nodes in n8n, uncomment the `ollama` and `ollama-pull-llama` services.
+- `n8n_storage`: Stores the `.n8n` folder, including SQLite (if used), binary data, and installed nodes.
+- `postgres_storage`: Stores the physical database files for both n8n and Keycloak.
+- `ollama_storage` / `qdrant_storage`: (Reserved) Stores downloaded AI models and vector embeddings.