feat: kyc onboarding bundle

.gitignore

@@ -7,10 +7,27 @@ Thumbs.db
 .idea/
 *.iml
 
+# Environment files (contains sensitive credentials)
+.env
+.env.local
+
 # Python
 __pycache__/
 *.pyc
 *.pyo
 *.pyd
 .Python
 *.so
+
+# Just keep some stuff locally
+.local/
+
+# Downloaded workflow outputs (local testing artifacts)
+downloaded-outputs/
+
+# Helm dependency artifacts
+**/chart/charts/
+
+# Claude Code
+CLAUDE.md
+.claude/

demos/cobol-modernization-bundle/Makefile

@@ -1,27 +1,43 @@
-.PHONY: help build build-data-seeder install install-with-argo uninstall cobol-demo upload-data clean
+.PHONY: help build build-data-seeder install install-with-argo uninstall cobol-demo upload-data delete-data-seeder-job clean
 
 NAMESPACE ?= default
 RELEASE_NAME ?= cobol-modernization-bundle
 SPEECH_MCP_IMAGE ?= speech-mcp-server:latest
 DATA_SEEDER_IMAGE ?= cobol-demo-data-seeder:latest
+DATA_SEEDER_ENABLED ?= true
+# Set to true to skip installing file-gateway and use the one already in the namespace (e.g. from KYC demo)
+USE_EXISTING_FILE_GATEWAY ?= false
+# Set to 1 to skip 'helm dependency build' (use if it hangs at "Deleting outdated charts"; run it once manually first)
+SKIP_DEP_BUILD ?= 0
+FILE_GATEWAY_SKIP_FLAGS := --set file-gateway.fileApi.enabled=false --set file-gateway.filesystemMcp.enabled=false --set file-gateway.versitygw.enabled=false --set file-gateway.versitygw.createSecret=false --set file-gateway.storage.enabled=false --set file-gateway.httpRoute.enabled=false --set file-gateway.fileApi.enabled=false --set file-gateway.filesystemMcp.enabled=false --set file-gateway.versitygw.enabled=false --set file-gateway.versitygw.createSecret=false --set file-gateway.storage.enabled=false --set file-gateway.httpRoute.enabled=false
+ifeq ($(USE_EXISTING_FILE_GATEWAY),true)
+FILE_GATEWAY_SKIP := $(FILE_GATEWAY_SKIP_FLAGS)
+else
+FILE_GATEWAY_SKIP :=
+endif
 
 help:
 	@echo "COBOL Modernization Demo Bundle"
 	@echo ""
 	@echo "Namespace: $(NAMESPACE) (override with NAMESPACE=my-ns; omit for default)"
 	@echo ""
 	@echo "Required steps (prerequisite: create a Model in target namespace for agents):"
+	@echo "  make uninstall            Remove bundle and file-gateway (run first for a clean install)"
 	@echo "  make build                Build speech-mcp-server Docker image (auto-loads into minikube)"
+	@echo "  make build-data-seeder    Build data-seeder image (only needed for Option A: automatic upload on install)"
 	@echo "  make install              Install bundle (agents + file-gateway)"
 	@echo "  make install-with-argo    Install bundle (agents + file-gateway + WorkflowTemplate)"
-	@echo "  make upload-data          Upload sample COBOL files and audio"
+	@echo "  make upload-data          Upload sample COBOL files and audio (Option B or re-upload)"
 	@echo "  make cobol-demo           Run COBOL modernization workflow"
-	@echo "  make uninstall            Remove bundle and cleanup"
+	@echo "  make delete-data-seeder-job  Delete the data-seeder Job (e.g. before testing Option B)"
 	@echo ""
-	@echo "Quick Start:"
-	@echo "  make build && make install-with-argo && make upload-data && make cobol-demo"
+	@echo "Use USE_EXISTING_FILE_GATEWAY=true if file-gateway is already in the namespace (e.g. from KYC)."
+	@echo "Use SKIP_DEP_BUILD=1 if install hangs at 'Deleting outdated charts' (run 'helm dependency build chart/' once manually first)."
+	@echo ""
+	@echo "Quick Start Option A: make uninstall && make build-data-seeder && make build && make install-with-argo && make cobol-demo"
+	@echo "Quick Start Option B: make uninstall && make build && make install-with-argo DATA_SEEDER_ENABLED=false && make upload-data && make cobol-demo"
 
-build: build-data-seeder
+build:
 	@echo "Building speech-mcp-server image..."
 	@docker build -t $(SPEECH_MCP_IMAGE) ../../mcps/speech-mcp-server/
 	@echo "Done. Image: $(SPEECH_MCP_IMAGE)"
@@ -34,10 +50,17 @@ build: build-data-seeder
 install:
 	@echo "Installing COBOL Modernization Bundle into namespace: $(NAMESPACE)"
 	@echo "Building Helm dependencies..."
-	@helm dependency build chart/
-	@helm upgrade --install $(RELEASE_NAME) chart/ \
+	@if [ "$(SKIP_DEP_BUILD)" = "1" ]; then echo "Skipping helm dependency build (SKIP_DEP_BUILD=1)"; else helm dependency build chart/; fi
+	@SKIP="$(FILE_GATEWAY_SKIP)"; \
+	if [ -z "$$SKIP" ] && kubectl get serviceaccount file-gateway-file-api -n $(NAMESPACE) --request-timeout=10s 2>/dev/null | grep -q file-gateway-file-api; then \
+		SKIP="$(FILE_GATEWAY_SKIP_FLAGS)"; \
+		echo "Detected existing file-gateway in namespace - skipping file-gateway install"; \
+	fi; \
+	echo "Installing/upgrading release (this may take 1–2 minutes)..."; \
+	helm upgrade --install $(RELEASE_NAME) chart/ \
 		--namespace $(NAMESPACE) \
 		--create-namespace \
+		$$SKIP \
 		--wait
 	@echo "Applying namespace discovery labels..."
 	@if [ "$(NAMESPACE)" = "default" ]; then \
@@ -54,10 +77,25 @@ install:
 install-with-argo: build
 	@echo "Installing COBOL Modernization Bundle with Argo support into namespace: $(NAMESPACE)"
 	@echo "Building Helm dependencies..."
-	@helm dependency build chart/
-	@helm upgrade --install $(RELEASE_NAME) chart/ \
+	@if [ "$(SKIP_DEP_BUILD)" = "1" ]; then echo "Skipping helm dependency build (SKIP_DEP_BUILD=1)"; else helm dependency build chart/; fi
+	@echo "Checking for existing file-gateway..."
+	@SKIP="$(FILE_GATEWAY_SKIP)"; NO_HOOKS=""; \
+	if [ -z "$$SKIP" ] && kubectl get serviceaccount file-gateway-file-api -n $(NAMESPACE) --request-timeout=10s 2>/dev/null | grep -q file-gateway-file-api; then \
+		SKIP="$(FILE_GATEWAY_SKIP_FLAGS)"; \
+		NO_HOOKS="--no-hooks"; \
+		echo "Detected existing file-gateway in namespace - skipping file-gateway install (hooks skipped; run make upload-data to upload sample data)"; \
+	fi; \
+	if [ "$(DATA_SEEDER_ENABLED)" = "false" ]; then \
+		NO_HOOKS="--no-hooks"; \
+		echo "Data seeder disabled - hooks skipped; run make upload-data to upload sample data"; \
+	fi; \
+	[ -n "$$NO_HOOKS" ] || true; \
+	echo "Installing/upgrading release (this may take 1–2 minutes)..."; \
+	helm upgrade --install $(RELEASE_NAME) chart/ \
 		--namespace $(NAMESPACE) \
 		--create-namespace \
+		--set dataSeeder.enabled=$(DATA_SEEDER_ENABLED) \
+		$$SKIP $$NO_HOOKS \
 		--wait
 	@echo "Applying namespace discovery labels..."
 	@if [ "$(NAMESPACE)" = "default" ]; then \
@@ -73,44 +111,45 @@ install-with-argo: build
 	@echo "  ✓ File-gateway with MCP tools"
 	@echo "  ✓ Argo Workflows RBAC + ServiceAccount"
 	@echo "  ✓ WorkflowTemplate (visible in ARK Dashboard)"
+	@echo ""
+	@echo "Run 'make upload-data' to upload sample data (or if using Option A, data-seeder hook may have already run)."
+	@echo ""
+
+delete-data-seeder-job:
+	@echo "Deleting data-seeder Job (if present)..."
+	@kubectl delete job data-seeder -n $(NAMESPACE) 2>/dev/null && echo "Deleted." || echo "No data-seeder job found (already deleted or not installed)."
 
 uninstall:
 	@echo "Uninstalling COBOL Modernization Bundle..."
 	@kubectl delete workflowtemplate cobol-modernization-template -n $(NAMESPACE) 2>/dev/null || true
 	@helm uninstall $(RELEASE_NAME) -n $(NAMESPACE) 2>/dev/null || true
-	@echo "Done."
+	@echo "Removing file-gateway (and related MCP resources) for a clean next install..."
+	@kubectl delete deployment,service,serviceaccount,secret,pvc -l app.kubernetes.io/name=file-gateway -n $(NAMESPACE) --ignore-not-found --timeout=60s 2>/dev/null || true
+	@kubectl delete httproute -l app.kubernetes.io/name=file-gateway -n $(NAMESPACE) --ignore-not-found 2>/dev/null || true
+	@kubectl delete mcpserver -l app.kubernetes.io/name=file-gateway -n $(NAMESPACE) --ignore-not-found 2>/dev/null || true
+	@kubectl delete job data-seeder -n $(NAMESPACE) --ignore-not-found 2>/dev/null || true
+	@echo "Done. Next install will deploy a fresh file-gateway (no 'existing file-gateway' skip)."
 
 upload-data:
-	@echo "Uploading sample COBOL files to file-gateway..."
-	@echo "Waiting for file-gateway filesystem-mcp deployment to be ready..."
-	@kubectl rollout status deployment/file-gateway-filesystem-mcp -n $(NAMESPACE) --timeout=120s
-	@echo "Creating directory structure..."
-	@kubectl exec -n $(NAMESPACE) deployment/file-gateway-filesystem-mcp -c filesystem-mcp -- mkdir -p \
-		/data/aas-files/cobol-source \
-		/data/aas-files/output/pseudocode \
-		/data/aas-files/output/transcription \
-		/data/aas-files/output/diagrams \
-		/data/aas-files/output/python
-	@echo "Starting port-forward to file-gateway API..."
+	@echo "Uploading sample COBOL files to file-gateway (via File API, same as data-seeder)..."
+	@echo "Waiting for file-gateway-api to be ready..."
+	@kubectl rollout status deployment/file-gateway-file-api -n $(NAMESPACE) --timeout=120s 2>/dev/null || true
+	@echo "Starting port-forward to file-gateway-api..."
 	@(kubectl port-forward -n $(NAMESPACE) svc/file-gateway-api 8181:80 > /dev/null 2>&1 &) && \
-	sleep 4 && \
-	echo "Testing connection..." && \
-	curl -s http://localhost:8181/health > /dev/null && \
+	sleep 3 && \
+	curl -sf http://localhost:8181/health > /dev/null || (echo "Error: file-gateway-api not reachable"; pkill -f "port-forward.*file-gateway-api" 2>/dev/null; exit 1) && \
 	echo "Uploading COBOL files..." && \
 	for file in examples/data/cobol-source/*.cbl; do \
-		echo "  Uploading $$(basename $$file)..." && \
-		curl -s -X POST http://localhost:8181/files \
-			-F "file=@$$file" \
-			-F "prefix=cobol-source/" > /dev/null; \
+		[ -f "$$file" ] || continue; \
+		echo "  → $$(basename $$file)"; \
+		curl -sf -X POST http://localhost:8181/files -F "file=@$$file" -F "prefix=cobol-source/" > /dev/null || (pkill -f "port-forward.*file-gateway-api" 2>/dev/null; exit 1); \
 	done && \
-	echo "Uploading audio file..." && \
-	curl -s -X POST http://localhost:8181/files \
+	echo "  → carddemo-interview.m4a" && \
+	curl -sf -X POST http://localhost:8181/files \
 		-F "file=@examples/data/cobol-source/carddemo-interview.m4a" \
-		-F "prefix=cobol-source/" > /dev/null && \
-	pkill -f "port-forward.*file-gateway-api" > /dev/null 2>&1 && \
-	echo "" && \
-	echo "Done. Sample data uploaded successfully." && \
-	echo "Files available at: cobol-source/"
+		-F "prefix=cobol-source/" > /dev/null || (pkill -f "port-forward.*file-gateway-api" 2>/dev/null; exit 1) && \
+	pkill -f "port-forward.*file-gateway-api" 2>/dev/null || true && \
+	echo "Done. Sample data uploaded; visible in ARK Dashboard → Files (cobol-source/)."
 
 cobol-demo:
 	@echo "Submitting COBOL modernization workflow..."

demos/cobol-modernization-bundle/README.md

@@ -17,45 +17,63 @@ This bundle deploys **6 AI agents** that mirror the LegacyX Groups:
 
 Plus supporting infrastructure:
 - `speech-mcp-server` for local audio transcription via Whisper
-- `file-gateway` MCP server for file operations (read/write/list)
+- **`file-gateway`** (installed as a **Helm dependency** )
 - Argo Workflow RBAC (ServiceAccount + Role + RoleBinding)
-- Data seeder (Helm post-install hook that uploads sample files automatically)
+- **Data seeder** (optional): Helm post-install hook uploads sample files; run `make build-data-seeder` before install for Option A
+- **`make upload-data`** (Option B): manual upload via file-gateway-api
 - WorkflowTemplate for the full COBOL modernization pipeline
 
 ## Prerequisites
 
 - **Model for agents:** A Model (e.g. named `default`) must exist in the namespace where you install the bundle. Agents use this Model to run, workflows will fail until it is present. Create the Model (e.g. via ARK Dashboard → Models or `ark models create default`) in the target namespace. 
 - ARK cluster (Azure OpenAI or other provider)
 - Argo Workflows installed
-- `file-gateway` service (installed as dependency)
-- Docker (to build the speech-mcp-server image)
+- **File Gateway** is installed automatically with this bundle (Helm dependency); no separate install step.
+- Docker (to build the speech-mcp-server and data-seeder images)
 - `kubectl` and `helm` CLI tools
 
 ## Local Development
 
-You can install the bundle in **any namespace**. If you don't set `NAMESPACE`, it installs in **default**. Ensure the prerequisites are met.
+Install in any namespace (default if omitted). Prerequisites must be met. If file-gateway is already in the namespace (e.g. from the KYC demo), use `USE_EXISTING_FILE_GATEWAY=true` so the bundle skips installing it and uses the existing one.
+
+**For a clean install** (recommended): run `make uninstall` first to remove any existing bundle and file-gateway; then run the steps below.
+
+### Option A: Data seeder (automatic upload on install)
+
+Build the data-seeder image, then build and install. A post-install hook runs the seeder and uploads sample COBOL/audio.
 
 ```bash
 cd agents-at-scale-marketplace/demos/cobol-modernization-bundle
-
-# Install in default namespace (no NAMESPACE needed)
+make uninstall
+make build-data-seeder
 make build
 make install-with-argo
-make upload-data
 make cobol-demo
+```
 
-# Or install in a specific namespace (create/copy Model there first if needed)
-make install-with-argo NAMESPACE=cobol-demo
-make upload-data NAMESPACE=cobol-demo
-make cobol-demo NAMESPACE=cobol-demo
+### Option B: Manual upload
 
-# View results (use the namespace you installed into)
-kubectl get workflows -n default   # or -n cobol-demo
+Build and install with the data-seeder disabled, then run `make upload-data` to upload from `examples/data/cobol-source/`.
 
-# Cleanup (use same NAMESPACE as install)
+```bash
+cd agents-at-scale-marketplace/demos/cobol-modernization-bundle
 make uninstall
+make build
+make install-with-argo DATA_SEEDER_ENABLED=false
+make upload-data
+make cobol-demo
 ```
 
+**Testing Option B after Option A:** Re-run install with the seeder disabled, then upload manually. Optionally delete the existing data-seeder Job first for a clean state:
+```bash
+make delete-data-seeder-job
+make install-with-argo DATA_SEEDER_ENABLED=false
+make upload-data
+make cobol-demo
+```
+
+Use `NAMESPACE=<ns>` for install/upload/cobol-demo when using a non-default namespace. Cleanup: `make uninstall`.
+
 ## Cloud Deployment
 
 ```bash

demos/cobol-modernization-bundle/chart/templates/data-seeder-job.yaml

@@ -12,13 +12,29 @@ metadata:
     helm.sh/hook-weight: "10"
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 spec:
-  backoffLimit: 3
+  backoffLimit: 10
+  activeDeadlineSeconds: 900
   template:
     metadata:
       labels:
         app: data-seeder
     spec:
       restartPolicy: OnFailure
+      initContainers:
+        - name: wait-for-gateway
+          image: "{{ .Values.dataSeeder.image.repository }}:{{ .Values.dataSeeder.image.tag }}"
+          imagePullPolicy: {{ .Values.dataSeeder.image.pullPolicy }}
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              GATEWAY="http://file-gateway-api:80"
+              echo "Waiting for file-gateway-api to be ready..."
+              for i in $(seq 1 90); do
+                if curl -sf "${GATEWAY}/health"; then echo "file-gateway-api is ready."; exit 0; fi
+                echo "  attempt $$i/90, retrying in 5s..."
+                sleep 5
+              done
+              echo "Timeout waiting for file-gateway-api"; exit 1
       containers:
         - name: data-seeder
           image: "{{ .Values.dataSeeder.image.repository }}:{{ .Values.dataSeeder.image.tag }}"