Add Transfer Script

Author: Lucas0T

.github/workflows/build.yml

@@ -155,6 +155,7 @@ jobs:
       - name: Load Torch Image
         run: docker load --input /tmp/torch.tar
 
+
       - name: Check out Git repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
@@ -177,12 +178,68 @@ jobs:
 
       - name: Run Blackbox Integration Tests
         run: mvn -P blackbox-integration-tests -B verify
+  
+  
+  script-integration-test:
+    needs: build
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download Torch Image
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: torch-image
+          path: /tmp
+
+      - name: Load Torch Image
+        run: docker load --input /tmp/torch.tar
+
+      - name: Check out Git repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
+      - name: Start Docker Compose
+        working-directory: .github/workflows/script-integration-test
+        run: docker compose up -d
+
+      - name: Wait for source-fhir-server to become healthy
+        run: |
+          set -e
+
+          echo "⏳ Waiting for source-fhir-server container to become healthy..."
+
+          # Get the container name (adjust if you're not using default naming)
+          CONTAINER=$(docker ps --filter "name=source-fhir-server" --format "{{.Names}}" | head -n1)
+
+          if [ -z "$CONTAINER" ]; then
+            echo "❌ No container found matching source-fhir-server"
+            docker ps -a
+            exit 1
+          fi
+
+          for i in {1..30}; do
+            STATUS=$(docker inspect --format='{{.State.Health.Status}}' "$CONTAINER" || echo "unavailable")
+            echo "Attempt $i - Status: $STATUS"
+            if [ "$STATUS" == "healthy" ]; then
+              echo "✅ $CONTAINER is healthy"
+              exit 0
+            fi
+            sleep 10
+          done
+
+          echo "❌ $CONTAINER did not become healthy in time"
+          docker logs "$CONTAINER"
+          exit 1
+
+      - name: Run integration script
+        run: .github/workflows/script-integration-test/integration-test.sh
+  
+  
+  
   push-image:
     needs:
       - test
       - image-scan
       - blackbox-integration-tests
+      - script-integration-test
     runs-on: ubuntu-24.04
     if: ${{ ! startsWith(github.head_ref, 'dependabot/')}}
 

.github/workflows/script-integration-test/docker-compose.yml

@@ -0,0 +1,69 @@
+services:
+  source-fhir-server:
+    container_name: source-fhir-server
+    image: samply/blaze:1.0
+    environment:
+      BASE_URL: http://source-fhir-server:8080
+      LOG_LEVEL: debug
+      JAVA_TOOL_OPTIONS: -Xmx2g
+    ports:
+      - "8083:8080"
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8080/health" ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 40s
+
+  target-fhir-server:
+    container_name: target-fhir-server
+    image: samply/blaze:1.0
+    environment:
+      BASE_URL: http://target-fhir-server:8080
+      LOG_LEVEL: debug
+      JAVA_TOOL_OPTIONS: -Xmx2g
+    ports:
+      - "8084:8080"
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8080/health" ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 40s
+  torch:
+    container_name: torch
+    restart: unless-stopped
+    image: torch:latest
+    ports:
+      - "8080"
+    environment:
+      TORCH_PROFILE_DIR: /app/structureDefinitions
+      TORCH_MAPPING_CONSENT: /app/mappings/consent-mappings_fhir.json
+      TORCH_MAPPING_TYPE_TO_CONSENT: /app/mappings/type_to_consent.json
+      TORCH_FHIR_URL: http://source-fhir-server:8080/fhir
+      TORCH_FHIR_MAX_CONNECTIONS: 4
+      TORCH_RESULTS_DIR: /app/output
+      LOG_LEVEL: debug
+      TORCH_OUTPUT_FILE_SERVER_URL: http://localhost:8082
+      TORCH_BASE_URL: http://localhost:8082
+      TORCH_BATCHSIZE: 100
+      TORCH_MAXCONCURRENCY: 4
+      TORCH_MAPPINGSFILE: /app/ontology/mapping_cql.json
+      TORCH_CONCEPTTREEFILE: /app/ontology/mapping_tree.json
+      TORCH_USECQL: true
+      JAVA_TOOL_OPTIONS: -Xmx4g
+    volumes:
+      - torch-data-store:/app/output
+
+  nginx:
+    container_name: nginx
+    restart: unless-stopped
+    image: nginxinc/nginx-unprivileged:1.25.5-alpine
+    ports:
+      - "8082:8080"
+    volumes:
+      - torch-data-store:/app/output
+      - ./nginx.conf:/etc/nginx/nginx.conf
+
+volumes:
+  torch-data-store:

.github/workflows/script-integration-test/integration-test.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+set -euo pipefail
+
+# 1. Install blazectl
+VERSION="1.0.0"
+URL="https://github.com/samply/blazectl/releases/download/v${VERSION}/blazectl-${VERSION}-linux-amd64.tar.gz"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+
+echo "📦 Downloading blazectl ${VERSION}..."
+curl -LO "$URL"
+
+echo "📂 Extracting binary..."
+tar xzf "blazectl-${VERSION}-linux-amd64.tar.gz"
+
+echo "➡️ Installing to /usr/local/bin..."
+sudo mv blazectl /usr/local/bin/blazectl
+chmod +x /usr/local/bin/blazectl
+
+echo "🔍 Installed version:"
+blazectl --version
+
+rm blazectl-${VERSION}-linux-amd64.tar.gz
+
+# 2. Upload BlazeBundle to source server (seed data)
+echo "➡️ Uploading initial data bundle..."
+
+echo "Root dir: $ROOT_DIR"
+ls "$ROOT_DIR/src/test/resources/BlazeBundle.json"
+
+if curl -s http://localhost:8083/fhir/metadata?_elements=software | jq -r '.software.name' | grep -iq blaze; then
+  echo "✅ Source FHIR Server Live"
+else
+  echo "❌ Source FHIR Server Not Working"
+  exit 1
+fi
+
+
+echo "📤 Posting BlazeBundle.json to http://localhost:8083/fhir"
+curl -i -s \
+  -X POST "http://localhost:8083/fhir" \
+  -H "Content-Type: application/fhir+json" \
+  --data-binary @"$ROOT_DIR/src/test/resources/BlazeBundle.json"
+
+TARGET_SERVER=http://localhost:8084/fhir
+
+# 3. Run extraction + transfer script
+echo "➡️ Running extraction and transfer..."
+TORCH_BASE_URL=http://localhost:8082 \
+"$ROOT_DIR/scripts/transfer-extraction-to-dup-fhir-server.sh" -c "$ROOT_DIR/src/test/resources/CRTDL/CRTDL_observation_all_fields_withoutReference.json" -t "$TARGET_SERVER"
+
+# 4. Assertions – Patient count
+EXPECTED_PATIENT_COUNT=4
+ACTUAL_PATIENT_COUNT=$(curl -s "${TARGET_SERVER}/Patient?_summary=count" | jq '.total')
+
+if [ "$ACTUAL_PATIENT_COUNT" -eq "$EXPECTED_PATIENT_COUNT" ]; then
+  echo "✅ Patient count is correct: $ACTUAL_PATIENT_COUNT"
+else
+  echo "❌ Patient count is $ACTUAL_PATIENT_COUNT, expected $EXPECTED_PATIENT_COUNT"
+  exit 1
+fi
+
+# 5. Assertions – Observation count
+EXPECTED_OBSERVATION_COUNT=4
+ACTUAL_OBSERVATION_COUNT=$(curl -s "${TARGET_SERVER}/Observation?_summary=count" | jq '.total')
+
+if [ "$ACTUAL_OBSERVATION_COUNT" -eq "$EXPECTED_OBSERVATION_COUNT" ]; then
+  echo "✅ Observation count is correct: $ACTUAL_OBSERVATION_COUNT"
+else
+  echo "❌ Observation count is $ACTUAL_OBSERVATION_COUNT, expected $EXPECTED_OBSERVATION_COUNT"
+  exit 1
+fi
+
+
+echo "🎉 All integration tests passed!"

.github/workflows/script-integration-test/nginx.conf

@@ -0,0 +1,38 @@
+events {}
+pid /tmp/nginx.pid;
+
+http {
+    server {
+        listen 8080;
+        server_name localhost;
+
+        location /fhir/ {
+            proxy_pass http://torch:8080/fhir/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            add_header X-Proxy-Target $proxy_host;
+            add_header X-Upstream-Addr $upstream_addr;
+        }
+
+        location / {
+            root /app/output;
+            index index.html;
+            try_files $uri $uri/ =404;
+        }
+
+        error_page 404 /404.html;
+        location = /404.html {
+            internal;
+        }
+
+        access_log /var/log/nginx/access.log;
+        error_log /var/log/nginx/error.log;
+    }
+}

.gitignore

@@ -20,4 +20,4 @@ dependencies
 package.json
 fhirpkg.lock.json
 .env
-exec-crtdl.sh
+exec-crtdl.sh

README.md

@@ -270,6 +270,37 @@ For example a CRTDL only extracting Condition.onset will result in this:
 }
 ```
 
+## Transfer Script
+
+TORCH provides a companion **transfer script** designed to automate the workflow of submitting a data extraction
+request, polling the status, and transferring the resulting files to a target FHIR server.
+
+The transfer script will:
+
+1. Take the **CRTDL** and generate a FHIR parameters resource to send to TORCH.
+2. Execute $extract-data operation on the TORCH server using parameters resource as input.
+3. Poll the TORCH status endpoint until the export is complete.
+4. Download the resulting patient-oriented FHIR bundles into a temp dir.
+5. Upload these files to a configured target FHIR server using the `blazectl` tool.
+6. Provide progress feedback and error handling at each step.
+
+### Usage Example
+
+```bash
+./transfer-extraction-to-dup-fhir-server.sh -c src/test/resources/CRTDL/CRTDL_observation.json -t  http://target-fhir-server:8080/fhir
+```
+
+### Environment Variables
+
+The transfer script respects several environment variables to configure server URLs, directories, retry counts, and
+timing:
+
+| Variable       | Default               | Description                                         |
+|----------------|-----------------------|-----------------------------------------------------|
+| TORCH_BASE_URL | http://localhost:8080 | Base URL of the TORCH server                        |
+| MAX_RETRIES    | 60                    | Number of status polling attempts before timing out |
+| SLEEP_SECONDS  | 5                     | Seconds to wait between polling attempts            |
+
 ## Supported Features
 
 - Loading of StructureDefinitions