API Client dpc-api integration test

.github/workflows/ci-workflow.yml

@@ -32,14 +32,15 @@ concurrency:
 
 env:
   VAULT_PW: ${{ secrets.VAULT_PW }}
-  REPORT_COVERAGE: true
   DPC_CA_CERT: ${{ secrets.DPC_CA_CERT }}
   ENV: "github-ci"
 
 jobs:
   build-api:
     name: "Build and Test API"
     runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}}
+    env:
+      REPORT_COVERAGE: true
     steps:
       - name: Assert Ownership
         run: sudo chmod -R 777 .
@@ -194,8 +195,8 @@ jobs:
         if: ${{ always() }}
         run: ./scripts/cleanup-docker.sh
 
-  build-dpc-client:
-    name: "Build and Test DPC Client"
+  build-dpc-api-client:
+    name: "Build and Test DPC API Client"
     runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}}
     steps:
       - name: Assert Ownership
@@ -204,9 +205,38 @@ jobs:
         uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
       - name: Cleanup Runner
         run: ./scripts/cleanup-docker.sh
-      - name: "DPC Client Build"
+      - name: Install docker compose manually
         run: |
-          make ci-api-client
+          mkdir -p /usr/local/lib/docker/cli-plugins
+          curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose
+          chown root:root /usr/local/lib/docker/cli-plugins/docker-compose
+          chmod +x /usr/local/lib/docker/cli-plugins/docker-compose
+      - name: "API Client Unit Tests"
+        run: make ci-api-client
+      - name: "Set up Python and install Ansible"
+        run: |
+          sudo dnf -y install python3 python3-pip
+          pip install ansible
+      - name: "Set up JDK 17"
+        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
+        with:
+          java-version: "17"
+          distribution: "corretto"
+          cache: maven
+      - name: Clean maven
+        run: mvn -ntp -U clean
+      - name: "API Client Integration test"
+        id: integration-test
+        run: make ci-api-client-integration
+      - name: "Debug db"
+        if: ${{ failure() && steps.integration-test.outcome == 'failure' }}
+        run: docker logs api-client-integration-app-db-1
+      - name: "Debug attribution"
+        if: ${{ failure() && steps.integration-test.outcome == 'failure' }}
+        run: docker logs api-client-integration-app-attribution-1
+      - name: "Debug api"
+        if: ${{ failure() && steps.integration-test.outcome == 'failure' }}
+        run: docker logs api-client-integration-app-api-1
       - name: Cleanup
         if: ${{ always() }}
         run: ./scripts/cleanup-docker.sh

Makefile

@@ -43,6 +43,8 @@ portal:
 	cp -r engines/api_client/ dpc-portal/vendor/api_client/
 	@docker build -f dpc-portal/Dockerfile . -t dpc-web-portal
 
+api-client:
+	@docker compose -f docker-compose.yml -f docker-compose.api-client.yml build dpc_api_client
 
 # Start commands
 # ==============
@@ -178,6 +180,8 @@ portal-sh: ## Run a portal shell
 portal-console: ## Run a rails console shell
 	@docker compose -f docker-compose.yml -f docker-compose.portals.yml exec -it dpc_portal bin/console
 
+api-client-sh:
+	@docker compose -f docker-compose.api-client.yml run --remove-orphans --entrypoint "sh" dpc_api_client
 
 # Build & Test commands
 # ======================
@@ -228,3 +232,7 @@ unit-tests:
 
 .PHONY: load-tests
 load-tests: start-api-load-tests start-load-tests down-dpc-load-tests
+
+.PHONY: ci-api-client-integration
+ci-api-client-integration: docker-base secure-envs
+	@bash ./dpc-api-client-integration-test.sh

docker-compose.api-client.yml

@@ -0,0 +1,18 @@
+services:
+  dpc_api_client:
+    build:
+      context: .
+      dockerfile: engines/api_client/Dockerfile
+    image: dpc-api-client:latest
+    volumes:
+      # Mount specific directories to avoid overwriting
+      # precompiled assets (public/assets/) and node_modules
+      - "./engines/api_client:/api-client"
+    environment:
+      # Application settings
+      - GOLDEN_MACAROON=${GOLDEN_MACAROON}
+      - API_METADATA_URL=http://api:3002/api/v1
+      - API_ADMIN_URL=http://api:9900
+      - RUBY_YJIT_ENABLE=1
+      - SKIP_SIMPLE_COV=${SKIP_SIMPLE_COV:-}
+      - ENV=local

docker-compose.base.yml

@@ -3,6 +3,6 @@ services:
     image: dpc-base:latest
     build:
       args:
-        PACE_CERT: $PACE_CERT
+        PACE_CERT: ${PACE_CERT:-}
       context: .
       dockerfile: docker/Dockerfiles/Dockerfile.base

docker-compose.yml

@@ -34,7 +34,7 @@ services:
       - ENV=local
       - USE_BFD_MOCK=${USE_BFD_MOCK:-true}
       - EMIT_AWS_METRICS=${EMIT_AWS_METRICS:-false}
-      - JACOCO=${REPORT_COVERAGE:-false}
+      - JACOCO=${REPORT_COVERAGE:-}
       - DEBUG_MODE=${DEBUG_MODE:-false}
     depends_on:
       db:
@@ -56,7 +56,7 @@ services:
         condition: service_healthy
     environment:
       - ENV=local
-      - JACOCO=${REPORT_COVERAGE:-false}
+      - JACOCO=${REPORT_COVERAGE:-}
       - DEBUG_MODE=${DEBUG_MODE:-false}
     ports:
       - "3500:8080"
@@ -79,7 +79,7 @@ services:
       - ./ops/config/decrypted/local.env
     environment:
       - ENV=local
-      - JACOCO=${REPORT_COVERAGE:-false}
+      - JACOCO=${REPORT_COVERAGE:-}
       - EXPORT_PATH=/app/data
       - AUTH_DISABLED=${AUTH_DISABLED:-false}
       - DEBUG_MODE=${DEBUG_MODE:-false}

dpc-api-client-integration-test.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -Ee
+
+echo "┌-------------------───────────────────────┐"
+echo "│                                          │"
+echo "│ Running API Client Gem Integration Tests |"
+echo "│                                          │"
+echo "└------------─────────-------──────────────┘"
+
+# Current working directory
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+# Configure the Maven log level
+export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.defaultLogLevel=info
+
+# Include secure environment variables
+set -o allexport
+[[ -f ${DIR}/ops/config/decrypted/local.env ]] && source ${DIR}/ops/config/decrypted/local.env
+set +o allexport
+
+# Remove jacocoReport directory
+if [ -d "${DIR}/jacocoReport" ]; then
+    rm -r "${DIR}/jacocoReport"
+fi
+
+function _finally {
+  # don't shut it down if running on ci
+  if [ "$ENV" != 'github-ci' ]; then
+    echo "SHUTTING EVERYTHING DOWN"
+    docker compose -p api-client-integration-app down
+    docker volume rm api-client-integration-app_pgdata16
+  fi
+}
+
+trap _finally EXIT
+
+# Build the application
+mvn -T 1.5C clean compile -Perror-prone -B -V -ntp -DskipTests
+mvn -T 1.5C package -Pci -ntp -DskipTests
+
+echo "Starting api server for end-to-end tests"
+USE_BFD_MOCK=true docker compose -p api-client-integration-app up api --wait
+
+echo "Starting integration tests"
+GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) \
+SKIP_SIMPLE_COV=true \
+docker compose -p api-client-integration-app \
+-f docker-compose.yml -f docker-compose.api-client.yml \
+run --remove-orphans \
+--entrypoint "bundle exec rspec --order defined --tag type:integration" \
+dpc_api_client
+
+echo "┌───────────-------──────────-------------──┐"
+echo "│                                           │"
+echo "│ API Client Gem Integration Tests Complete |"
+echo "│                                           │"
+echo "└────────────────────--------------------───┘"

dpc-api-client-test.sh

@@ -8,16 +8,16 @@ echo "│                       │"
 echo "└───────────────────────┘"
 
 # Build the container
-docker build -f engines/api_client/Dockerfile engines/api_client -t api-client
+docker compose -f docker-compose.api-client.yml build dpc_api_client
 
 # Run the tests
 echo "┌───────────────────────────┐"
 echo "│                           │"
 echo "│  Running Api Gem Tests    │"
 echo "│                           │"
 echo "└───────────────────────────┘"
-docker run --rm -v ${PWD}/engines/api_client/coverage:/api-client/coverage api-client bundle exec rspec
-docker run --rm api-client bundle exec rubocop
+docker run --rm -v ${PWD}/engines/api_client/coverage:/api-client/coverage dpc-api-client bundle exec rspec
+docker run --rm dpc-api-client bundle exec rubocop
 
 echo "┌────────────────────────────────┐"
 echo "│                                │"

engines/api_client/Dockerfile

@@ -12,13 +12,13 @@ RUN mkdir /api-client
 WORKDIR /api-client
 
 # Copy over the files needed to fetch dependencies
-COPY Gemfile Gemfile.lock api_client.gemspec /api-client/
-COPY lib /api-client/lib
+COPY engines/api_client/Gemfile engines/api_client/Gemfile.lock engines/api_client/api_client.gemspec /api-client/
+COPY engines/api_client/lib /api-client/lib
 RUN gem install bundler --no-document && \
     bundle config set force_ruby_platform true && \
     bundle install
 
-COPY app /api-client/app
-COPY spec /api-client/spec
-COPY coverage /api-client/coverage
-COPY .rubocop.yml /api-client/.rubocop.yml
+COPY engines/api_client/app /api-client/app
+COPY engines/api_client/spec /api-client/spec
+COPY engines/api_client/coverage /api-client/coverage
+COPY engines/api_client/.rubocop.yml /api-client/.rubocop.yml

engines/api_client/README.md

@@ -20,20 +20,27 @@ And then execute:
 $ bundle install
 ```
 
-## Testing
-Build the docker image
-
-In the api_client directory
+## Debugging and Development
+To build an image, use the `make` command in the project root directory.
 ```bash
-$ make build
+make api-client
 ```
 
-Run the tests until they pass
+
+To ssh into a Docker container with the dpc_client code, use the `make` command in the project root directory.
+```bash
+make api-client-sh
 ```
-$ make test
+
+## Testing
+Test using `make` commands in the project root directory.
+
+### Unit Tests
+```bash
+make ci-api-client
 ```
 
-Jump into the docker shell for iterative development
+### Integration tests with the API
+```bash
+make ci-api-client-integration
 ```
-make run
-```