From d62842f138ed6bf002b152c6034d9530c9f28d90 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:59:53 -0500 Subject: [PATCH 01/28] api client integration tests working --- Makefile | 9 +- docker-compose.portals.yml | 17 ++ dpc-client-integration-test.sh | 28 +++ engines/api_client/.rspec | 1 + engines/api_client/Dockerfile | 12 +- engines/api_client/Dockerfile.local | 24 +++ engines/api_client/Makefile | 2 +- .../spec/integration/dpc_client_spec.rb | 168 ++++++++++++++++++ engines/api_client/spec/spec_helper.rb | 1 + 9 files changed, 254 insertions(+), 8 deletions(-) create mode 100755 dpc-client-integration-test.sh create mode 100644 engines/api_client/Dockerfile.local create mode 100644 engines/api_client/spec/integration/dpc_client_spec.rb diff --git a/Makefile b/Makefile index a7e046d6db..5fdd5ce269 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,9 @@ SMOKE_THREADS ?= 10 venv: venv/bin/activate +clean-jacoco: + mkdir -p jacocoReport && rm -r jacocoReport + venv/bin/activate: requirements.txt test -d venv || python3 -m venv venv . venv/bin/activate @@ -25,7 +28,7 @@ smoke/local: start-dpc # ============== api: ## Builds the Java API services -api: secure-envs +api: secure-envs clean-jacoco mvn clean compile -Perror-prone -B -V -ntp -T 4 -DskipTests mvn package -Pci -ntp -T 4 -DskipTests @@ -228,3 +231,7 @@ unit-tests: .PHONY: load-tests load-tests: start-api-load-tests start-load-tests down-dpc-load-tests + +.PHONY: api-client-integration-tests +api-client-integration-tests: clean-jacoco docker-base api + @bash ./dpc-client-integration-test.sh diff --git a/docker-compose.portals.yml b/docker-compose.portals.yml index 8b4a6bd04a..d3905fbf90 100644 --- a/docker-compose.portals.yml +++ b/docker-compose.portals.yml @@ -155,3 +155,20 @@ services: timeout: 5s start_period: 30s retries: 5 + + dpc_client: + build: + context: . + dockerfile: engines/api_client/Dockerfile + image: dpc-client:latest + volumes: + # Mount specific directories to avoid overwriting + # precompiled assets (public/assets/) and node_modules + - "./engines/api_client:/api-client" + environment: + # Application settings + - GOLDEN_MACAROON=${GOLDEN_MACAROON} + - API_METADATA_URL=http://api:3002/api/v1 + - API_ADMIN_URL=http://api:9900 + - RUBY_YJIT_ENABLE=1 + - ENV=local diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh new file mode 100755 index 0000000000..a6fe1945b0 --- /dev/null +++ b/dpc-client-integration-test.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -Ee + +function _finally { + # don't shut it down if running on ci + if [ "$ENV" != 'github-ci' ]; then + echo "SHUTTING EVERYTHING DOWN" + docker compose -p client-integration-app down + docker volume rm client-integration-app_pgdata16 + fi +} + +trap _finally EXIT + +# Build the application +# mvn clean compile -Perror-prone -B -V -ntp -T 4 -DskipTests +# mvn package -Pci -ntp -T 4 -DskipTests + +echo "Starting api server for client integration tests" +USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait + +export GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) + +echo "Starting end-to-end tests" +docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --entrypoint "bundle exec rspec --tag type:integration" dpc_client + +# Wait for Jacoco to finish writing the output files +docker compose -p client-integration-app down -t 60 diff --git a/engines/api_client/.rspec b/engines/api_client/.rspec index c99d2e7396..f63b1c606a 100644 --- a/engines/api_client/.rspec +++ b/engines/api_client/.rspec @@ -1 +1,2 @@ --require spec_helper +--order defined \ No newline at end of file diff --git a/engines/api_client/Dockerfile b/engines/api_client/Dockerfile index 08ba06f593..089a371e2a 100644 --- a/engines/api_client/Dockerfile +++ b/engines/api_client/Dockerfile @@ -12,13 +12,13 @@ RUN mkdir /api-client WORKDIR /api-client # Copy over the files needed to fetch dependencies -COPY Gemfile Gemfile.lock api_client.gemspec /api-client/ -COPY lib /api-client/lib +COPY engines/api_client/Gemfile engines/api_client/Gemfile.lock engines/api_client/api_client.gemspec /api-client/ +COPY engines/api_client/lib /api-client/lib RUN gem install bundler --no-document && \ bundle config set force_ruby_platform true && \ bundle install -COPY app /api-client/app -COPY spec /api-client/spec -COPY coverage /api-client/coverage -COPY .rubocop.yml /api-client/.rubocop.yml +COPY engines/api_client/app /api-client/app +COPY engines/api_client/spec /api-client/spec +COPY engines/api_client/coverage /api-client/coverage +COPY engines/api_client/.rubocop.yml /api-client/.rubocop.yml diff --git a/engines/api_client/Dockerfile.local b/engines/api_client/Dockerfile.local new file mode 100644 index 0000000000..08ba06f593 --- /dev/null +++ b/engines/api_client/Dockerfile.local @@ -0,0 +1,24 @@ +FROM artifactory.cloud.cms.gov/docker/ruby:3.3-alpine AS ruby_builder + +# Install build dependencies +RUN apk update && \ + apk add --no-cache libffi-dev && \ + apk add --no-cache yaml-dev && \ + apk add --no-cache --virtual build-deps alpine-sdk tzdata && \ + apk add --no-cache gcompat + +# Set the working directory +RUN mkdir /api-client +WORKDIR /api-client + +# Copy over the files needed to fetch dependencies +COPY Gemfile Gemfile.lock api_client.gemspec /api-client/ +COPY lib /api-client/lib +RUN gem install bundler --no-document && \ + bundle config set force_ruby_platform true && \ + bundle install + +COPY app /api-client/app +COPY spec /api-client/spec +COPY coverage /api-client/coverage +COPY .rubocop.yml /api-client/.rubocop.yml diff --git a/engines/api_client/Makefile b/engines/api_client/Makefile index cd367607fe..5a43db5be1 100644 --- a/engines/api_client/Makefile +++ b/engines/api_client/Makefile @@ -1,5 +1,5 @@ build: - docker build . -t api_client + docker build -f Dockerfile.local . -t api_client test: docker run --rm -v ${PWD}:/api-client -w /api-client api_client bundle exec rspec docker run --rm -v ${PWD}:/api-client -w /api-client api_client bundle exec rubocop diff --git a/engines/api_client/spec/integration/dpc_client_spec.rb b/engines/api_client/spec/integration/dpc_client_spec.rb new file mode 100644 index 0000000000..73fb80f6af --- /dev/null +++ b/engines/api_client/spec/integration/dpc_client_spec.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe DpcClient, type: :integration do + let(:client) { DpcClient.new } + let(:npi) { '1111111112' } + let!(:org) do + double('organization', + npi:, name: 'Org 2', address_use: 'work', address_type: 'both', + address_city: 'Akron', address_state: 'OH', address_street: '111 Main ST', 'address_street_2' => 'STE 5', + id: '8453e48b-0b42-4ddf-8b43-07c7aa2a3d88', + address_zip: '22222') + end + let!(:reg_org) do + double('RegisteredOrg', api_id: 'some-api-key') + end + + before(:all) do + WebMock.disable! + end + after(:all) do + WebMock.enable! + end + + describe '#create_organization' do + context 'successful API request' do + it 'sends data to API and sets response instance variables' do + client.create_organization(org) + + expect(client.response_status).to eq(200) + expect(client.response_body.dig('identifier', 0, 'value')).to eq npi + end + end + end + + describe '#get_organization_by_npi' do + context 'successful API request' do + it 'retrieves organization data from API' do + response = client.get_organization_by_npi(npi) + expect(response&.entry).to_not be_nil + expect(response.entry.length).to eq 1 + expect(response.entry.first.resource.identifier.first.value).to eq npi + end + end + end + + describe '#post_create' do + let(:org_id) do + response = client.get_organization_by_npi(npi) + response.entry.first.resource.id + end + + describe '#get_organization' do + context 'successful API request' do + it 'retrieves organization data from API' do + response = client.get_organization(org_id) + expect(response).to_not be_nil + expect(response.resourceType).to eq 'Organization' + end + end + end + + describe '#update_organization' do + context 'successful request' do + it 'sends org data to API' do + expect(client.update_organization(org, org_id)).to eq(client) + expect(client.response_successful?).to eq(true) + end + end + end + + describe '#create_client_token' do + context 'successful API request' do + it 'sends data to API and sets response instance variables' do + client.create_client_token(org_id, params: { label: 'Sandbox Token 1' }) + + expect(client.response_status).to eq(200) + expect(client.response_body['label']).to eq('Sandbox Token 1') + end + end + end + + describe '#get_client_tokens' do + context 'successful API request' do + it 'sends data to API and sets response instance variables' do + client.get_client_tokens(org_id) + + expect(client.response_status).to eq(200) + expect(client.response_body['count']).to be > 0 + expect(client.response_body['entities'].first['label']).to eq('Sandbox Token 1') + end + end + end + + describe '#delete_client_token' do + context 'successful API request' do + it 'returns success' do + client.get_client_tokens(org_id) + expect(client.response_status).to eq(200) + start_count = client.response_body['count'] + expect(start_count).to be > 0 + token_id = client.response_body['entities'].first['id'] + client.delete_client_token(org_id, token_id) + expect(client.response_status).to eq(204) + client.get_client_tokens(org_id) + expect(client.response_status).to eq(200) + expect(client.response_body['count'] + 1).to eq start_count + end + end + end + + describe '#get_public_keys' do + context 'successful API request' do + it 'sends data to API and sets response instance variables' do + client.get_public_keys(org_id) + + expect(client.response_status).to eq(200) + end + end + end + + describe '#create_ip_address' do + context 'successful API request' do + it 'sends data to API and sets response instance variables' do + client.create_ip_address(org_id, params: { label: 'Sandbox IP 1', ip_address: '136.226.19.87' }) + expect(client.response_status).to eq(200) + expect(client.response_body['label']).to eq 'Sandbox IP 1' + end + end + end + + describe '#get_ip_addresses' do + context 'successful API request' do + it 'sends data to API and sets response instance variables' do + client.get_ip_addresses(org_id) + expect(client.response_status).to eq(200) + expect(client.response_body['count']).to be > 0 + expect(client.response_body['entities'].first['label']).to eq('Sandbox IP 1') + end + end + end + + describe '#delete_ip_address' do + context 'successful API request' do + it 'sends data to API and sets response instance variables' do + client.get_ip_addresses(org_id) + expect(client.response_status).to eq(200) + start_count = client.response_body['count'] + expect(start_count).to be > 0 + ip_address_id = client.response_body['entities'].first['id'] + client.delete_ip_address( + org_id, + ip_address_id + ) + expect(client.response_status).to eq(204) + client.get_ip_addresses(org_id) + expect(client.response_status).to eq(200) + expect(client.response_body['count'] + 1).to eq start_count + end + end + end + end + + def stubbed_key + file_fixture('stubbed_key.pem').read + end +end diff --git a/engines/api_client/spec/spec_helper.rb b/engines/api_client/spec/spec_helper.rb index fb1b33ad55..7ef5e7ce14 100644 --- a/engines/api_client/spec/spec_helper.rb +++ b/engines/api_client/spec/spec_helper.rb @@ -16,6 +16,7 @@ expectations.include_chain_clauses_in_custom_matcher_descriptions = true end + config.filter_run_excluding type: :integration config.mock_with :rspec do |mocks| # Prevents you from mocking or stubbing a method that does not exist on # a real object. This is generally recommended, and will default to From 5ac64ffa11e19dce3add09bd6813174995e2929c Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 09:41:27 -0500 Subject: [PATCH 02/28] add public key tests --- .../spec/integration/dpc_client_spec.rb | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/engines/api_client/spec/integration/dpc_client_spec.rb b/engines/api_client/spec/integration/dpc_client_spec.rb index 73fb80f6af..ed74b7273a 100644 --- a/engines/api_client/spec/integration/dpc_client_spec.rb +++ b/engines/api_client/spec/integration/dpc_client_spec.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'base64' +require 'openssl' require 'rails_helper' RSpec.describe DpcClient, type: :integration do @@ -110,12 +112,63 @@ end end + describe '#create_public_key' do + context 'successful API request' do + it 'sends data to API and sets response instance variables' do + rsa_key = OpenSSL::PKey::RSA.new(4096) + + rsa_key.to_pem + + public_key = rsa_key.public_key.to_pem + + message = 'This is the snippet used to verify a key pair in DPC.' + + digest = OpenSSL::Digest.new('SHA256') + + signature_binary = rsa_key.sign(digest, message) + + snippet_signature = Base64.encode64(signature_binary) + + label = 'Sandbox Key 1' + client.create_public_key( + org_id, + params: { label:, public_key:, snippet_signature: } + ) + + expect(client.response_status).to eq(200) + expect(client.response_body['label']).to eq label + end + end + end + describe '#get_public_keys' do context 'successful API request' do it 'sends data to API and sets response instance variables' do client.get_public_keys(org_id) expect(client.response_status).to eq(200) + expect(client.response_body['count']).to be > 0 + expect(client.response_body['entities'].first['label']).to eq('Sandbox Key 1') + end + end + end + + describe '#delete_public_key' do + context 'successful API request' do + it 'sends data to API and sets response instance variables' do + client.get_public_keys(org_id) + expect(client.response_status).to eq(200) + start_count = client.response_body['count'] + expect(start_count).to be > 0 + public_key_id = client.response_body['entities'].first['id'] + client.delete_public_key( + org_id, + public_key_id + ) + expect(client.response_status).to eq(200) + client.get_public_keys(org_id) + expect(client.response_status).to eq(200) + expect(client.response_body['count'] + 1).to eq start_count end end end From ebc47098687f674d9c72cb79f1884fbd1bbcac03 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 09:51:19 -0500 Subject: [PATCH 03/28] skip simple cov in integration test --- docker-compose.portals.yml | 1 + dpc-client-integration-test.sh | 2 +- engines/api_client/spec/spec_helper.rb | 15 +++++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docker-compose.portals.yml b/docker-compose.portals.yml index d3905fbf90..2958d57515 100644 --- a/docker-compose.portals.yml +++ b/docker-compose.portals.yml @@ -171,4 +171,5 @@ services: - API_METADATA_URL=http://api:3002/api/v1 - API_ADMIN_URL=http://api:9900 - RUBY_YJIT_ENABLE=1 + - SKIP_SIMPLE_COV=${SKIP_SIMPLE_COV} - ENV=local diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh index a6fe1945b0..ed7278ca17 100755 --- a/dpc-client-integration-test.sh +++ b/dpc-client-integration-test.sh @@ -22,7 +22,7 @@ USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait export GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) echo "Starting end-to-end tests" -docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --entrypoint "bundle exec rspec --tag type:integration" dpc_client +SKIP_SIMPLE_COV=true docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --entrypoint "bundle exec rspec --tag type:integration" dpc_client # Wait for Jacoco to finish writing the output files docker compose -p client-integration-app down -t 60 diff --git a/engines/api_client/spec/spec_helper.rb b/engines/api_client/spec/spec_helper.rb index 7ef5e7ce14..ce3e6d1820 100644 --- a/engines/api_client/spec/spec_helper.rb +++ b/engines/api_client/spec/spec_helper.rb @@ -3,12 +3,15 @@ require 'simplecov' require 'webmock/rspec' -SimpleCov.start do - track_files '**/{app,lib}/**/*.rb' - add_filter 'lib/api_client/version.rb' - add_filter %r{/dummy/} - SimpleCov.minimum_coverage 80 - SimpleCov.minimum_coverage_by_file 0 + +unless ENV['SKIP_SIMPLE_COV'] == 'true' + SimpleCov.start do + track_files '**/{app,lib}/**/*.rb' + add_filter 'lib/api_client/version.rb' + add_filter %r{/dummy/} + SimpleCov.minimum_coverage 80 + SimpleCov.minimum_coverage_by_file 0 + end end RSpec.configure do |config| From e51c6d7081221c2546af7ea3d9b7d9e40a596c92 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 09:52:21 -0500 Subject: [PATCH 04/28] run integration tests in ci --- .github/workflows/ci-workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 048e3e7c74..30e66fceff 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -207,6 +207,7 @@ jobs: - name: "DPC Client Build" run: | make ci-api-client + make api-client-integration-tests - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh From 704498b72e71f71690049c3b2b2e4de797b132c4 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 09:53:07 -0500 Subject: [PATCH 05/28] remove other ci tests for quick check --- .github/workflows/ci-workflow.yml | 319 ------------------------------ 1 file changed, 319 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 30e66fceff..7547d68f9d 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -37,163 +37,6 @@ env: ENV: "github-ci" jobs: - build-api: - name: "Build and Test API" - runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} - steps: - - name: Assert Ownership - run: sudo chmod -R 777 . - - name: "Checkout code" - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - - name: Cleanup Runner - run: ./scripts/cleanup-docker.sh - - name: "Set up JDK 17" - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 - with: - java-version: "17" - distribution: "corretto" - cache: maven - - name: Clean maven - run: mvn -ntp -U clean - - name: "Set up Python and install Ansible" - run: | - sudo dnf -y install python3 python3-pip - pip install ansible - - name: Install docker compose manually - run: | - mkdir -p /usr/local/lib/docker/cli-plugins - curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose - chown root:root /usr/local/lib/docker/cli-plugins/docker-compose - chmod +x /usr/local/lib/docker/cli-plugins/docker-compose - - name: "API Build" - id: ci-app - run: | - export PATH=$PATH:~/.local/bin - make ci-app - - name: "Debug db" - if: ${{ failure() && steps.ci-app.outcome == 'failure' }} - run: docker logs start-v1-app-db-1 - - name: "Debug attribution" - if: ${{ failure() && steps.ci-app.outcome == 'failure' }} - run: docker logs start-v1-app-attribution-1 - - name: "Debug aggregation" - if: ${{ failure() && steps.ci-app.outcome == 'failure' }} - run: docker logs start-v1-app-aggregation-1 - - name: "Debug api" - if: ${{ failure() && steps.ci-app.outcome == 'failure' }} - run: docker logs start-v1-app-api-1 - - name: "Move jacoco reports" - run: | - sudo mkdir jacoco-reports - sudo cp ./dpc-aggregation/target/site/jacoco-it/jacoco.xml jacoco-reports/dpc-aggregation-it-jacoco.xml - sudo cp ./dpc-aggregation/target/site/jacoco/jacoco.xml jacoco-reports/dpc-aggregation-jacoco.xml - sudo cp ./dpc-api/target/site/jacoco-it/jacoco.xml jacoco-reports/dpc-api-it-jacoco.xml - sudo cp ./dpc-api/target/site/jacoco/jacoco.xml jacoco-reports/dpc-api-jacoco.xml - sudo cp ./dpc-attribution/target/site/jacoco-it/jacoco.xml jacoco-reports/dpc-attribution-it-jacoco.xml - sudo cp ./dpc-attribution/target/site/jacoco/jacoco.xml jacoco-reports/dpc-attribution-jacoco.xml - sudo cp ./dpc-bluebutton/target/site/jacoco/jacoco.xml jacoco-reports/dpc-bluebutton-jacoco.xml - sudo cp ./dpc-common/target/site/jacoco/jacoco.xml jacoco-reports/dpc-common-jacoco.xml - sudo cp ./dpc-macaroons/target/site/jacoco/jacoco.xml jacoco-reports/dpc-macaroons-jacoco.xml - sudo cp ./dpc-queue/target/site/jacoco/jacoco.xml jacoco-reports/dpc-queue-jacoco.xml - - name: Upload jacoco reports - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: code-coverage-report-dpc-api - path: ./jacoco-reports - - name: Cleanup - if: ${{ always() }} - run: ./scripts/cleanup-docker.sh - - build-dpc-web: - name: "Build and Test DPC Web" - runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} - steps: - - name: Assert Ownership - run: sudo chmod -R 777 . - - name: "Checkout code" - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - - name: Cleanup Runner - run: ./scripts/cleanup-docker.sh - - name: Install docker compose manually - run: | - mkdir -p /usr/local/lib/docker/cli-plugins - curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose - chown root:root /usr/local/lib/docker/cli-plugins/docker-compose - chmod +x /usr/local/lib/docker/cli-plugins/docker-compose - - name: "DPC Web Build" - run: | - make ci-web-portal - - name: "Copy test results" - run: sudo cp dpc-web/coverage/.resultset.json web-resultset-raw.json - - name: Archive code coverage results - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: code-coverage-report-dpc-web - path: ./web-resultset-raw.json - - name: Cleanup - if: ${{ always() }} - run: ./scripts/cleanup-docker.sh - - build-dpc-admin: - name: "Build and Test DPC Admin Portal" - runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} - steps: - - name: Assert Ownership - run: sudo chmod -R 777 . - - name: "Checkout code" - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - - name: Cleanup Runner - run: ./scripts/cleanup-docker.sh - - name: Install docker compose manually - run: | - mkdir -p /usr/local/lib/docker/cli-plugins - curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose - chown root:root /usr/local/lib/docker/cli-plugins/docker-compose - chmod +x /usr/local/lib/docker/cli-plugins/docker-compose - - name: "DPC Admin Portal Build" - run: | - make ci-admin-portal - - name: "Copy test results" - run: sudo cp dpc-admin/coverage/.resultset.json admin-resultset-raw.json - - name: Archive code coverage results - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: code-coverage-report-dpc-admin - path: ./admin-resultset-raw.json - - name: Cleanup - if: ${{ always() }} - run: ./scripts/cleanup-docker.sh - - build-dpc-portal: - name: "Build and Test DPC Portal" - runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} - steps: - - name: Assert Ownership - run: sudo chmod -R 777 . - - name: "Checkout code" - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - - name: Cleanup Runner - run: ./scripts/cleanup-docker.sh - - name: Install docker compose manually - run: | - mkdir -p /usr/local/lib/docker/cli-plugins - curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose - chown root:root /usr/local/lib/docker/cli-plugins/docker-compose - chmod +x /usr/local/lib/docker/cli-plugins/docker-compose - - name: "DPC Portal Build" - run: | - make ci-portal - - name: "Copy test results" - run: sudo cp dpc-portal/coverage/.resultset.json portal-resultset-raw.json - - name: Archive code coverage results - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: code-coverage-report-dpc-portal - path: ./portal-resultset-raw.json - - name: Cleanup - if: ${{ always() }} - run: ./scripts/cleanup-docker.sh - build-dpc-client: name: "Build and Test DPC Client" runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} @@ -211,165 +54,3 @@ jobs: - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh - - sonar-quality-gate-dpc-web-and-admin: - name: Sonarqube Quality Gate for dpc-web and dpc-admin - needs: [build-dpc-admin, build-dpc-web] - runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} - env: - # Workaround until https://jira.cms.gov/browse/PLT-338 is implemented. - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" - steps: - - name: Assert Ownership - run: sudo chmod -R 777 . - - name: "Checkout code" - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - with: - fetch-depth: 0 - - name: Cleanup Runner - run: ./scripts/cleanup-docker.sh - - name: Download web code coverage - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: code-coverage-report-dpc-web - - name: Download admin code coverage - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: code-coverage-report-dpc-admin - - name: "Reformat test results" # Sonarqube will run in a docker container and wants the paths to be from /github/workspace - run: | - sudo jq '.RSpec.coverage |= with_entries(if .key | contains("dpc-web") then .key |= sub("/dpc-web"; "${{ github.workspace }}/dpc-web") else . end)' web-resultset-raw.json > web-resultset.json - sudo jq '.RSpec.coverage |= with_entries(if .key | contains("dpc-admin") then .key |= sub("/dpc-admin"; "${{ github.workspace }}/dpc-admin") else . end)' admin-resultset-raw.json > admin-resultset.json - - name: Set env vars from AWS params - uses: cmsgov/cdap/actions/aws-params-env-action@main - env: - AWS_REGION: ${{ vars.AWS_REGION }} - with: - params: | - SONAR_HOST_URL=/sonarqube/url - SONAR_TOKEN=/sonarqube/token - - name: Run quality gate scan - uses: sonarsource/sonarqube-scan-action@fd88b7d7ccbaefd23d8f36f73b59db7a3d246602 # v6.0.0 - with: - args: - -Dsonar.projectKey=bcda-dpc-web - -Dsonar.sources=./dpc-web/app,./dpc-web/lib,./dpc-admin/app,./dpc-admin/lib - -Dsonar.ruby.coverage.reportPaths=./web-resultset.json,./admin-resultset.json - -Dsonar.working.directory=./sonar_workspace - -Dsonar.branch.name=${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }} - -Dsonar.projectVersion=${{ github.ref_name == 'main' && github.sha || 'branch' }} - -Dsonar.qualitygate.wait=true - -Dsonar.ci.autoconfig.disabled=true - -Dsonar.branch.target=${{ github.base_ref }} - - name: Cleanup - if: ${{ always() }} - run: ./scripts/cleanup-docker.sh - - sonar-quality-gate-dpc-portal: - name: Sonarqube Quality Gate for dpc-portal - needs: build-dpc-portal - runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} - env: - # Workaround until https://jira.cms.gov/browse/PLT-338 is implemented. - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" - steps: - - name: Assert Ownership - run: sudo chmod -R 777 . - - name: "Checkout code" - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - with: - fetch-depth: 0 - - name: Cleanup Runner - run: ./scripts/cleanup-docker.sh - - name: Download code coverage - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: code-coverage-report-dpc-portal - - name: "Reformat test results" # Sonarqube will run in a docker container and wants the paths to be from /github/workspace - run: | - sudo jq '.RSpec.coverage |= with_entries(if .key | contains("dpc-portal") then .key |= sub("/dpc-portal"; "${{ github.workspace }}/dpc-portal") else . end)' portal-resultset-raw.json > portal-resultset.json - - name: Set env vars from AWS params - uses: cmsgov/cdap/actions/aws-params-env-action@main - env: - AWS_REGION: ${{ vars.AWS_REGION }} - with: - params: | - SONAR_HOST_URL=/sonarqube/url - SONAR_TOKEN=/sonarqube/token - - name: Run quality gate scan - uses: sonarsource/sonarqube-scan-action@fd88b7d7ccbaefd23d8f36f73b59db7a3d246602 # v6.0.0 - with: - args: - -Dsonar.projectKey=bcda-dpc-portal - -Dsonar.sources=./dpc-portal/app,./dpc-portal/lib - -Dsonar.coverage.exclusions=**/*_preview.rb,**/*html.erb,**/application_* - -Dsonar.ruby.coverage.reportPaths=./portal-resultset.json - -Dsonar.working.directory=./sonar_workspace - -Dsonar.branch.name=${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }} - -Dsonar.projectVersion=${{ github.ref_name == 'main' && github.sha || 'branch' }} - -Dsonar.qualitygate.wait=true - -Dsonar.ci.autoconfig.disabled=true - -Dsonar.branch.target=${{ github.base_ref }} - - name: Cleanup - if: ${{ always() }} - run: ./scripts/cleanup-docker.sh - - sonar-quality-gate-dpc-api: - name: Sonarqube Quality Gate for dpc-api - needs: build-api - runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} - env: - # Workaround until https://jira.cms.gov/browse/PLT-338 is implemented. - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - steps: - - name: Assert Ownership - run: sudo chmod -R 777 . - - name: Checkout Code - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - with: - fetch-depth: 0 - - name: Cleanup Runner - run: ./scripts/cleanup-docker.sh - - name: Setup Java - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 - with: - java-version: '17' - distribution: temurin - cache: maven - - name: Set env vars from AWS params - uses: cmsgov/cdap/actions/aws-params-env-action@main - env: - AWS_REGION: ${{ vars.AWS_REGION }} - with: - params: | - SONAR_HOST_URL=/sonarqube/url - SONAR_TOKEN=/sonarqube/token - - name: Install Maven 3.6.3 - run: | - export PATH="$PATH:/opt/maven/bin" - echo "PATH=$PATH" >> $GITHUB_ENV - if mvn -v; then echo "Maven already installed" && exit 0; else echo "Installing Maven"; fi - tmpdir="$(mktemp -d)" - curl -LsS https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz | tar xzf - -C "$tmpdir" - sudo rm -rf /opt/maven - sudo mv "$tmpdir/apache-maven-3.6.3" /opt/maven - - name: Clean maven - run: | - mvn -ntp -U clean - - name: Compile Project - run: | - mvn clean compile -Perror-prone -B -V -ntp - - name: Download code coverage - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: code-coverage-report-dpc-api - path: jacoco-reports - - name: Verify download - run: | - find . -name dpc-api-jacoco.xml - - name: Run quality gate scan - run: | - mvn org.sonarsource.scanner.maven:sonar-maven-plugin:4.0.0.4121:sonar -Dsonar.projectKey=bcda-dpc-api -Dsonar.branch.name=${{ github.event_name == 'pull_request' && github.head_ref || github.event_name == 'pull_request' && github.head_ref || github.ref_name }} -Dsonar.working.directory=./.sonar_workspace -Dsonar.projectVersion=${{ github.ref_name == 'main' && github.sha || 'branch' }} -Dsonar.qualitygate.wait=true -Dsonar.coverage.jacoco.xmlReportPaths="../jacoco-reports/*.xml" -Dsonar.ci.autoconfig.disabled=true -Dsonar.branch.target=${{ github.base_ref }} - - name: Cleanup - if: ${{ always() }} - run: ./scripts/cleanup-docker.sh From 85ec33e4773168ecbee49f4746cb64d300e02937 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 09:57:34 -0500 Subject: [PATCH 06/28] fix ci-api-test --- dpc-api-client-test.sh | 2 +- engines/api_client/spec/spec_helper.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dpc-api-client-test.sh b/dpc-api-client-test.sh index 28ae35cb51..c2066abce3 100755 --- a/dpc-api-client-test.sh +++ b/dpc-api-client-test.sh @@ -8,7 +8,7 @@ echo "│ │" echo "└───────────────────────┘" # Build the container -docker build -f engines/api_client/Dockerfile engines/api_client -t api-client +docker build -f engines/api_client/Dockerfile.local engines/api_client -t api-client # Run the tests echo "┌───────────────────────────┐" diff --git a/engines/api_client/spec/spec_helper.rb b/engines/api_client/spec/spec_helper.rb index ce3e6d1820..991cfce00f 100644 --- a/engines/api_client/spec/spec_helper.rb +++ b/engines/api_client/spec/spec_helper.rb @@ -3,7 +3,6 @@ require 'simplecov' require 'webmock/rspec' - unless ENV['SKIP_SIMPLE_COV'] == 'true' SimpleCov.start do track_files '**/{app,lib}/**/*.rb' From f3343d9ae8c4327dbbc4f37e3bff30c4c6f8215c Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 10:48:53 -0500 Subject: [PATCH 07/28] consolidate dpc-client build --- .github/workflows/ci-workflow.yml | 2 +- Makefile | 8 ++++++-- dpc-api-client-test.sh | 6 +++--- dpc-client-integration-test | 28 ++++++++++++++++++++++++++++ dpc-client-integration-test.sh | 20 +++++++------------- 5 files changed, 45 insertions(+), 19 deletions(-) create mode 100755 dpc-client-integration-test diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 7547d68f9d..19bbe8cfda 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -50,7 +50,7 @@ jobs: - name: "DPC Client Build" run: | make ci-api-client - make api-client-integration-tests + make api-client-integration-test - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh diff --git a/Makefile b/Makefile index 5fdd5ce269..8d6151a863 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,8 @@ portal: cp -r engines/api_client/ dpc-portal/vendor/api_client/ @docker build -f dpc-portal/Dockerfile . -t dpc-web-portal +dpc-client: + @docker compose -f docker-compose.yml -f docker-compose.portals.yml build dpc_client # Start commands # ============== @@ -181,6 +183,8 @@ portal-sh: ## Run a portal shell portal-console: ## Run a rails console shell @docker compose -f docker-compose.yml -f docker-compose.portals.yml exec -it dpc_portal bin/console +dpc-client-sh: + @docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --remove-orphans --entrypoint "sh" dpc_client # Build & Test commands # ====================== @@ -232,6 +236,6 @@ unit-tests: .PHONY: load-tests load-tests: start-api-load-tests start-load-tests down-dpc-load-tests -.PHONY: api-client-integration-tests -api-client-integration-tests: clean-jacoco docker-base api +.PHONY: dpc-client-integration-test +dpc-client-integration-test: clean-jacoco docker-base api @bash ./dpc-client-integration-test.sh diff --git a/dpc-api-client-test.sh b/dpc-api-client-test.sh index c2066abce3..1113114d00 100755 --- a/dpc-api-client-test.sh +++ b/dpc-api-client-test.sh @@ -8,7 +8,7 @@ echo "│ │" echo "└───────────────────────┘" # Build the container -docker build -f engines/api_client/Dockerfile.local engines/api_client -t api-client +docker compose -f docker-compose.yml -f docker-compose.portals.yml build dpc_client # Run the tests echo "┌───────────────────────────┐" @@ -16,8 +16,8 @@ echo "│ │" echo "│ Running Api Gem Tests │" echo "│ │" echo "└───────────────────────────┘" -docker run --rm -v ${PWD}/engines/api_client/coverage:/api-client/coverage api-client bundle exec rspec -docker run --rm api-client bundle exec rubocop +docker run --rm -v ${PWD}/engines/api_client/coverage:/api-client/coverage dpc-client bundle exec rspec +docker run --rm dpc-client bundle exec rubocop echo "┌────────────────────────────────┐" echo "│ │" diff --git a/dpc-client-integration-test b/dpc-client-integration-test new file mode 100755 index 0000000000..ed7278ca17 --- /dev/null +++ b/dpc-client-integration-test @@ -0,0 +1,28 @@ +#!/bin/bash +set -Ee + +function _finally { + # don't shut it down if running on ci + if [ "$ENV" != 'github-ci' ]; then + echo "SHUTTING EVERYTHING DOWN" + docker compose -p client-integration-app down + docker volume rm client-integration-app_pgdata16 + fi +} + +trap _finally EXIT + +# Build the application +# mvn clean compile -Perror-prone -B -V -ntp -T 4 -DskipTests +# mvn package -Pci -ntp -T 4 -DskipTests + +echo "Starting api server for client integration tests" +USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait + +export GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) + +echo "Starting end-to-end tests" +SKIP_SIMPLE_COV=true docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --entrypoint "bundle exec rspec --tag type:integration" dpc_client + +# Wait for Jacoco to finish writing the output files +docker compose -p client-integration-app down -t 60 diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh index ed7278ca17..6e3b8d5d5c 100755 --- a/dpc-client-integration-test.sh +++ b/dpc-client-integration-test.sh @@ -2,12 +2,9 @@ set -Ee function _finally { - # don't shut it down if running on ci - if [ "$ENV" != 'github-ci' ]; then - echo "SHUTTING EVERYTHING DOWN" - docker compose -p client-integration-app down - docker volume rm client-integration-app_pgdata16 - fi + echo "SHUTTING EVERYTHING DOWN" + docker compose -p client-integration-app down + docker volume rm client-integration-app_pgdata16 } trap _finally EXIT @@ -19,10 +16,7 @@ trap _finally EXIT echo "Starting api server for client integration tests" USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait -export GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) - -echo "Starting end-to-end tests" -SKIP_SIMPLE_COV=true docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --entrypoint "bundle exec rspec --tag type:integration" dpc_client - -# Wait for Jacoco to finish writing the output files -docker compose -p client-integration-app down -t 60 +echo "Starting integration tests" +GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) \ +SKIP_SIMPLE_COV=true \ +docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --remove-orphans --entrypoint "bundle exec rspec --tag type:integration" dpc_client From 84cca8434496b3d48921ee5ce7e416d0b8114f80 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 10:55:28 -0500 Subject: [PATCH 08/28] typo --- .github/workflows/ci-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 19bbe8cfda..3b405098ba 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -50,7 +50,7 @@ jobs: - name: "DPC Client Build" run: | make ci-api-client - make api-client-integration-test + make dpc-client-integration-test - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh From a8ed34ea73cb62d3fa779bfe8ade38ea1cfe86f1 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 11:08:58 -0500 Subject: [PATCH 09/28] debug attribution not starting --- .github/workflows/ci-workflow.yml | 3 ++- dpc-client-integration-test.sh | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 3b405098ba..b1e87a05bc 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -49,8 +49,9 @@ jobs: run: ./scripts/cleanup-docker.sh - name: "DPC Client Build" run: | - make ci-api-client make dpc-client-integration-test + - name: "Debug attribution" + run: docker logs client-integration-app-attribution-1 - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh index 6e3b8d5d5c..bebc9560d7 100755 --- a/dpc-client-integration-test.sh +++ b/dpc-client-integration-test.sh @@ -2,9 +2,12 @@ set -Ee function _finally { - echo "SHUTTING EVERYTHING DOWN" - docker compose -p client-integration-app down - docker volume rm client-integration-app_pgdata16 + # don't shut it down if running on ci + if [ "$ENV" != 'github-ci' ]; then + echo "SHUTTING EVERYTHING DOWN" + docker compose -p client-integration-app down + docker volume rm client-integration-app_pgdata16 + fi } trap _finally EXIT From 4b8406cb5c411f795f18f057aac7abc90b40cb0d Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 11:22:09 -0500 Subject: [PATCH 10/28] force debug --- .github/workflows/ci-workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index b1e87a05bc..78705697a5 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -51,6 +51,7 @@ jobs: run: | make dpc-client-integration-test - name: "Debug attribution" + if: ${{ always() }} run: docker logs client-integration-app-attribution-1 - name: Cleanup if: ${{ always() }} From 1104b36cb7fcde2e0e8e9f76e4788aa2c94d49b7 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 11:58:53 -0500 Subject: [PATCH 11/28] explicitly turn off jacoco --- dpc-client-integration-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpc-client-integration-test b/dpc-client-integration-test index ed7278ca17..b0781aee3a 100755 --- a/dpc-client-integration-test +++ b/dpc-client-integration-test @@ -17,7 +17,7 @@ trap _finally EXIT # mvn package -Pci -ntp -T 4 -DskipTests echo "Starting api server for client integration tests" -USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait +JACOCO=false USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait export GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) From eb616d7bdc72eaa29a935a15dd84391e2fa64e18 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:47:36 -0500 Subject: [PATCH 12/28] unset report coverage --- dpc-client-integration-test | 28 ---------------------------- dpc-client-integration-test.sh | 1 + 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100755 dpc-client-integration-test diff --git a/dpc-client-integration-test b/dpc-client-integration-test deleted file mode 100755 index b0781aee3a..0000000000 --- a/dpc-client-integration-test +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -Ee - -function _finally { - # don't shut it down if running on ci - if [ "$ENV" != 'github-ci' ]; then - echo "SHUTTING EVERYTHING DOWN" - docker compose -p client-integration-app down - docker volume rm client-integration-app_pgdata16 - fi -} - -trap _finally EXIT - -# Build the application -# mvn clean compile -Perror-prone -B -V -ntp -T 4 -DskipTests -# mvn package -Pci -ntp -T 4 -DskipTests - -echo "Starting api server for client integration tests" -JACOCO=false USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait - -export GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) - -echo "Starting end-to-end tests" -SKIP_SIMPLE_COV=true docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --entrypoint "bundle exec rspec --tag type:integration" dpc_client - -# Wait for Jacoco to finish writing the output files -docker compose -p client-integration-app down -t 60 diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh index bebc9560d7..9a73312aa8 100755 --- a/dpc-client-integration-test.sh +++ b/dpc-client-integration-test.sh @@ -16,6 +16,7 @@ trap _finally EXIT # mvn clean compile -Perror-prone -B -V -ntp -T 4 -DskipTests # mvn package -Pci -ntp -T 4 -DskipTests +unset REPORT_COVERAGE echo "Starting api server for client integration tests" USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait From 82c43526204e62be575aa03f2a7cafbc9f3bd185 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:35:41 -0500 Subject: [PATCH 13/28] allow jacoco to be false --- docker-compose.yml | 6 +++--- dpc-attribution/docker/entrypoint.sh | 3 +++ dpc-client-integration-test.sh | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3fb29356ce..4f38bd496d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: - ENV=local - USE_BFD_MOCK=${USE_BFD_MOCK:-true} - EMIT_AWS_METRICS=${EMIT_AWS_METRICS:-false} - - JACOCO=${REPORT_COVERAGE:-false} + - JACOCO=${REPORT_COVERAGE:-} - DEBUG_MODE=${DEBUG_MODE:-false} depends_on: db: @@ -56,7 +56,7 @@ services: condition: service_healthy environment: - ENV=local - - JACOCO=${REPORT_COVERAGE:-false} + - JACOCO=${REPORT_COVERAGE:-} - DEBUG_MODE=${DEBUG_MODE:-false} ports: - "3500:8080" @@ -79,7 +79,7 @@ services: - ./ops/config/decrypted/local.env environment: - ENV=local - - JACOCO=${REPORT_COVERAGE:-false} + - JACOCO=${REPORT_COVERAGE:-} - EXPORT_PATH=/app/data - AUTH_DISABLED=${AUTH_DISABLED:-false} - DEBUG_MODE=${DEBUG_MODE:-false} diff --git a/dpc-attribution/docker/entrypoint.sh b/dpc-attribution/docker/entrypoint.sh index d3c254879b..f3aa1fcfbe 100755 --- a/dpc-attribution/docker/entrypoint.sh +++ b/dpc-attribution/docker/entrypoint.sh @@ -2,9 +2,12 @@ set -e +echo "JACOCO: ${JACOCO}" if [ -n "$JACOCO" ]; then + echo 'JACOCO SET' JACOCO="-javaagent:/org.jacoco.agent-runtime.jar=destfile=/jacoco-report/jacoco-it.exec" else + echo 'JACOCO NOT SET' JACOCO="" fi diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh index 9a73312aa8..078786b2e5 100755 --- a/dpc-client-integration-test.sh +++ b/dpc-client-integration-test.sh @@ -16,8 +16,8 @@ trap _finally EXIT # mvn clean compile -Perror-prone -B -V -ntp -T 4 -DskipTests # mvn package -Pci -ntp -T 4 -DskipTests -unset REPORT_COVERAGE echo "Starting api server for client integration tests" +unset REPORT_COVERAGE USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait echo "Starting integration tests" From 16f4a3767b43525efda9995a2fe0addd0cbe71b7 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:45:16 -0500 Subject: [PATCH 14/28] now debug api --- .github/workflows/ci-workflow.yml | 3 +++ dpc-api/docker/entrypoint.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 78705697a5..70be9c9550 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -53,6 +53,9 @@ jobs: - name: "Debug attribution" if: ${{ always() }} run: docker logs client-integration-app-attribution-1 + - name: "Debug api" + if: ${{ always() }} + run: docker logs client-integration-app-api-1 - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh diff --git a/dpc-api/docker/entrypoint.sh b/dpc-api/docker/entrypoint.sh index 4be3f3bc1a..08ddf89d35 100755 --- a/dpc-api/docker/entrypoint.sh +++ b/dpc-api/docker/entrypoint.sh @@ -11,9 +11,12 @@ bootstrap_config() { aws s3 sync "s3://$bucket" config/ } +echo "JACOCO: ${JACOCO}" if [ -n "$JACOCO" ]; then + echo 'JACOCO SET' JACOCO="-javaagent:/org.jacoco.agent-runtime.jar=destfile=/jacoco-report/jacoco-it.exec" else + echo 'JACOCO NOT SET' JACOCO="" fi From d2262f69cbe1afbff508e6f84fd555eb64c5ece0 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:59:24 -0500 Subject: [PATCH 15/28] debug --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8d6151a863..289c2ac87d 100644 --- a/Makefile +++ b/Makefile @@ -237,5 +237,5 @@ unit-tests: load-tests: start-api-load-tests start-load-tests down-dpc-load-tests .PHONY: dpc-client-integration-test -dpc-client-integration-test: clean-jacoco docker-base api +dpc-client-integration-test: clean-jacoco docker-base api secure-envs @bash ./dpc-client-integration-test.sh From 455d4a804cb35d09f202f29e05df1d36e99185d8 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 14:20:39 -0500 Subject: [PATCH 16/28] try to build with tests --- Makefile | 2 +- docker-compose.base.yml | 2 +- docker-compose.yml | 6 +++--- dpc-client-integration-test.sh | 30 +++++++++++++++++++++++++++--- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 289c2ac87d..841a8ebe04 100644 --- a/Makefile +++ b/Makefile @@ -237,5 +237,5 @@ unit-tests: load-tests: start-api-load-tests start-load-tests down-dpc-load-tests .PHONY: dpc-client-integration-test -dpc-client-integration-test: clean-jacoco docker-base api secure-envs +dpc-client-integration-test: docker-base secure-envs @bash ./dpc-client-integration-test.sh diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 4af782a5dd..e5e63149a3 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -3,6 +3,6 @@ services: image: dpc-base:latest build: args: - PACE_CERT: $PACE_CERT + PACE_CERT: ${PACE_CERT:-} context: . dockerfile: docker/Dockerfiles/Dockerfile.base diff --git a/docker-compose.yml b/docker-compose.yml index 4f38bd496d..3fb29356ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: - ENV=local - USE_BFD_MOCK=${USE_BFD_MOCK:-true} - EMIT_AWS_METRICS=${EMIT_AWS_METRICS:-false} - - JACOCO=${REPORT_COVERAGE:-} + - JACOCO=${REPORT_COVERAGE:-false} - DEBUG_MODE=${DEBUG_MODE:-false} depends_on: db: @@ -56,7 +56,7 @@ services: condition: service_healthy environment: - ENV=local - - JACOCO=${REPORT_COVERAGE:-} + - JACOCO=${REPORT_COVERAGE:-false} - DEBUG_MODE=${DEBUG_MODE:-false} ports: - "3500:8080" @@ -79,7 +79,7 @@ services: - ./ops/config/decrypted/local.env environment: - ENV=local - - JACOCO=${REPORT_COVERAGE:-} + - JACOCO=${REPORT_COVERAGE:-false} - EXPORT_PATH=/app/data - AUTH_DISABLED=${AUTH_DISABLED:-false} - DEBUG_MODE=${DEBUG_MODE:-false} diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh index 078786b2e5..0ecc5382e6 100755 --- a/dpc-client-integration-test.sh +++ b/dpc-client-integration-test.sh @@ -1,6 +1,30 @@ #!/bin/bash set -Ee +# Current working directory +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +# Configure the Maven log level +export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.defaultLogLevel=info + +# Include secure environment variables +set -o allexport +[[ -f ${DIR}/ops/config/decrypted/local.env ]] && source ${DIR}/ops/config/decrypted/local.env +set +o allexport + +# Remove jacocoReport directory +if [ -d "${DIR}/jacocoReport" ]; then + rm -r "${DIR}/jacocoReport" +fi + +# Create jacocoReport and make accessible for writing output from containers if we're running tests +if [ "$ENV" = 'local' ] || [ "$ENV" = 'github-ci' ]; then + mkdir -p "${DIR}"/jacocoReport/dpc-api + mkdir -p "${DIR}"/jacocoReport/dpc-attribution + mkdir -p "${DIR}"/jacocoReport/dpc-aggregation + chown -R nobody:nobody "${DIR}"/jacocoReport +fi + function _finally { # don't shut it down if running on ci if [ "$ENV" != 'github-ci' ]; then @@ -13,11 +37,11 @@ function _finally { trap _finally EXIT # Build the application -# mvn clean compile -Perror-prone -B -V -ntp -T 4 -DskipTests -# mvn package -Pci -ntp -T 4 -DskipTests +docker compose -p client-integration-app up db --wait +mvn -T 1.5C clean compile -Perror-prone -B -V -ntp +mvn -T 1.5C package -Pci -ntp echo "Starting api server for client integration tests" -unset REPORT_COVERAGE USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait echo "Starting integration tests" From 0e84b1fc1134e729617d2f77f1392d99dbe00347 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 14:51:32 -0500 Subject: [PATCH 17/28] try starting from ci-app --- .github/workflows/ci-workflow.yml | 43 +++++++++++++++---- dpc-client-integration-test.sh | 2 +- dpc-test.sh | 31 ++----------- .../spec/integration/dpc_client_spec.rb | 5 +-- 4 files changed, 42 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 70be9c9550..112537e752 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -37,8 +37,8 @@ env: ENV: "github-ci" jobs: - build-dpc-client: - name: "Build and Test DPC Client" + build-api: + name: "Build and Test API" runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} steps: - name: Assert Ownership @@ -47,15 +47,42 @@ jobs: uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Cleanup Runner run: ./scripts/cleanup-docker.sh - - name: "DPC Client Build" + - name: "Set up JDK 17" + uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 + with: + java-version: "17" + distribution: "corretto" + cache: maven + - name: Clean maven + run: mvn -ntp -U clean + - name: "Set up Python and install Ansible" run: | - make dpc-client-integration-test + sudo dnf -y install python3 python3-pip + pip install ansible + - name: Install docker compose manually + run: | + mkdir -p /usr/local/lib/docker/cli-plugins + curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose + chown root:root /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + - name: "API Build" + id: ci-app + run: | + export PATH=$PATH:~/.local/bin + make ci-app + - name: "Debug db" + if: ${{ failure() && steps.ci-app.outcome == 'failure' }} + run: docker logs start-v1-app-db-1 - name: "Debug attribution" - if: ${{ always() }} - run: docker logs client-integration-app-attribution-1 + if: ${{ failure() && steps.ci-app.outcome == 'failure' }} + run: docker logs start-v1-app-attribution-1 + - name: "Debug aggregation" + if: ${{ failure() && steps.ci-app.outcome == 'failure' }} + run: docker logs start-v1-app-aggregation-1 - name: "Debug api" - if: ${{ always() }} - run: docker logs client-integration-app-api-1 + if: ${{ failure() && steps.ci-app.outcome == 'failure' }} + run: docker logs start-v1-app-api-1 - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh + diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh index 0ecc5382e6..0e823b5923 100755 --- a/dpc-client-integration-test.sh +++ b/dpc-client-integration-test.sh @@ -39,7 +39,7 @@ trap _finally EXIT # Build the application docker compose -p client-integration-app up db --wait mvn -T 1.5C clean compile -Perror-prone -B -V -ntp -mvn -T 1.5C package -Pci -ntp +mvn -T 1.5C package -Pci -ntp -DskipTests echo "Starting api server for client integration tests" USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait diff --git a/dpc-test.sh b/dpc-test.sh index 0cf19ea465..4f2d666c88 100755 --- a/dpc-test.sh +++ b/dpc-test.sh @@ -56,34 +56,11 @@ docker compose -p start-v1-app up db --wait mvn -T 1.5C clean compile -Perror-prone -B -V -ntp mvn -T 1.5C package -Pci -ntp -# Format the test results -if [ -n "$REPORT_COVERAGE" ]; then - mvn jacoco:report -ntp -fi - docker compose -p start-v1-app down -USE_BFD_MOCK=true docker compose -p start-v1-app up db attribution aggregation --wait - -# Run the integration tests -USE_BFD_MOCK=true docker compose -p start-v1-app up --exit-code-from tests tests - echo "Starting api server for end-to-end tests" USE_BFD_MOCK=true docker compose -p start-v1-app up api --wait - -echo "Starting end-to-end tests" -docker run --rm -v $(pwd)/dpc-load-testing:/src --env-file $(pwd)/ops/config/decrypted/local.env --add-host host.docker.internal=host-gateway -e ENVIRONMENT=local -i grafana/k6 run /src/ci-app.js - -# Wait for Jacoco to finish writing the output files -docker compose -p start-v1-app down -t 60 - -# Collect the coverage reports for the Docker integration tests -if [ -n "$REPORT_COVERAGE" ]; then - mvn jacoco:report-integration -Pci -ntp -fi - -echo "┌──────────────────────────────────────────┐" -echo "│ │" -echo "│ All Tests Complete │" -echo "│ │" -echo "└──────────────────────────────────────────┘" +echo "Starting integration tests" +GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) \ +SKIP_SIMPLE_COV=true \ +docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --remove-orphans --entrypoint "bundle exec rspec --tag type:integration" dpc_client diff --git a/engines/api_client/spec/integration/dpc_client_spec.rb b/engines/api_client/spec/integration/dpc_client_spec.rb index ed74b7273a..0511ffa319 100644 --- a/engines/api_client/spec/integration/dpc_client_spec.rb +++ b/engines/api_client/spec/integration/dpc_client_spec.rb @@ -176,9 +176,9 @@ describe '#create_ip_address' do context 'successful API request' do it 'sends data to API and sets response instance variables' do - client.create_ip_address(org_id, params: { label: 'Sandbox IP 1', ip_address: '136.226.19.87' }) + client.create_ip_address(org_id, params: { ip_address: '136.226.19.87' }) expect(client.response_status).to eq(200) - expect(client.response_body['label']).to eq 'Sandbox IP 1' + expect(client.response_body['id']).to_not be_blank end end end @@ -189,7 +189,6 @@ client.get_ip_addresses(org_id) expect(client.response_status).to eq(200) expect(client.response_body['count']).to be > 0 - expect(client.response_body['entities'].first['label']).to eq('Sandbox IP 1') end end end From 2d25c832bc3b2d3118d6d32314aca7aebb40fcc4 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:23:05 -0500 Subject: [PATCH 18/28] set correct profile --- .github/workflows/ci-workflow.yml | 41 ++++++++++++++----------------- Makefile | 2 +- docker-compose.yml | 6 ++--- dpc-client-integration-test.sh | 14 ++--------- dpc-test.sh | 31 ++++++++++++++++++++--- 5 files changed, 52 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 112537e752..49a5b1a6cc 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -32,13 +32,12 @@ concurrency: env: VAULT_PW: ${{ secrets.VAULT_PW }} - REPORT_COVERAGE: true DPC_CA_CERT: ${{ secrets.DPC_CA_CERT }} ENV: "github-ci" jobs: - build-api: - name: "Build and Test API" + build-dpc-client: + name: "Build and Test DPC Client" runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} steps: - name: Assert Ownership @@ -47,6 +46,15 @@ jobs: uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Cleanup Runner run: ./scripts/cleanup-docker.sh + - name: Install docker compose manually + run: | + mkdir -p /usr/local/lib/docker/cli-plugins + curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose + chown root:root /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + - name: "Client Unit Tests" + run: | + make ci-api-client - name: "Set up JDK 17" uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 with: @@ -59,29 +67,18 @@ jobs: run: | sudo dnf -y install python3 python3-pip pip install ansible - - name: Install docker compose manually - run: | - mkdir -p /usr/local/lib/docker/cli-plugins - curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose - chown root:root /usr/local/lib/docker/cli-plugins/docker-compose - chmod +x /usr/local/lib/docker/cli-plugins/docker-compose - - name: "API Build" - id: ci-app + - name: "API Integration test" run: | - export PATH=$PATH:~/.local/bin - make ci-app + make ci-api-client-integration-test - name: "Debug db" - if: ${{ failure() && steps.ci-app.outcome == 'failure' }} - run: docker logs start-v1-app-db-1 + if: ${{ failure() }} + run: docker logs client-integration-app-db-1 - name: "Debug attribution" - if: ${{ failure() && steps.ci-app.outcome == 'failure' }} - run: docker logs start-v1-app-attribution-1 - - name: "Debug aggregation" - if: ${{ failure() && steps.ci-app.outcome == 'failure' }} - run: docker logs start-v1-app-aggregation-1 + if: ${{ failure() }} + run: docker logs client-integration-app-attribution-1 - name: "Debug api" - if: ${{ failure() && steps.ci-app.outcome == 'failure' }} - run: docker logs start-v1-app-api-1 + if: ${{ failure() }} + run: docker logs client-integration-app-api-1 - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh diff --git a/Makefile b/Makefile index 841a8ebe04..4447584420 100644 --- a/Makefile +++ b/Makefile @@ -237,5 +237,5 @@ unit-tests: load-tests: start-api-load-tests start-load-tests down-dpc-load-tests .PHONY: dpc-client-integration-test -dpc-client-integration-test: docker-base secure-envs +ci-api-client-integration-test: docker-base secure-envs @bash ./dpc-client-integration-test.sh diff --git a/docker-compose.yml b/docker-compose.yml index 3fb29356ce..4f38bd496d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: - ENV=local - USE_BFD_MOCK=${USE_BFD_MOCK:-true} - EMIT_AWS_METRICS=${EMIT_AWS_METRICS:-false} - - JACOCO=${REPORT_COVERAGE:-false} + - JACOCO=${REPORT_COVERAGE:-} - DEBUG_MODE=${DEBUG_MODE:-false} depends_on: db: @@ -56,7 +56,7 @@ services: condition: service_healthy environment: - ENV=local - - JACOCO=${REPORT_COVERAGE:-false} + - JACOCO=${REPORT_COVERAGE:-} - DEBUG_MODE=${DEBUG_MODE:-false} ports: - "3500:8080" @@ -79,7 +79,7 @@ services: - ./ops/config/decrypted/local.env environment: - ENV=local - - JACOCO=${REPORT_COVERAGE:-false} + - JACOCO=${REPORT_COVERAGE:-} - EXPORT_PATH=/app/data - AUTH_DISABLED=${AUTH_DISABLED:-false} - DEBUG_MODE=${DEBUG_MODE:-false} diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh index 0e823b5923..57488b41f2 100755 --- a/dpc-client-integration-test.sh +++ b/dpc-client-integration-test.sh @@ -17,14 +17,6 @@ if [ -d "${DIR}/jacocoReport" ]; then rm -r "${DIR}/jacocoReport" fi -# Create jacocoReport and make accessible for writing output from containers if we're running tests -if [ "$ENV" = 'local' ] || [ "$ENV" = 'github-ci' ]; then - mkdir -p "${DIR}"/jacocoReport/dpc-api - mkdir -p "${DIR}"/jacocoReport/dpc-attribution - mkdir -p "${DIR}"/jacocoReport/dpc-aggregation - chown -R nobody:nobody "${DIR}"/jacocoReport -fi - function _finally { # don't shut it down if running on ci if [ "$ENV" != 'github-ci' ]; then @@ -37,13 +29,11 @@ function _finally { trap _finally EXIT # Build the application -docker compose -p client-integration-app up db --wait -mvn -T 1.5C clean compile -Perror-prone -B -V -ntp +mvn -T 1.5C clean compile -Perror-prone -B -V -ntp -DskipTests mvn -T 1.5C package -Pci -ntp -DskipTests -echo "Starting api server for client integration tests" +echo "Starting api server for end-to-end tests" USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait - echo "Starting integration tests" GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) \ SKIP_SIMPLE_COV=true \ diff --git a/dpc-test.sh b/dpc-test.sh index 4f2d666c88..0cf19ea465 100755 --- a/dpc-test.sh +++ b/dpc-test.sh @@ -56,11 +56,34 @@ docker compose -p start-v1-app up db --wait mvn -T 1.5C clean compile -Perror-prone -B -V -ntp mvn -T 1.5C package -Pci -ntp +# Format the test results +if [ -n "$REPORT_COVERAGE" ]; then + mvn jacoco:report -ntp +fi + docker compose -p start-v1-app down +USE_BFD_MOCK=true docker compose -p start-v1-app up db attribution aggregation --wait + +# Run the integration tests +USE_BFD_MOCK=true docker compose -p start-v1-app up --exit-code-from tests tests + echo "Starting api server for end-to-end tests" USE_BFD_MOCK=true docker compose -p start-v1-app up api --wait -echo "Starting integration tests" -GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) \ -SKIP_SIMPLE_COV=true \ -docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --remove-orphans --entrypoint "bundle exec rspec --tag type:integration" dpc_client + +echo "Starting end-to-end tests" +docker run --rm -v $(pwd)/dpc-load-testing:/src --env-file $(pwd)/ops/config/decrypted/local.env --add-host host.docker.internal=host-gateway -e ENVIRONMENT=local -i grafana/k6 run /src/ci-app.js + +# Wait for Jacoco to finish writing the output files +docker compose -p start-v1-app down -t 60 + +# Collect the coverage reports for the Docker integration tests +if [ -n "$REPORT_COVERAGE" ]; then + mvn jacoco:report-integration -Pci -ntp +fi + +echo "┌──────────────────────────────────────────┐" +echo "│ │" +echo "│ All Tests Complete │" +echo "│ │" +echo "└──────────────────────────────────────────┘" From 8bacbf77af8f3f9ddeb8c49d9d60ef633ace8e94 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:26:53 -0500 Subject: [PATCH 19/28] add secure envs --- .github/workflows/ci-workflow.yml | 8 ++++---- Makefile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 49a5b1a6cc..bdd0b9c243 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -52,6 +52,10 @@ jobs: curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose chown root:root /usr/local/lib/docker/cli-plugins/docker-compose chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + - name: "Set up Python and install Ansible" + run: | + sudo dnf -y install python3 python3-pip + pip install ansible - name: "Client Unit Tests" run: | make ci-api-client @@ -63,10 +67,6 @@ jobs: cache: maven - name: Clean maven run: mvn -ntp -U clean - - name: "Set up Python and install Ansible" - run: | - sudo dnf -y install python3 python3-pip - pip install ansible - name: "API Integration test" run: | make ci-api-client-integration-test diff --git a/Makefile b/Makefile index 4447584420..618a7b68ab 100644 --- a/Makefile +++ b/Makefile @@ -226,7 +226,7 @@ ci-web-portal: secure-envs @./dpc-web-portal-test.sh .PHONY: ci-api-client -ci-api-client: +ci-api-client: secure-envs @./dpc-api-client-test.sh .PHONY: unit-tests From ef39fa71e85dbfde50a1c345db4686fd02b7a8f2 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:01:45 -0500 Subject: [PATCH 20/28] change too many things at once and hope --- .github/workflows/ci-workflow.yml | 6 +++--- Makefile | 15 ++++++--------- docker-compose.portals.yml | 18 ------------------ dpc-api-client-test.sh | 2 +- dpc-api/docker/entrypoint.sh | 3 --- dpc-attribution/docker/entrypoint.sh | 3 --- dpc-client-integration-test.sh | 2 +- engines/api_client/.rspec | 2 +- engines/api_client/Dockerfile.local | 24 ------------------------ engines/api_client/Makefile | 8 -------- engines/api_client/README.md | 23 ++++++++++++----------- engines/api_client/spec/spec_helper.rb | 5 +++-- 12 files changed, 27 insertions(+), 84 deletions(-) delete mode 100644 engines/api_client/Dockerfile.local delete mode 100644 engines/api_client/Makefile diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index bdd0b9c243..dc4393470b 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -52,13 +52,13 @@ jobs: curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose chown root:root /usr/local/lib/docker/cli-plugins/docker-compose chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + - name: "Client Unit Tests" + run: | + make ci-api-client - name: "Set up Python and install Ansible" run: | sudo dnf -y install python3 python3-pip pip install ansible - - name: "Client Unit Tests" - run: | - make ci-api-client - name: "Set up JDK 17" uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 with: diff --git a/Makefile b/Makefile index 618a7b68ab..7e99ceeee8 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,6 @@ SMOKE_THREADS ?= 10 venv: venv/bin/activate -clean-jacoco: - mkdir -p jacocoReport && rm -r jacocoReport - venv/bin/activate: requirements.txt test -d venv || python3 -m venv venv . venv/bin/activate @@ -28,7 +25,7 @@ smoke/local: start-dpc # ============== api: ## Builds the Java API services -api: secure-envs clean-jacoco +api: secure-envs mvn clean compile -Perror-prone -B -V -ntp -T 4 -DskipTests mvn package -Pci -ntp -T 4 -DskipTests @@ -47,7 +44,7 @@ portal: @docker build -f dpc-portal/Dockerfile . -t dpc-web-portal dpc-client: - @docker compose -f docker-compose.yml -f docker-compose.portals.yml build dpc_client + @docker compose -f docker-compose.yml -f docker-compose.dpc-client.yml build dpc_client # Start commands # ============== @@ -184,7 +181,7 @@ portal-console: ## Run a rails console shell @docker compose -f docker-compose.yml -f docker-compose.portals.yml exec -it dpc_portal bin/console dpc-client-sh: - @docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --remove-orphans --entrypoint "sh" dpc_client + @docker compose -f docker-compose.dpc-client.yml run --remove-orphans --entrypoint "sh" dpc_client # Build & Test commands # ====================== @@ -226,7 +223,7 @@ ci-web-portal: secure-envs @./dpc-web-portal-test.sh .PHONY: ci-api-client -ci-api-client: secure-envs +ci-api-client: @./dpc-api-client-test.sh .PHONY: unit-tests @@ -236,6 +233,6 @@ unit-tests: .PHONY: load-tests load-tests: start-api-load-tests start-load-tests down-dpc-load-tests -.PHONY: dpc-client-integration-test -ci-api-client-integration-test: docker-base secure-envs +.PHONY: ci-api-client-integration +ci-api-client-integration: docker-base secure-envs @bash ./dpc-client-integration-test.sh diff --git a/docker-compose.portals.yml b/docker-compose.portals.yml index 2958d57515..8b4a6bd04a 100644 --- a/docker-compose.portals.yml +++ b/docker-compose.portals.yml @@ -155,21 +155,3 @@ services: timeout: 5s start_period: 30s retries: 5 - - dpc_client: - build: - context: . - dockerfile: engines/api_client/Dockerfile - image: dpc-client:latest - volumes: - # Mount specific directories to avoid overwriting - # precompiled assets (public/assets/) and node_modules - - "./engines/api_client:/api-client" - environment: - # Application settings - - GOLDEN_MACAROON=${GOLDEN_MACAROON} - - API_METADATA_URL=http://api:3002/api/v1 - - API_ADMIN_URL=http://api:9900 - - RUBY_YJIT_ENABLE=1 - - SKIP_SIMPLE_COV=${SKIP_SIMPLE_COV} - - ENV=local diff --git a/dpc-api-client-test.sh b/dpc-api-client-test.sh index 1113114d00..860c9c7f00 100755 --- a/dpc-api-client-test.sh +++ b/dpc-api-client-test.sh @@ -8,7 +8,7 @@ echo "│ │" echo "└───────────────────────┘" # Build the container -docker compose -f docker-compose.yml -f docker-compose.portals.yml build dpc_client +docker compose -f docker-compose.dpc-client.yml build dpc_client # Run the tests echo "┌───────────────────────────┐" diff --git a/dpc-api/docker/entrypoint.sh b/dpc-api/docker/entrypoint.sh index 08ddf89d35..4be3f3bc1a 100755 --- a/dpc-api/docker/entrypoint.sh +++ b/dpc-api/docker/entrypoint.sh @@ -11,12 +11,9 @@ bootstrap_config() { aws s3 sync "s3://$bucket" config/ } -echo "JACOCO: ${JACOCO}" if [ -n "$JACOCO" ]; then - echo 'JACOCO SET' JACOCO="-javaagent:/org.jacoco.agent-runtime.jar=destfile=/jacoco-report/jacoco-it.exec" else - echo 'JACOCO NOT SET' JACOCO="" fi diff --git a/dpc-attribution/docker/entrypoint.sh b/dpc-attribution/docker/entrypoint.sh index f3aa1fcfbe..d3c254879b 100755 --- a/dpc-attribution/docker/entrypoint.sh +++ b/dpc-attribution/docker/entrypoint.sh @@ -2,12 +2,9 @@ set -e -echo "JACOCO: ${JACOCO}" if [ -n "$JACOCO" ]; then - echo 'JACOCO SET' JACOCO="-javaagent:/org.jacoco.agent-runtime.jar=destfile=/jacoco-report/jacoco-it.exec" else - echo 'JACOCO NOT SET' JACOCO="" fi diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh index 57488b41f2..aa538c9be8 100755 --- a/dpc-client-integration-test.sh +++ b/dpc-client-integration-test.sh @@ -37,4 +37,4 @@ USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait echo "Starting integration tests" GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) \ SKIP_SIMPLE_COV=true \ -docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.portals.yml run --remove-orphans --entrypoint "bundle exec rspec --tag type:integration" dpc_client +docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.dpc-client.yml run --remove-orphans --entrypoint "bundle exec rspec --tag type:integration" dpc_client diff --git a/engines/api_client/.rspec b/engines/api_client/.rspec index f63b1c606a..f7e71acc14 100644 --- a/engines/api_client/.rspec +++ b/engines/api_client/.rspec @@ -1,2 +1,2 @@ --require spec_helper ---order defined \ No newline at end of file +--order defined diff --git a/engines/api_client/Dockerfile.local b/engines/api_client/Dockerfile.local deleted file mode 100644 index 08ba06f593..0000000000 --- a/engines/api_client/Dockerfile.local +++ /dev/null @@ -1,24 +0,0 @@ -FROM artifactory.cloud.cms.gov/docker/ruby:3.3-alpine AS ruby_builder - -# Install build dependencies -RUN apk update && \ - apk add --no-cache libffi-dev && \ - apk add --no-cache yaml-dev && \ - apk add --no-cache --virtual build-deps alpine-sdk tzdata && \ - apk add --no-cache gcompat - -# Set the working directory -RUN mkdir /api-client -WORKDIR /api-client - -# Copy over the files needed to fetch dependencies -COPY Gemfile Gemfile.lock api_client.gemspec /api-client/ -COPY lib /api-client/lib -RUN gem install bundler --no-document && \ - bundle config set force_ruby_platform true && \ - bundle install - -COPY app /api-client/app -COPY spec /api-client/spec -COPY coverage /api-client/coverage -COPY .rubocop.yml /api-client/.rubocop.yml diff --git a/engines/api_client/Makefile b/engines/api_client/Makefile deleted file mode 100644 index 5a43db5be1..0000000000 --- a/engines/api_client/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -build: - docker build -f Dockerfile.local . -t api_client -test: - docker run --rm -v ${PWD}:/api-client -w /api-client api_client bundle exec rspec - docker run --rm -v ${PWD}:/api-client -w /api-client api_client bundle exec rubocop - -run: - docker run -it --rm -v ${PWD}:/api-client -w /api-client api_client sh diff --git a/engines/api_client/README.md b/engines/api_client/README.md index 01fff1cbf1..f3ce2dafa0 100644 --- a/engines/api_client/README.md +++ b/engines/api_client/README.md @@ -19,21 +19,22 @@ And then execute: ```bash $ bundle install ``` +## Debugging and Development +To ssh into the container with the dpc_client, use the `make` command in the project root directory. +```bash +make dpc-client-sh +``` + ## Testing -Build the docker image +Testing is from the Makefile in the project root directory. -In the api_client directory +### Unit Tests ```bash -$ make build +make ci-api-client ``` -Run the tests until they pass -``` -$ make test -``` - -Jump into the docker shell for iterative development +### Integration tests with the API +```bash +make ci-api-client-integration ``` -make run -``` \ No newline at end of file diff --git a/engines/api_client/spec/spec_helper.rb b/engines/api_client/spec/spec_helper.rb index 991cfce00f..65c284ba0c 100644 --- a/engines/api_client/spec/spec_helper.rb +++ b/engines/api_client/spec/spec_helper.rb @@ -3,7 +3,7 @@ require 'simplecov' require 'webmock/rspec' -unless ENV['SKIP_SIMPLE_COV'] == 'true' +unless ENV.fetch('SKIP_SIMPLE_COV', 'false') == 'true' SimpleCov.start do track_files '**/{app,lib}/**/*.rb' add_filter 'lib/api_client/version.rb' @@ -14,11 +14,12 @@ end RSpec.configure do |config| + config.filter_run_excluding type: :integration + config.expect_with :rspec do |expectations| expectations.include_chain_clauses_in_custom_matcher_descriptions = true end - config.filter_run_excluding type: :integration config.mock_with :rspec do |mocks| # Prevents you from mocking or stubbing a method that does not exist on # a real object. This is generally recommended, and will default to From 9331740a91967397f53c03f219b2aa4a0a7ce5b9 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:03:23 -0500 Subject: [PATCH 21/28] add dpc-client-specific file for docker compose --- docker-compose.dpc-client.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docker-compose.dpc-client.yml diff --git a/docker-compose.dpc-client.yml b/docker-compose.dpc-client.yml new file mode 100644 index 0000000000..b246b95598 --- /dev/null +++ b/docker-compose.dpc-client.yml @@ -0,0 +1,22 @@ +volumes: + coverage: + + +services: + dpc_client: + build: + context: . + dockerfile: engines/api_client/Dockerfile + image: dpc-client:latest + volumes: + # Mount specific directories to avoid overwriting + # precompiled assets (public/assets/) and node_modules + - "./engines/api_client:/api-client" + environment: + # Application settings + - GOLDEN_MACAROON=${GOLDEN_MACAROON} + - API_METADATA_URL=http://api:3002/api/v1 + - API_ADMIN_URL=http://api:9900 + - RUBY_YJIT_ENABLE=1 + - SKIP_SIMPLE_COV=${SKIP_SIMPLE_COV:-} + - ENV=local From 5d3e4d327b2e47af5e176229543f496dd8e9c155 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:09:48 -0500 Subject: [PATCH 22/28] typo --- .github/workflows/ci-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index dc4393470b..62b863d378 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -69,7 +69,7 @@ jobs: run: mvn -ntp -U clean - name: "API Integration test" run: | - make ci-api-client-integration-test + make ci-api-client-integration - name: "Debug db" if: ${{ failure() }} run: docker logs client-integration-app-db-1 From 4092bda7df1c98dc2f5dc4ea6123634a1ed4aa91 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:50:52 -0500 Subject: [PATCH 23/28] clean up test --- .../spec/integration/dpc_client_spec.rb | 111 +++++++----------- 1 file changed, 43 insertions(+), 68 deletions(-) diff --git a/engines/api_client/spec/integration/dpc_client_spec.rb b/engines/api_client/spec/integration/dpc_client_spec.rb index 0511ffa319..3feb46eee0 100644 --- a/engines/api_client/spec/integration/dpc_client_spec.rb +++ b/engines/api_client/spec/integration/dpc_client_spec.rb @@ -26,54 +26,45 @@ end describe '#create_organization' do - context 'successful API request' do - it 'sends data to API and sets response instance variables' do - client.create_organization(org) - - expect(client.response_status).to eq(200) - expect(client.response_body.dig('identifier', 0, 'value')).to eq npi - end + it 'sends data to API and sets response instance variables' do + client.create_organization(org) + expect(client.response_status).to eq(200) + expect(client.response_body.dig('identifier', 0, 'value')).to eq npi end end describe '#get_organization_by_npi' do - context 'successful API request' do - it 'retrieves organization data from API' do - response = client.get_organization_by_npi(npi) - expect(response&.entry).to_not be_nil - expect(response.entry.length).to eq 1 - expect(response.entry.first.resource.identifier.first.value).to eq npi - end + it 'retrieves organization data from API' do + response = client.get_organization_by_npi(npi) + expect(response&.entry).to_not be_nil + expect(response.entry.length).to eq 1 + expect(response.entry.first.resource.identifier.first.value).to eq npi end end - describe '#post_create' do + describe 'with existing org' do let(:org_id) do response = client.get_organization_by_npi(npi) response.entry.first.resource.id end describe '#get_organization' do - context 'successful API request' do - it 'retrieves organization data from API' do - response = client.get_organization(org_id) - expect(response).to_not be_nil - expect(response.resourceType).to eq 'Organization' - end + it 'retrieves organization data from API' do + response = client.get_organization(org_id) + expect(response).to_not be_nil + expect(response.resourceType).to eq 'Organization' end end describe '#update_organization' do - context 'successful request' do - it 'sends org data to API' do - expect(client.update_organization(org, org_id)).to eq(client) - expect(client.response_successful?).to eq(true) - end + it 'sends org data to API' do + expect(client.update_organization(org, org_id)).to eq(client) + expect(client.response_successful?).to eq(true) end end - describe '#create_client_token' do - context 'successful API request' do + describe 'client tokens' do + context 'create' do it 'sends data to API and sets response instance variables' do client.create_client_token(org_id, params: { label: 'Sandbox Token 1' }) @@ -81,10 +72,8 @@ expect(client.response_body['label']).to eq('Sandbox Token 1') end end - end - describe '#get_client_tokens' do - context 'successful API request' do + context 'list' do it 'sends data to API and sets response instance variables' do client.get_client_tokens(org_id) @@ -93,18 +82,20 @@ expect(client.response_body['entities'].first['label']).to eq('Sandbox Token 1') end end - end - describe '#delete_client_token' do - context 'successful API request' do + context 'delete' do it 'returns success' do + # Need to get an existing id first client.get_client_tokens(org_id) expect(client.response_status).to eq(200) start_count = client.response_body['count'] expect(start_count).to be > 0 token_id = client.response_body['entities'].first['id'] + client.delete_client_token(org_id, token_id) + expect(client.response_status).to eq(204) + client.get_client_tokens(org_id) expect(client.response_status).to eq(200) expect(client.response_body['count'] + 1).to eq start_count @@ -112,21 +103,15 @@ end end - describe '#create_public_key' do - context 'successful API request' do + describe 'public keys' do + context 'create' do it 'sends data to API and sets response instance variables' do rsa_key = OpenSSL::PKey::RSA.new(4096) - rsa_key.to_pem - public_key = rsa_key.public_key.to_pem - message = 'This is the snippet used to verify a key pair in DPC.' - digest = OpenSSL::Digest.new('SHA256') - signature_binary = rsa_key.sign(digest, message) - snippet_signature = Base64.encode64(signature_binary) label = 'Sandbox Key 1' @@ -139,10 +124,8 @@ expect(client.response_body['label']).to eq label end end - end - describe '#get_public_keys' do - context 'successful API request' do + context 'list' do it 'sends data to API and sets response instance variables' do client.get_public_keys(org_id) @@ -151,21 +134,20 @@ expect(client.response_body['entities'].first['label']).to eq('Sandbox Key 1') end end - end - describe '#delete_public_key' do - context 'successful API request' do + context 'delete' do it 'sends data to API and sets response instance variables' do + # Need to get an existing id first client.get_public_keys(org_id) expect(client.response_status).to eq(200) start_count = client.response_body['count'] expect(start_count).to be > 0 public_key_id = client.response_body['entities'].first['id'] - client.delete_public_key( - org_id, - public_key_id - ) + + client.delete_public_key(org_id, public_key_id) + expect(client.response_status).to eq(200) + client.get_public_keys(org_id) expect(client.response_status).to eq(200) expect(client.response_body['count'] + 1).to eq start_count @@ -173,39 +155,36 @@ end end - describe '#create_ip_address' do - context 'successful API request' do + describe 'ip address' do + context 'create' do it 'sends data to API and sets response instance variables' do client.create_ip_address(org_id, params: { ip_address: '136.226.19.87' }) expect(client.response_status).to eq(200) expect(client.response_body['id']).to_not be_blank end end - end - describe '#get_ip_addresses' do - context 'successful API request' do + context 'list' do it 'sends data to API and sets response instance variables' do client.get_ip_addresses(org_id) expect(client.response_status).to eq(200) expect(client.response_body['count']).to be > 0 end end - end - describe '#delete_ip_address' do - context 'successful API request' do + context 'delete' do it 'sends data to API and sets response instance variables' do + # Need to get an existing id first client.get_ip_addresses(org_id) expect(client.response_status).to eq(200) start_count = client.response_body['count'] expect(start_count).to be > 0 ip_address_id = client.response_body['entities'].first['id'] - client.delete_ip_address( - org_id, - ip_address_id - ) + + client.delete_ip_address(org_id, ip_address_id) + expect(client.response_status).to eq(204) + client.get_ip_addresses(org_id) expect(client.response_status).to eq(200) expect(client.response_body['count'] + 1).to eq start_count @@ -213,8 +192,4 @@ end end end - - def stubbed_key - file_fixture('stubbed_key.pem').read - end end From 57b17fad439a95f6c19ed2084a2efd15aa447f53 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:55:46 -0500 Subject: [PATCH 24/28] let labels --- .../api_client/spec/integration/dpc_client_spec.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/engines/api_client/spec/integration/dpc_client_spec.rb b/engines/api_client/spec/integration/dpc_client_spec.rb index 3feb46eee0..dfc27a2224 100644 --- a/engines/api_client/spec/integration/dpc_client_spec.rb +++ b/engines/api_client/spec/integration/dpc_client_spec.rb @@ -64,12 +64,13 @@ end describe 'client tokens' do + let(:label) { 'Sandbox Token 1' } context 'create' do it 'sends data to API and sets response instance variables' do - client.create_client_token(org_id, params: { label: 'Sandbox Token 1' }) + client.create_client_token(org_id, params: { label: label }) expect(client.response_status).to eq(200) - expect(client.response_body['label']).to eq('Sandbox Token 1') + expect(client.response_body['label']).to eq(label) end end @@ -79,7 +80,7 @@ expect(client.response_status).to eq(200) expect(client.response_body['count']).to be > 0 - expect(client.response_body['entities'].first['label']).to eq('Sandbox Token 1') + expect(client.response_body['entities'].first['label']).to eq(label) end end @@ -104,6 +105,7 @@ end describe 'public keys' do + let(:label) { 'Sandbox Key 1' } context 'create' do it 'sends data to API and sets response instance variables' do rsa_key = OpenSSL::PKey::RSA.new(4096) @@ -114,7 +116,6 @@ signature_binary = rsa_key.sign(digest, message) snippet_signature = Base64.encode64(signature_binary) - label = 'Sandbox Key 1' client.create_public_key( org_id, params: { label:, public_key:, snippet_signature: } @@ -131,7 +132,7 @@ expect(client.response_status).to eq(200) expect(client.response_body['count']).to be > 0 - expect(client.response_body['entities'].first['label']).to eq('Sandbox Key 1') + expect(client.response_body['entities'].first['label']).to eq(label) end end From 78480c8749c2178a7c95fffcb8c2cb430d6efedb Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:59:01 -0500 Subject: [PATCH 25/28] cleanup --- docker-compose.dpc-client.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose.dpc-client.yml b/docker-compose.dpc-client.yml index b246b95598..a380ff5a8b 100644 --- a/docker-compose.dpc-client.yml +++ b/docker-compose.dpc-client.yml @@ -1,7 +1,3 @@ -volumes: - coverage: - - services: dpc_client: build: From 4e78e54783abaa2672e5fe5154ead86c617f7973 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Fri, 6 Mar 2026 09:44:37 -0500 Subject: [PATCH 26/28] cleanup --- .github/workflows/ci-workflow.yml | 9 ++- Makefile | 10 ++-- ...lient.yml => docker-compose.api-client.yml | 6 +- dpc-api-client-integration-test.sh | 57 +++++++++++++++++++ dpc-api-client-test.sh | 6 +- dpc-client-integration-test.sh | 40 ------------- engines/api_client/.rspec | 1 - engines/api_client/README.md | 12 +++- .../spec/integration/dpc_client_spec.rb | 4 -- 9 files changed, 81 insertions(+), 64 deletions(-) rename docker-compose.dpc-client.yml => docker-compose.api-client.yml (81%) create mode 100755 dpc-api-client-integration-test.sh delete mode 100755 dpc-client-integration-test.sh diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 62b863d378..d07ae26c76 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -36,7 +36,7 @@ env: ENV: "github-ci" jobs: - build-dpc-client: + build-dpc-api-client: name: "Build and Test DPC Client" runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} steps: @@ -72,14 +72,13 @@ jobs: make ci-api-client-integration - name: "Debug db" if: ${{ failure() }} - run: docker logs client-integration-app-db-1 + run: docker logs api-client-integration-app-db-1 - name: "Debug attribution" if: ${{ failure() }} - run: docker logs client-integration-app-attribution-1 + run: docker logs api-client-integration-app-attribution-1 - name: "Debug api" if: ${{ failure() }} - run: docker logs client-integration-app-api-1 + run: docker logs api-client-integration-app-api-1 - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh - diff --git a/Makefile b/Makefile index 7e99ceeee8..8850929169 100644 --- a/Makefile +++ b/Makefile @@ -43,8 +43,8 @@ portal: cp -r engines/api_client/ dpc-portal/vendor/api_client/ @docker build -f dpc-portal/Dockerfile . -t dpc-web-portal -dpc-client: - @docker compose -f docker-compose.yml -f docker-compose.dpc-client.yml build dpc_client +api-client: + @docker compose -f docker-compose.yml -f docker-compose.api-client.yml build dpc_api_client # Start commands # ============== @@ -180,8 +180,8 @@ portal-sh: ## Run a portal shell portal-console: ## Run a rails console shell @docker compose -f docker-compose.yml -f docker-compose.portals.yml exec -it dpc_portal bin/console -dpc-client-sh: - @docker compose -f docker-compose.dpc-client.yml run --remove-orphans --entrypoint "sh" dpc_client +api-client-sh: + @docker compose -f docker-compose.api-client.yml run --remove-orphans --entrypoint "sh" dpc_api_client # Build & Test commands # ====================== @@ -235,4 +235,4 @@ load-tests: start-api-load-tests start-load-tests down-dpc-load-tests .PHONY: ci-api-client-integration ci-api-client-integration: docker-base secure-envs - @bash ./dpc-client-integration-test.sh + @bash ./dpc-api-client-integration-test.sh diff --git a/docker-compose.dpc-client.yml b/docker-compose.api-client.yml similarity index 81% rename from docker-compose.dpc-client.yml rename to docker-compose.api-client.yml index a380ff5a8b..776641003d 100644 --- a/docker-compose.dpc-client.yml +++ b/docker-compose.api-client.yml @@ -1,11 +1,11 @@ services: - dpc_client: + dpc_api_client: build: context: . dockerfile: engines/api_client/Dockerfile - image: dpc-client:latest + image: dpc-api-client:latest volumes: - # Mount specific directories to avoid overwriting + # Mount specific directories to avoid overwriting # precompiled assets (public/assets/) and node_modules - "./engines/api_client:/api-client" environment: diff --git a/dpc-api-client-integration-test.sh b/dpc-api-client-integration-test.sh new file mode 100755 index 0000000000..f4107e8fa4 --- /dev/null +++ b/dpc-api-client-integration-test.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -Ee + +echo "┌------------───────────────────────┐" +echo "│ │" +echo "│ Running Api Gem Integration Tests |" +echo "│ │" +echo "└------------───────────────────────┘" + +# Current working directory +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +# Configure the Maven log level +export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.defaultLogLevel=info + +# Include secure environment variables +set -o allexport +[[ -f ${DIR}/ops/config/decrypted/local.env ]] && source ${DIR}/ops/config/decrypted/local.env +set +o allexport + +# Remove jacocoReport directory +if [ -d "${DIR}/jacocoReport" ]; then + rm -r "${DIR}/jacocoReport" +fi + +function _finally { + # don't shut it down if running on ci + if [ "$ENV" != 'github-ci' ]; then + echo "SHUTTING EVERYTHING DOWN" + docker compose -p api-client-integration-app down + docker volume rm api-client-integration-app_pgdata16 + fi +} + +trap _finally EXIT + +# Build the application +mvn -T 1.5C clean compile -Perror-prone -B -V -ntp -DskipTests +mvn -T 1.5C package -Pci -ntp -DskipTests + +echo "Starting api server for end-to-end tests" +USE_BFD_MOCK=true docker compose -p api-client-integration-app up api --wait + +echo "Starting integration tests" +GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) \ +SKIP_SIMPLE_COV=true \ +docker compose -p api-client-integration-app \ +-f docker-compose.yml -f docker-compose.api-client.yml \ +run --remove-orphans \ +--entrypoint "bundle exec rspec --order defined --tag type:integration" \ +dpc_api_client + +echo "┌─────────────────────-------------──┐" +echo "│ │" +echo "│ Api Gem Integration Tests Complete |" +echo "│ │" +echo "└────────────────────-------------───┘" diff --git a/dpc-api-client-test.sh b/dpc-api-client-test.sh index 860c9c7f00..76d126bd92 100755 --- a/dpc-api-client-test.sh +++ b/dpc-api-client-test.sh @@ -8,7 +8,7 @@ echo "│ │" echo "└───────────────────────┘" # Build the container -docker compose -f docker-compose.dpc-client.yml build dpc_client +docker compose -f docker-compose.api-client.yml build dpc_api_client # Run the tests echo "┌───────────────────────────┐" @@ -16,8 +16,8 @@ echo "│ │" echo "│ Running Api Gem Tests │" echo "│ │" echo "└───────────────────────────┘" -docker run --rm -v ${PWD}/engines/api_client/coverage:/api-client/coverage dpc-client bundle exec rspec -docker run --rm dpc-client bundle exec rubocop +docker run --rm -v ${PWD}/engines/api_client/coverage:/api-client/coverage dpc-api-client bundle exec rspec +docker run --rm dpc-api-client bundle exec rubocop echo "┌────────────────────────────────┐" echo "│ │" diff --git a/dpc-client-integration-test.sh b/dpc-client-integration-test.sh deleted file mode 100755 index aa538c9be8..0000000000 --- a/dpc-client-integration-test.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -Ee - -# Current working directory -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" - -# Configure the Maven log level -export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.defaultLogLevel=info - -# Include secure environment variables -set -o allexport -[[ -f ${DIR}/ops/config/decrypted/local.env ]] && source ${DIR}/ops/config/decrypted/local.env -set +o allexport - -# Remove jacocoReport directory -if [ -d "${DIR}/jacocoReport" ]; then - rm -r "${DIR}/jacocoReport" -fi - -function _finally { - # don't shut it down if running on ci - if [ "$ENV" != 'github-ci' ]; then - echo "SHUTTING EVERYTHING DOWN" - docker compose -p client-integration-app down - docker volume rm client-integration-app_pgdata16 - fi -} - -trap _finally EXIT - -# Build the application -mvn -T 1.5C clean compile -Perror-prone -B -V -ntp -DskipTests -mvn -T 1.5C package -Pci -ntp -DskipTests - -echo "Starting api server for end-to-end tests" -USE_BFD_MOCK=true docker compose -p client-integration-app up api --wait -echo "Starting integration tests" -GOLDEN_MACAROON=$(curl -X POST http://localhost:9903/tasks/generate-token) \ -SKIP_SIMPLE_COV=true \ -docker compose -p client-integration-app -f docker-compose.yml -f docker-compose.dpc-client.yml run --remove-orphans --entrypoint "bundle exec rspec --tag type:integration" dpc_client diff --git a/engines/api_client/.rspec b/engines/api_client/.rspec index f7e71acc14..c99d2e7396 100644 --- a/engines/api_client/.rspec +++ b/engines/api_client/.rspec @@ -1,2 +1 @@ --require spec_helper ---order defined diff --git a/engines/api_client/README.md b/engines/api_client/README.md index f3ce2dafa0..76c0ac60f4 100644 --- a/engines/api_client/README.md +++ b/engines/api_client/README.md @@ -19,15 +19,21 @@ And then execute: ```bash $ bundle install ``` + ## Debugging and Development -To ssh into the container with the dpc_client, use the `make` command in the project root directory. +To build an image, use the `make` command in the project root directory. ```bash -make dpc-client-sh +make api-client ``` +To ssh into a Docker container with the dpc_client code, use the `make` command in the project root directory. +```bash +make api-client-sh +``` + ## Testing -Testing is from the Makefile in the project root directory. +Test using `make` commands in the project root directory. ### Unit Tests ```bash diff --git a/engines/api_client/spec/integration/dpc_client_spec.rb b/engines/api_client/spec/integration/dpc_client_spec.rb index dfc27a2224..3bae4cc463 100644 --- a/engines/api_client/spec/integration/dpc_client_spec.rb +++ b/engines/api_client/spec/integration/dpc_client_spec.rb @@ -14,9 +14,6 @@ id: '8453e48b-0b42-4ddf-8b43-07c7aa2a3d88', address_zip: '22222') end - let!(:reg_org) do - double('RegisteredOrg', api_id: 'some-api-key') - end before(:all) do WebMock.disable! @@ -109,7 +106,6 @@ context 'create' do it 'sends data to API and sets response instance variables' do rsa_key = OpenSSL::PKey::RSA.new(4096) - rsa_key.to_pem public_key = rsa_key.public_key.to_pem message = 'This is the snippet used to verify a key pair in DPC.' digest = OpenSSL::Digest.new('SHA256') From 1945a46bb606c1c1dd3da2461fd3346b2036d74e Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Fri, 6 Mar 2026 09:57:07 -0500 Subject: [PATCH 27/28] Put back all ci tests --- .github/workflows/ci-workflow.yml | 334 ++++++++++++++++++++++++++++- dpc-api-client-integration-test.sh | 20 +- 2 files changed, 338 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index d07ae26c76..3208d6c1a8 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -36,8 +36,167 @@ env: ENV: "github-ci" jobs: + build-api: + name: "Build and Test API" + runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} + env: + REPORT_COVERAGE: true + steps: + - name: Assert Ownership + run: sudo chmod -R 777 . + - name: "Checkout code" + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - name: Cleanup Runner + run: ./scripts/cleanup-docker.sh + - name: "Set up JDK 17" + uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 + with: + java-version: "17" + distribution: "corretto" + cache: maven + - name: Clean maven + run: mvn -ntp -U clean + - name: "Set up Python and install Ansible" + run: | + sudo dnf -y install python3 python3-pip + pip install ansible + - name: Install docker compose manually + run: | + mkdir -p /usr/local/lib/docker/cli-plugins + curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose + chown root:root /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + - name: "API Build" + id: ci-app + run: | + export PATH=$PATH:~/.local/bin + make ci-app + - name: "Debug db" + if: ${{ failure() && steps.ci-app.outcome == 'failure' }} + run: docker logs start-v1-app-db-1 + - name: "Debug attribution" + if: ${{ failure() && steps.ci-app.outcome == 'failure' }} + run: docker logs start-v1-app-attribution-1 + - name: "Debug aggregation" + if: ${{ failure() && steps.ci-app.outcome == 'failure' }} + run: docker logs start-v1-app-aggregation-1 + - name: "Debug api" + if: ${{ failure() && steps.ci-app.outcome == 'failure' }} + run: docker logs start-v1-app-api-1 + - name: "Move jacoco reports" + run: | + sudo mkdir jacoco-reports + sudo cp ./dpc-aggregation/target/site/jacoco-it/jacoco.xml jacoco-reports/dpc-aggregation-it-jacoco.xml + sudo cp ./dpc-aggregation/target/site/jacoco/jacoco.xml jacoco-reports/dpc-aggregation-jacoco.xml + sudo cp ./dpc-api/target/site/jacoco-it/jacoco.xml jacoco-reports/dpc-api-it-jacoco.xml + sudo cp ./dpc-api/target/site/jacoco/jacoco.xml jacoco-reports/dpc-api-jacoco.xml + sudo cp ./dpc-attribution/target/site/jacoco-it/jacoco.xml jacoco-reports/dpc-attribution-it-jacoco.xml + sudo cp ./dpc-attribution/target/site/jacoco/jacoco.xml jacoco-reports/dpc-attribution-jacoco.xml + sudo cp ./dpc-bluebutton/target/site/jacoco/jacoco.xml jacoco-reports/dpc-bluebutton-jacoco.xml + sudo cp ./dpc-common/target/site/jacoco/jacoco.xml jacoco-reports/dpc-common-jacoco.xml + sudo cp ./dpc-macaroons/target/site/jacoco/jacoco.xml jacoco-reports/dpc-macaroons-jacoco.xml + sudo cp ./dpc-queue/target/site/jacoco/jacoco.xml jacoco-reports/dpc-queue-jacoco.xml + - name: Upload jacoco reports + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: code-coverage-report-dpc-api + path: ./jacoco-reports + - name: Cleanup + if: ${{ always() }} + run: ./scripts/cleanup-docker.sh + + build-dpc-web: + name: "Build and Test DPC Web" + runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} + steps: + - name: Assert Ownership + run: sudo chmod -R 777 . + - name: "Checkout code" + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - name: Cleanup Runner + run: ./scripts/cleanup-docker.sh + - name: Install docker compose manually + run: | + mkdir -p /usr/local/lib/docker/cli-plugins + curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose + chown root:root /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + - name: "DPC Web Build" + run: | + make ci-web-portal + - name: "Copy test results" + run: sudo cp dpc-web/coverage/.resultset.json web-resultset-raw.json + - name: Archive code coverage results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: code-coverage-report-dpc-web + path: ./web-resultset-raw.json + - name: Cleanup + if: ${{ always() }} + run: ./scripts/cleanup-docker.sh + + build-dpc-admin: + name: "Build and Test DPC Admin Portal" + runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} + steps: + - name: Assert Ownership + run: sudo chmod -R 777 . + - name: "Checkout code" + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - name: Cleanup Runner + run: ./scripts/cleanup-docker.sh + - name: Install docker compose manually + run: | + mkdir -p /usr/local/lib/docker/cli-plugins + curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose + chown root:root /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + - name: "DPC Admin Portal Build" + run: | + make ci-admin-portal + - name: "Copy test results" + run: sudo cp dpc-admin/coverage/.resultset.json admin-resultset-raw.json + - name: Archive code coverage results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: code-coverage-report-dpc-admin + path: ./admin-resultset-raw.json + - name: Cleanup + if: ${{ always() }} + run: ./scripts/cleanup-docker.sh + + build-dpc-portal: + name: "Build and Test DPC Portal" + runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} + steps: + - name: Assert Ownership + run: sudo chmod -R 777 . + - name: "Checkout code" + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - name: Cleanup Runner + run: ./scripts/cleanup-docker.sh + - name: Install docker compose manually + run: | + mkdir -p /usr/local/lib/docker/cli-plugins + curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose + chown root:root /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + - name: "DPC Portal Build" + run: | + make ci-portal + - name: "Copy test results" + run: sudo cp dpc-portal/coverage/.resultset.json portal-resultset-raw.json + - name: Archive code coverage results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: code-coverage-report-dpc-portal + path: ./portal-resultset-raw.json + - name: Cleanup + if: ${{ always() }} + run: ./scripts/cleanup-docker.sh + build-dpc-api-client: - name: "Build and Test DPC Client" + name: "Build and Test DPC API Client" runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} steps: - name: Assert Ownership @@ -52,7 +211,7 @@ jobs: curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose chown root:root /usr/local/lib/docker/cli-plugins/docker-compose chmod +x /usr/local/lib/docker/cli-plugins/docker-compose - - name: "Client Unit Tests" + - name: "API Client Unit Tests" run: | make ci-api-client - name: "Set up Python and install Ansible" @@ -67,18 +226,181 @@ jobs: cache: maven - name: Clean maven run: mvn -ntp -U clean - - name: "API Integration test" + - name: "API Client Integration test" + id: integration-test run: | make ci-api-client-integration - name: "Debug db" - if: ${{ failure() }} + if: ${{ failure() && steps.integration-test.outcome == 'failure' }} run: docker logs api-client-integration-app-db-1 - name: "Debug attribution" - if: ${{ failure() }} + if: ${{ failure() && steps.integration-test.outcome == 'failure' }} run: docker logs api-client-integration-app-attribution-1 - name: "Debug api" - if: ${{ failure() }} + if: ${{ failure() && steps.integration-test.outcome == 'failure' }} run: docker logs api-client-integration-app-api-1 - name: Cleanup if: ${{ always() }} run: ./scripts/cleanup-docker.sh + + sonar-quality-gate-dpc-web-and-admin: + name: Sonarqube Quality Gate for dpc-web and dpc-admin + needs: [build-dpc-admin, build-dpc-web] + runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} + env: + # Workaround until https://jira.cms.gov/browse/PLT-338 is implemented. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + steps: + - name: Assert Ownership + run: sudo chmod -R 777 . + - name: "Checkout code" + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + with: + fetch-depth: 0 + - name: Cleanup Runner + run: ./scripts/cleanup-docker.sh + - name: Download web code coverage + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: code-coverage-report-dpc-web + - name: Download admin code coverage + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: code-coverage-report-dpc-admin + - name: "Reformat test results" # Sonarqube will run in a docker container and wants the paths to be from /github/workspace + run: | + sudo jq '.RSpec.coverage |= with_entries(if .key | contains("dpc-web") then .key |= sub("/dpc-web"; "${{ github.workspace }}/dpc-web") else . end)' web-resultset-raw.json > web-resultset.json + sudo jq '.RSpec.coverage |= with_entries(if .key | contains("dpc-admin") then .key |= sub("/dpc-admin"; "${{ github.workspace }}/dpc-admin") else . end)' admin-resultset-raw.json > admin-resultset.json + - name: Set env vars from AWS params + uses: cmsgov/cdap/actions/aws-params-env-action@main + env: + AWS_REGION: ${{ vars.AWS_REGION }} + with: + params: | + SONAR_HOST_URL=/sonarqube/url + SONAR_TOKEN=/sonarqube/token + - name: Run quality gate scan + uses: sonarsource/sonarqube-scan-action@fd88b7d7ccbaefd23d8f36f73b59db7a3d246602 # v6.0.0 + with: + args: + -Dsonar.projectKey=bcda-dpc-web + -Dsonar.sources=./dpc-web/app,./dpc-web/lib,./dpc-admin/app,./dpc-admin/lib + -Dsonar.ruby.coverage.reportPaths=./web-resultset.json,./admin-resultset.json + -Dsonar.working.directory=./sonar_workspace + -Dsonar.branch.name=${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }} + -Dsonar.projectVersion=${{ github.ref_name == 'main' && github.sha || 'branch' }} + -Dsonar.qualitygate.wait=true + -Dsonar.ci.autoconfig.disabled=true + -Dsonar.branch.target=${{ github.base_ref }} + - name: Cleanup + if: ${{ always() }} + run: ./scripts/cleanup-docker.sh + + sonar-quality-gate-dpc-portal: + name: Sonarqube Quality Gate for dpc-portal + needs: build-dpc-portal + runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} + env: + # Workaround until https://jira.cms.gov/browse/PLT-338 is implemented. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" + steps: + - name: Assert Ownership + run: sudo chmod -R 777 . + - name: "Checkout code" + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + with: + fetch-depth: 0 + - name: Cleanup Runner + run: ./scripts/cleanup-docker.sh + - name: Download code coverage + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: code-coverage-report-dpc-portal + - name: "Reformat test results" # Sonarqube will run in a docker container and wants the paths to be from /github/workspace + run: | + sudo jq '.RSpec.coverage |= with_entries(if .key | contains("dpc-portal") then .key |= sub("/dpc-portal"; "${{ github.workspace }}/dpc-portal") else . end)' portal-resultset-raw.json > portal-resultset.json + - name: Set env vars from AWS params + uses: cmsgov/cdap/actions/aws-params-env-action@main + env: + AWS_REGION: ${{ vars.AWS_REGION }} + with: + params: | + SONAR_HOST_URL=/sonarqube/url + SONAR_TOKEN=/sonarqube/token + - name: Run quality gate scan + uses: sonarsource/sonarqube-scan-action@fd88b7d7ccbaefd23d8f36f73b59db7a3d246602 # v6.0.0 + with: + args: + -Dsonar.projectKey=bcda-dpc-portal + -Dsonar.sources=./dpc-portal/app,./dpc-portal/lib + -Dsonar.coverage.exclusions=**/*_preview.rb,**/*html.erb,**/application_* + -Dsonar.ruby.coverage.reportPaths=./portal-resultset.json + -Dsonar.working.directory=./sonar_workspace + -Dsonar.branch.name=${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }} + -Dsonar.projectVersion=${{ github.ref_name == 'main' && github.sha || 'branch' }} + -Dsonar.qualitygate.wait=true + -Dsonar.ci.autoconfig.disabled=true + -Dsonar.branch.target=${{ github.base_ref }} + - name: Cleanup + if: ${{ always() }} + run: ./scripts/cleanup-docker.sh + + sonar-quality-gate-dpc-api: + name: Sonarqube Quality Gate for dpc-api + needs: build-api + runs-on: codebuild-dpc-app-${{github.run_id}}-${{github.run_attempt}} + env: + # Workaround until https://jira.cms.gov/browse/PLT-338 is implemented. + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + steps: + - name: Assert Ownership + run: sudo chmod -R 777 . + - name: Checkout Code + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + with: + fetch-depth: 0 + - name: Cleanup Runner + run: ./scripts/cleanup-docker.sh + - name: Setup Java + uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 + with: + java-version: '17' + distribution: temurin + cache: maven + - name: Set env vars from AWS params + uses: cmsgov/cdap/actions/aws-params-env-action@main + env: + AWS_REGION: ${{ vars.AWS_REGION }} + with: + params: | + SONAR_HOST_URL=/sonarqube/url + SONAR_TOKEN=/sonarqube/token + - name: Install Maven 3.6.3 + run: | + export PATH="$PATH:/opt/maven/bin" + echo "PATH=$PATH" >> $GITHUB_ENV + if mvn -v; then echo "Maven already installed" && exit 0; else echo "Installing Maven"; fi + tmpdir="$(mktemp -d)" + curl -LsS https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz | tar xzf - -C "$tmpdir" + sudo rm -rf /opt/maven + sudo mv "$tmpdir/apache-maven-3.6.3" /opt/maven + - name: Clean maven + run: | + mvn -ntp -U clean + - name: Compile Project + run: | + mvn clean compile -Perror-prone -B -V -ntp + - name: Download code coverage + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: code-coverage-report-dpc-api + path: jacoco-reports + - name: Verify download + run: | + find . -name dpc-api-jacoco.xml + - name: Run quality gate scan + run: | + mvn org.sonarsource.scanner.maven:sonar-maven-plugin:4.0.0.4121:sonar -Dsonar.projectKey=bcda-dpc-api -Dsonar.branch.name=${{ github.event_name == 'pull_request' && github.head_ref || github.event_name == 'pull_request' && github.head_ref || github.ref_name }} -Dsonar.working.directory=./.sonar_workspace -Dsonar.projectVersion=${{ github.ref_name == 'main' && github.sha || 'branch' }} -Dsonar.qualitygate.wait=true -Dsonar.coverage.jacoco.xmlReportPaths="../jacoco-reports/*.xml" -Dsonar.ci.autoconfig.disabled=true -Dsonar.branch.target=${{ github.base_ref }} + - name: Cleanup + if: ${{ always() }} + run: ./scripts/cleanup-docker.sh diff --git a/dpc-api-client-integration-test.sh b/dpc-api-client-integration-test.sh index f4107e8fa4..043d15b011 100755 --- a/dpc-api-client-integration-test.sh +++ b/dpc-api-client-integration-test.sh @@ -1,11 +1,11 @@ #!/bin/bash set -Ee -echo "┌------------───────────────────────┐" -echo "│ │" -echo "│ Running Api Gem Integration Tests |" -echo "│ │" -echo "└------------───────────────────────┘" +echo "┌-------------------───────────────────────┐" +echo "│ │" +echo "│ Running API Client Gem Integration Tests |" +echo "│ │" +echo "└------------─────────-------──────────────┘" # Current working directory DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" @@ -50,8 +50,8 @@ run --remove-orphans \ --entrypoint "bundle exec rspec --order defined --tag type:integration" \ dpc_api_client -echo "┌─────────────────────-------------──┐" -echo "│ │" -echo "│ Api Gem Integration Tests Complete |" -echo "│ │" -echo "└────────────────────-------------───┘" +echo "┌───────────-------──────────-------------──┐" +echo "│ │" +echo "│ API Client Gem Integration Tests Complete |" +echo "│ │" +echo "└────────────────────--------------------───┘" From 27b7fe8853539a67836a19a41c47e2883d756bc3 Mon Sep 17 00:00:00 2001 From: jdettmannnava <145699825+jdettmannnava@users.noreply.github.com> Date: Fri, 6 Mar 2026 10:56:40 -0500 Subject: [PATCH 28/28] single-line single cmd --- .github/workflows/ci-workflow.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 3208d6c1a8..70f8676cf0 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -212,8 +212,7 @@ jobs: chown root:root /usr/local/lib/docker/cli-plugins/docker-compose chmod +x /usr/local/lib/docker/cli-plugins/docker-compose - name: "API Client Unit Tests" - run: | - make ci-api-client + run: make ci-api-client - name: "Set up Python and install Ansible" run: | sudo dnf -y install python3 python3-pip @@ -228,8 +227,7 @@ jobs: run: mvn -ntp -U clean - name: "API Client Integration test" id: integration-test - run: | - make ci-api-client-integration + run: make ci-api-client-integration - name: "Debug db" if: ${{ failure() && steps.integration-test.outcome == 'failure' }} run: docker logs api-client-integration-app-db-1