Merge pull request #763 from ELIXIR-NO/chore/improve-build-ci

chore: eliminate redundant Gradle builds from dev and CI pipelines

Author: yasinmiran

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.github
+.gradle
+.idea
+**/build
+**/*.md
+dev.sh
+*.png
+*.jpg

.github/workflows/build-and-test.yml

@@ -29,9 +29,9 @@ jobs:
         uses: actions/cache@v5
         with:
           path: |
-            ~/.gradle/cache
+            ~/.gradle/caches
             ~/.gradle/wrapper
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties', 'gradle.properties') }}
           restore-keys: |
             ${{ runner.os }}-gradle-
 
@@ -42,24 +42,21 @@ jobs:
         run: ./gradlew :lib:crypt4gh:test :lib:clearinghouse:test :lib:tsd-file-api-client:test :services:tsd-api-mock:test :services:mq-interceptor:test :services:localega-tsd-proxy:test :services:cega-mock:test :cli:lega-commander:test
 
 
-      - name: Run e2e tests
-        run: |
-          ./gradlew clean
-          ./gradlew start-docker-containers
+      - name: Start e2e environment
+        run: ./dev.sh start
 
-          # Wait for the e2e-tests container to complete
+      - name: Wait for e2e tests
+        run: |
           echo "Waiting for e2e-tests container to complete..."
           while [ "$(docker inspect --format='{{.State.Running}}' e2e-tests 2>/dev/null)" == "true" ]; do
             echo "e2e-tests still running, waiting for 5 seconds..."
             sleep 5
           done
 
-          # Show the logs from the e2e-tests container
           echo "=============== e2e-tests Container Logs ==============="
           docker logs e2e-tests
           echo "========================================================"
 
-          # Get the exit code once the container has finished
           EXIT_CODE=$(docker inspect --format='{{.State.ExitCode}}' e2e-tests)
           if [ $EXIT_CODE -ne 0 ]; then
             echo "e2e tests failed with exit code $EXIT_CODE"

dev.sh

@@ -67,7 +67,7 @@ function log_step() {
 }
 
 function show_header() {
-    clear
+    [[ -t 1 ]] && clear
     printf "${PURPLE}${BOLD}"
     printf "===============================================================\n"
     printf "                   FEGA Development Tools                      \n"
@@ -82,9 +82,17 @@ function start() {
     show_header
     log_step "Starting development environment"
 
-    if ./gradlew clean && bash -c "./gradlew start-docker-containers"; then
+    log_step "Preparing configuration"
+    cd e2eTests
+    source env.sh
+    ./scripts/bootstrap.sh apply_configs
+    ./scripts/bootstrap.sh check_requirements
+    log_step "Starting containers (Docker will build images)"
+    if docker compose up --build -d; then
+        cd ..
         log_success "Development environment started successfully!"
     else
+        cd ..
         log_error "Failed to start development environment"
         return 1
     fi
@@ -94,9 +102,12 @@ function stop() {
     show_header
     log_step "Stopping development environment"
 
-    if ./gradlew stop-docker-containers; then
+    cd e2eTests
+    if docker compose down --rmi local -v; then
+        cd ..
         log_success "Development environment stopped successfully!"
     else
+        cd ..
         log_error "Failed to stop development environment"
         return 1
     fi
@@ -106,37 +117,22 @@ function reexecute_tests_in_container() {
     show_header
     log_step "Rebuilding and reexecuting E2E tests"
 
-    ./gradlew :e2eTests:clean > /dev/null &&
-    ./gradlew :e2eTests:assemble > /dev/null &&
     docker rm e2e-tests -f > /dev/null 2>&1 &&
     docker rmi fega-norway-e2e-tests:latest -f > /dev/null 2>&1 &&
     cd e2eTests &&
-    docker compose up -d e2e-tests > /dev/null &&
+    docker compose up --build -d e2e-tests > /dev/null &&
     cd .. &&
     log_success "E2E tests rebuilt and reexecuted!"
 }
 
-function reexecute_tests_in_container() {
-  ./gradlew :e2eTests:clean > /dev/null &&
-  ./gradlew :e2eTests:assemble > /dev/null &&
-  docker rm e2e-tests -f > /dev/null &&
-  docker rmi fega-norway-e2e-tests:latest -f > /dev/null &&
-  cd e2eTests &&
-  docker compose up -d e2e-tests > /dev/null &&
-  cd .. &&
-  echo "Task done ✅ Built and reexecuting e2e-tests."
-}
-
 function rebuild_and_deploy_proxy() {
     show_header
     log_step "Rebuilding and deploying proxy service"
 
-    ./gradlew :services:localega-tsd-proxy:clean > /dev/null &&
-    ./gradlew :services:localega-tsd-proxy:assemble > /dev/null &&
     docker rm proxy -f > /dev/null 2>&1 &&
     docker rmi tsd-proxy:latest -f > /dev/null 2>&1 &&
     cd e2eTests &&
-    docker compose up -d proxy > /dev/null &&
+    docker compose up --build -d proxy > /dev/null &&
     cd .. &&
     log_success "Proxy service rebuilt and deployed!"
 }
@@ -145,12 +141,10 @@ function rebuild_and_deploy_mq_interceptor() {
     show_header
     log_step "Rebuilding and deploying MQ interceptor"
 
-    ./gradlew :services:mq-interceptor:clean > /dev/null &&
-    ./gradlew :services:mq-interceptor:assemble > /dev/null &&
     docker rm interceptor -f > /dev/null 2>&1 &&
     docker rmi mq-interceptor:latest -f > /dev/null 2>&1 &&
     cd e2eTests &&
-    docker compose up -d interceptor > /dev/null &&
+    docker compose up --build -d interceptor > /dev/null &&
     cd .. &&
     log_success "MQ interceptor rebuilt and deployed!"
 }
@@ -184,12 +178,10 @@ function rebuild_and_deploy_tsd() {
     show_header
     log_step "Rebuilding and deploying TSD API mock"
 
-    ./gradlew :services:tsd-api-mock:clean > /dev/null &&
-    ./gradlew :services:tsd-api-mock:assemble > /dev/null &&
     docker rm tsd -f > /dev/null 2>&1 &&
     docker rmi tsd-api-mock:latest -f > /dev/null 2>&1 &&
     cd e2eTests &&
-    docker compose up -d tsd > /dev/null &&
+    docker compose up --build -d tsd > /dev/null &&
     cd .. &&
     log_success "TSD API mock rebuilt and deployed!"
 }
@@ -198,7 +190,6 @@ function rebuild_clearinghouse() {
     show_header
     log_step "Rebuilding clearinghouse library"
 
-    ./gradlew :lib:clearinghouse:clean > /dev/null &&
     ./gradlew :lib:clearinghouse:assemble > /dev/null &&
     log_success "Clearinghouse library rebuilt!"
     log_info "This library is used by localega-tsd-proxy."
@@ -212,7 +203,6 @@ function rebuild_tsd_file_api_client() {
     show_header
     log_step "Rebuilding TSD file API client"
 
-    ./gradlew :lib:tsd-file-api-client:clean > /dev/null &&
     ./gradlew :lib:tsd-file-api-client:assemble > /dev/null &&
     log_success "TSD file API client rebuilt!"
     log_info "This library is used by localega-tsd-proxy."
@@ -226,7 +216,6 @@ function rebuild_crypt4gh() {
     show_header
     log_step "Rebuilding crypt4gh library"
 
-    ./gradlew :lib:crypt4gh:clean > /dev/null &&
     ./gradlew :lib:crypt4gh:assemble > /dev/null &&
     log_success "Crypt4gh library rebuilt!"
 }

e2eTests/e2e-tests.Dockerfile

@@ -1,26 +1,40 @@
-# Use Temurin 21 as the base image for Java 21
 FROM eclipse-temurin:21-jdk-alpine AS builder
-
 WORKDIR /app
 
-COPY . .
+# Layer 1: build infrastructure (rarely changes)
+COPY gradlew settings.gradle.kts build.gradle.kts gradle.properties ./
+COPY gradle/ gradle/
+COPY buildSrc/ buildSrc/
+
+# Layer 2: all module build files so Gradle can configure the project graph
+COPY lib/clearinghouse/build.gradle.kts lib/clearinghouse/
+COPY lib/tsd-file-api-client/build.gradle.kts lib/tsd-file-api-client/
+COPY lib/crypt4gh/build.gradle.kts lib/crypt4gh/
+COPY services/localega-tsd-proxy/build.gradle.kts services/localega-tsd-proxy/
+COPY services/tsd-api-mock/build.gradle.kts services/tsd-api-mock/
+COPY services/cega-mock/build.gradle.kts services/cega-mock/
+COPY services/mq-interceptor/build.gradle.kts services/mq-interceptor/
+COPY e2eTests/build.gradle.kts e2eTests/
+RUN mkdir -p cli/lega-commander
+
+# Layer 3: resolve dependencies (cached until build.gradle.kts changes)
+RUN ./gradlew :e2eTests:dependencies --no-daemon 2>&1 || true
+
+# Layer 4: source code (changes frequently — only what this module needs)
+COPY lib/crypt4gh/src/ lib/crypt4gh/src/
+COPY e2eTests/src/ e2eTests/src/
 
 RUN ./gradlew :e2eTests:jar --no-daemon
 
 FROM eclipse-temurin:21-jre-alpine
 
-# Install bash
 RUN apk add --no-cache bash
 
-# Set the working directory in the container
 WORKDIR /fega-norway
 
-# Copy the application JAR, env, and scripts
 COPY --from=builder /app/e2eTests/build/libs/e2eTests.jar /fega-norway/e2eTests.jar
-COPY --from=builder /app/e2eTests/entrypoint.sh /fega-norway/entrypoint.sh
+COPY e2eTests/entrypoint.sh /fega-norway/entrypoint.sh
 
-# Make entrypoint executable
 RUN chmod +x /fega-norway/entrypoint.sh
 
-# Run the entrypoint using bash
 CMD ["/bin/bash", "/fega-norway/entrypoint.sh"]

gradle.properties

@@ -0,0 +1,4 @@
+org.gradle.parallel=true
+org.gradle.caching=true
+org.gradle.daemon=true
+org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError

services/localega-tsd-proxy/Dockerfile

@@ -1,11 +1,34 @@
 FROM eclipse-temurin:21-jdk-alpine AS builder
 WORKDIR /app
 
-COPY . .
-RUN ./gradlew :services:localega-tsd-proxy:clean :services:localega-tsd-proxy:build -x test --no-daemon
+# Layer 1: build infrastructure (rarely changes)
+COPY gradlew settings.gradle.kts build.gradle.kts gradle.properties ./
+COPY gradle/ gradle/
+COPY buildSrc/ buildSrc/
+
+# Layer 2: all module build files so Gradle can configure the project graph
+COPY lib/clearinghouse/build.gradle.kts lib/clearinghouse/
+COPY lib/tsd-file-api-client/build.gradle.kts lib/tsd-file-api-client/
+COPY lib/crypt4gh/build.gradle.kts lib/crypt4gh/
+COPY services/localega-tsd-proxy/build.gradle.kts services/localega-tsd-proxy/
+COPY services/tsd-api-mock/build.gradle.kts services/tsd-api-mock/
+COPY services/cega-mock/build.gradle.kts services/cega-mock/
+COPY services/mq-interceptor/build.gradle.kts services/mq-interceptor/
+COPY e2eTests/build.gradle.kts e2eTests/
+RUN mkdir -p cli/lega-commander
+
+# Layer 3: resolve dependencies (cached until build.gradle.kts changes)
+RUN ./gradlew :services:localega-tsd-proxy:dependencies --no-daemon || true
+
+# Layer 4: source code (changes frequently — only what this service needs)
+COPY lib/clearinghouse/src/ lib/clearinghouse/src/
+COPY lib/tsd-file-api-client/src/ lib/tsd-file-api-client/src/
+COPY services/localega-tsd-proxy/src/ services/localega-tsd-proxy/src/
+
+RUN ./gradlew :services:localega-tsd-proxy:build -x test --no-daemon
 
 FROM eclipse-temurin:21-jre-alpine
 WORKDIR /app
-COPY --from=builder app/services/localega-tsd-proxy/build/libs/localega-tsd-proxy.jar /app/app.jar
+COPY --from=builder /app/services/localega-tsd-proxy/build/libs/localega-tsd-proxy.jar /app/app.jar
 EXPOSE 8080
-CMD ["java","-XX:+UseG1GC", "-jar","app.jar"]
+CMD ["java", "-XX:+UseG1GC", "-jar", "app.jar"]

services/tsd-api-mock/Dockerfile

@@ -1,12 +1,33 @@
 FROM eclipse-temurin:21-jdk-alpine AS builder
 WORKDIR /app
 
-COPY . .
-RUN ./gradlew :services:tsd-api-mock:clean :services:tsd-api-mock:build -x test --no-daemon
+# Layer 1: build infrastructure (rarely changes)
+COPY gradlew settings.gradle.kts build.gradle.kts gradle.properties ./
+COPY gradle/ gradle/
+COPY buildSrc/ buildSrc/
+
+# Layer 2: all module build files so Gradle can configure the project graph
+COPY lib/clearinghouse/build.gradle.kts lib/clearinghouse/
+COPY lib/tsd-file-api-client/build.gradle.kts lib/tsd-file-api-client/
+COPY lib/crypt4gh/build.gradle.kts lib/crypt4gh/
+COPY services/localega-tsd-proxy/build.gradle.kts services/localega-tsd-proxy/
+COPY services/tsd-api-mock/build.gradle.kts services/tsd-api-mock/
+COPY services/cega-mock/build.gradle.kts services/cega-mock/
+COPY services/mq-interceptor/build.gradle.kts services/mq-interceptor/
+COPY e2eTests/build.gradle.kts e2eTests/
+RUN mkdir -p cli/lega-commander
+
+# Layer 3: resolve dependencies (cached until build.gradle.kts changes)
+RUN ./gradlew :services:tsd-api-mock:dependencies --no-daemon || true
+
+# Layer 4: source code (changes frequently — only what this service needs)
+COPY services/tsd-api-mock/src/ services/tsd-api-mock/src/
+
+RUN ./gradlew :services:tsd-api-mock:build -x test --no-daemon
 
 FROM eclipse-temurin:21-jre-alpine
 RUN mkdir -p /etc/jwt/public_keys
 WORKDIR /app
-COPY --from=builder app/services/tsd-api-mock/build/libs/tsd-api-mock.jar /app/app.jar
+COPY --from=builder /app/services/tsd-api-mock/build/libs/tsd-api-mock.jar /app/app.jar
 EXPOSE 8080
-CMD ["java","-XX:+UseG1GC", "-jar","app.jar"]
+CMD ["java", "-XX:+UseG1GC", "-jar", "app.jar"]