commit analysis

.codecov.yaml

@@ -0,0 +1,20 @@
+github_checks:
+    annotations: false
+
+codecov:
+  notify:
+      after_n_builds: 3 # 3 since we upload in 3 separate test jobs (unit, integration, integrationH2)
+
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true
+
+comment:
+  layout: "diff, flags, files, footer"
+  behavior: default
+  require_changes: true

.dockerignore

@@ -0,0 +1,18 @@
+.git
+.gitignore
+**/.gitignore
+.github
+.idea
+docker
+**/*.iml
+jenkinsfiles
+.patches
+.mailmap
+.tx
+*.sh
+*.md
+LICENSE
+
+**/target
+**/src/test
+dhis-2/dhis-test-e2e

.github/workflows/analyse-pr.yml

@@ -1,58 +1,50 @@
 name: SonarQube analysis
-
+env:
+  # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  SONAR_TOKEN: ${{ secrets.DHIS2_BOT_SONARCLOUD_TOKEN }}
 on:
   push:
     branches:
       - master
       - '2.3[1-9]'
+      - '2.4[0-9]'
   pull_request:
     branches:
       - master
       - '2.3[1-9]'
+      - '2.4[0-9]'
 concurrency:
     group: ${{ github.workflow}}-${{ github.ref }}
     cancel-in-progress: true
 jobs:
   sonarqube:
     runs-on: ubuntu-latest
-
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
-
-      - name: Set up JDK 11
-        uses: actions/setup-java@v1
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
         with:
-          java-version: 11
-
-      - name: Cache maven artifacts
-        uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }}
-          restore-keys: ${{ runner.os }}-maven-
+          java-version: 17
+          distribution: temurin
+          cache: maven
 
       - name: Analyse PR
         env:
           BASE_BRANCH: ${{ github.base_ref }}
           BRANCH: ${{ github.ref }}
           PR: ${{ github.event.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          MAVEN_BUILD_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
         if: github.event_name == 'pull_request'
         run: |
-          mvn -f dhis-2/pom.xml clean install -Psonarqube -Pjdk11 $MAVEN_BUILD_OPTS
-          mvn -f dhis-2/pom.xml sonar:sonar -Dsonar.scm.revision=${{ github.event.pull_request.head.sha }} -Dsonar.projectKey=dhis2_dhis2-core -Dsonar.coverage.jacoco.xmlReportPaths=$(find "$(pwd)" -path '*jacoco.xml' | sed 's/.*/&/' | tr '\n' ',') $MAVEN_BUILD_OPTS
+          mvn -f dhis-2/pom.xml clean install --threads 2C --batch-mode --no-transfer-progress -DskipTests
+          mvn -f dhis-2/pom.xml sonar:sonar -Dsonar.internal.analysis.dbd=false --batch-mode --no-transfer-progress -Dsonar.scm.revision=${{ github.event.pull_request.head.sha }} -Dsonar.projectKey=dhis2_dhis2-core
 
       - name: Analyse long-living branch
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          MAVEN_BUILD_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
         if: github.event_name != 'pull_request'
         run: |
-          mvn -f dhis-2/pom.xml clean install -Psonarqube -Pjdk11 $MAVEN_BUILD_OPTS
-          mvn -f dhis-2/pom.xml sonar:sonar -Dsonar.branch.name=${GITHUB_REF#refs/heads/} -Dsonar.projectKey=dhis2_dhis2-core -Dsonar.coverage.jacoco.xmlReportPaths=$(find "$(pwd)" -path '*jacoco.xml' | sed 's/.*/&/' | tr '\n' ',') $MAVEN_BUILD_OPTS
+          mvn -f dhis-2/pom.xml clean install --threads 2C --batch-mode --no-transfer-progress -DskipTests
+          mvn -f dhis-2/pom.xml sonar:sonar -Dsonar.internal.analysis.dbd=false --batch-mode --no-transfer-progress -Dsonar.branch.name=${GITHUB_REF#refs/heads/} -Dsonar.projectKey=dhis2_dhis2-core

.github/workflows/check-formatting.yml

@@ -1,32 +1,26 @@
 name: Check formatting
+env:
+  # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
 
 on: [ pull_request ]
 
 jobs:
   check-formatting:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up JDK 11
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
           java-version: 11
-
-      - name: Cache maven artifacts
-        uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }}
-          restore-keys: ${{ runner.os }}-maven-
+          distribution: temurin
 
       - name: Check formatting in core
-        env:
-          # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
-          MAVEN_BUILD_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
-        run: mvn speedy-spotless:check -Pdefault -Pjdk11 --update-snapshots -q -f ./dhis-2/pom.xml $MAVEN_BUILD_OPTS
+        run: mvn spotless:check --threads 2C --batch-mode --no-transfer-progress -q -f ./dhis-2/pom.xml
 
       - name: Check formatting in web
-        env:
-          # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
-          MAVEN_BUILD_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
-        run: mvn speedy-spotless:check -Pdefault -Pjdk11 --update-snapshots -q -f ./dhis-2/dhis-web/pom.xml $MAVEN_BUILD_OPTS
+        run: mvn spotless:check --threads 2C --batch-mode --no-transfer-progress -q -f ./dhis-2/dhis-web/pom.xml
+
+      - name: Check formatting in e2e tests
+        run: mvn spotless:check --batch-mode --no-transfer-progress -q -f ./dhis-2/dhis-test-e2e/pom.xml

.github/workflows/codeql-analysis.yml

@@ -4,14 +4,19 @@
 # You may wish to alter this file to override the set of languages analyzed,
 # or to provide custom queries or build logic.
 name: "CodeQL"
+env:
+  # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
 
 on:
   pull_request:
     branches:
       - master
   schedule:
     - cron: '0 12 * * *'
-
+concurrency:
+    group: ${{ github.workflow}}-${{ github.ref }}
+    cancel-in-progress: true
 jobs:
   analyze:
     name: Analyze
@@ -28,7 +33,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           # We must fetch at least the immediate parents so that if this is
           # a pull request then we can checkout the head.
@@ -39,34 +44,23 @@ jobs:
       - run: git checkout HEAD^2
         if: ${{ github.event_name == 'pull_request' }}
 
-      - name: Cache maven artifacts
-        uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }}
-          restore-keys: ${{ runner.os }}-maven-
-
-      - name: Setup-java
-        uses: actions/setup-java@v1
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
         with:
           java-version: 11
+          distribution: temurin
+          cache: maven
 
       - name: Codeql-init
-        uses: github/codeql-action/init@v1
+        uses: github/codeql-action/init@v3
         with:
           languages: java
           config-file: ./.github/codeql/codeql-config.yml
 
       - name: Build core
-        env:
-          # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
-          MAVEN_BUILD_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
-        run: mvn clean install -f ./dhis-2/pom.xml -Pdev -Pjdk11 -DskipTests=true -Dmaven.javadoc.skip=true -B -V $MAVEN_BUILD_OPTS
+        run: mvn clean install -f ./dhis-2/pom.xml --batch-mode --no-transfer-progress -Pdev -DskipTests=true -Dmaven.javadoc.skip=true -V
       - name: Build web
-        env:
-          # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
-          MAVEN_BUILD_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
-        run: mvn clean install -f ./dhis-2/dhis-web/pom.xml -Pdev -Pjdk11 -DskipTests=true -Dmaven.javadoc.skip=true -B -V $MAVEN_BUILD_OPTS
+        run: mvn clean install -f ./dhis-2/dhis-web/pom.xml --batch-mode --no-transfer-progress -Pdev -DskipTests=true -Dmaven.javadoc.skip=true -V
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1
+        uses: github/codeql-action/analyze@v3

.github/workflows/dependabot.yml

@@ -0,0 +1,29 @@
+name: Dependabot auto-merge
+
+on:
+  pull_request:
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Fetch metadata on dependabot PR
+        id: metadata
+        uses: dependabot/[email protected]
+      - name: Approve dependabot PR using GitHub actions bot
+        if: ${{ steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'version-update:semver-patch' }}
+        run: gh pr review --approve "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      - name: Enable auto-merge for dependabot PR
+        if: ${{ steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'version-update:semver-patch' }}
+        run: gh pr merge --auto --squash "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/run-api-analytics-tests.yml

@@ -0,0 +1,91 @@
+name: Run analytics api tests
+env:
+  # This is to make sure Maven don't timeout fetching dependencies. See: https://github.com/actions/virtual-environments/issues/1499
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=125
+
+on:
+  pull_request:
+    types: [ opened, labeled, synchronize ]
+  schedule:
+    - cron: "16 1 * * *" # Run at 01:16 AM
+concurrency:
+  group: ${{ github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  api-test:
+    env:
+      CORE_IMAGE_NAME: "dhis2/core-dev:local"
+      PR_NUMBER: ${{ github.event.number }}
+      DOCKER_CHANNEL: "dhis2/core-pr"
+      SIERRA_LEONE_DB_PATH: "/tmp/db/sierra-leone"
+      SIERRA_LEONE_DB_VERSION: "2.39.0"
+
+    runs-on: ubuntu-latest
+    if: "contains(github.event.pull_request.labels.*.name, 'run-api-analytics-tests')"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: temurin
+          cache: maven
+
+      - name: Build core image
+        run: |
+          mvn clean install --threads 2C --batch-mode --no-transfer-progress -DskipTests -Dmaven.test.skip=true -f dhis-2/pom.xml -pl -dhis-web-embedded-jetty,-dhis-test-integration,-dhis-test-coverage
+          mvn clean install --threads 2C --batch-mode --no-transfer-progress -DskipTests -Dmaven.test.skip=true -f dhis-2/dhis-web/pom.xml
+          mvn --batch-mode --no-transfer-progress -DskipTests -Dmaven.test.skip=true -f ./dhis-2/dhis-web/dhis-web-portal/pom.xml jib:dockerBuild -Djib.to.image=$CORE_IMAGE_NAME
+
+      - name: Get Sierra Leone DB from cache
+        uses: actions/cache@v4
+        id: cache-sierra-leone-db
+        with:
+          path: ${{ env.SIERRA_LEONE_DB_PATH }}
+          key: sierra-leone-db-analytics-${{ env.SIERRA_LEONE_DB_VERSION }}
+
+      - name: Download Sierra Leone DB
+        if: ${{ steps.cache-sierra-leone-db.outputs.cache-hit != 'true' }}
+        run: |
+          mkdir -p ${{ env.SIERRA_LEONE_DB_PATH }}
+          wget https://databases.dhis2.org/sierra-leone/${{ env.SIERRA_LEONE_DB_VERSION }}/analytics_be/dhis2-db-sierra-leone.sql.gz --no-clobber --directory-prefix ${{ env.SIERRA_LEONE_DB_PATH }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run tests
+        run: |
+          cd dhis-2/dhis-test-e2e
+          DHIS2_IMAGE="$CORE_IMAGE_NAME" \
+          DHIS2_E2E_TEST_DUMP_PATH="$SIERRA_LEONE_DB_PATH" \
+          docker compose -f docker-compose.yml -f docker-compose.e2e.yml -f docker-compose.e2e-analytics.yml \
+            up --remove-orphans --exit-code-from test
+
+      - name: Upload logs
+        if: failure()
+        run: |
+          cd dhis-2/dhis-test-e2e
+          docker-compose logs web > ~/logs.txt
+
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: "tomcat_logs"
+          path: '~/logs.txt'
+
+  send-slack-message:
+    runs-on: ubuntu-latest
+    if: |
+      always() &&
+      contains(needs.*.result, 'failure') &&
+      github.ref == 'refs/heads/master'
+
+    needs: [ api-test ]
+    steps:
+      - uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_BACKEND_WEBHOOK }}
+          SLACK_CHANNEL: 'team-backend'
+          SLACK_MESSAGE: "Latest analytics e2e test run on master failed and needs investigation :detective-duck:. \n Commit message: ${{ github.event.head_commit.message }}"
+          SLACK_COLOR: '#ff0000'