diff --git a/.github/actions/get-job-data/action.yml b/.github/actions/get-job-data/action.yml new file mode 100644 index 0000000000000..a0b15c0dd8030 --- /dev/null +++ b/.github/actions/get-job-data/action.yml @@ -0,0 +1,93 @@ +name: 'Get Job Data from GitHub Actions' +description: 'Fetches the GitHub Actions job data from the GitHub API' +inputs: + job-name: + description: 'The name of the job to find' + required: true + github-token: + description: 'GitHub token for API authentication' + required: true + repository: + description: 'Repository in owner/repo format' + required: true + run-id: + description: 'GitHub Actions run ID' + required: true +outputs: + job_html_url: + description: 'The HTML URL of the job' + value: ${{ steps.get_url.outputs.job_html_url }} +runs: + using: 'composite' + steps: + - name: Fetch job URL from GitHub API + id: get_url + shell: bash + run: | + # Fetch the numeric job ID from GitHub API + CURL_ERROR_FILE=$(mktemp) + NETRC_FILE=$(mktemp) + + # Write GitHub API credentials to a temporary netrc file to avoid + # passing the token directly on the curl command line. + printf '%s\n' \ + 'machine api.github.com' \ + ' login x-access-token' \ + " password ${{ inputs.github-token }}" \ + > "$NETRC_FILE" + chmod 600 "$NETRC_FILE" + + # Ensure temporary file cleanup on exit + cleanup() { + if [ -n "$CURL_ERROR_FILE" ] && [ -f "$CURL_ERROR_FILE" ]; then + rm -f "$CURL_ERROR_FILE" + fi + if [ -n "$NETRC_FILE" ] && [ -f "$NETRC_FILE" ]; then + rm -f "$NETRC_FILE" + fi + } + trap cleanup EXIT + + API_RESPONSE=$(curl -sS -w "\n%{http_code}" \ + --netrc-file "$NETRC_FILE" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ inputs.repository }}/actions/runs/${{ inputs.run-id }}/jobs" 2>"$CURL_ERROR_FILE") + + CURL_EXIT_CODE=$? + if [ $CURL_EXIT_CODE -ne 0 ]; then + echo "❌ ERROR: curl request to GitHub API failed with exit code $CURL_EXIT_CODE" + if [ -s "$CURL_ERROR_FILE" ]; then + echo "curl error output:" + cat "$CURL_ERROR_FILE" + fi + echo "job_html_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + HTTP_CODE=$(echo "$API_RESPONSE" | tail -n1) + RESPONSE_BODY=$(echo "$API_RESPONSE" | sed '$d') + + if [ "$HTTP_CODE" != "200" ]; then + echo "⚠️ WARNING: GitHub API request failed with $HTTP_CODE" + echo "job_html_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + EXPECTED_JOB_NAME="${{ inputs.job-name }}" + JOB_URL=$(echo "$RESPONSE_BODY" | jq -r \ + --arg job_name "$EXPECTED_JOB_NAME" \ + '.jobs[] | select(.name == $job_name) | .html_url') + + if [ -z "$JOB_URL" ] || [ "$JOB_URL" = "null" ]; then + echo "⚠️ WARNING: Failed to extract job URL from response for job name '$EXPECTED_JOB_NAME'." + echo "Possible causes:" + echo " - The job name does not match exactly (including spaces and case)." + echo " - The job has not started yet at the time this action ran." + echo " - The GitHub API response format was unexpected." + echo "Please verify that the job has started before this action runs and double-check the exact job name in the GitHub Actions UI." + echo "job_html_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "job_html_url=$JOB_URL" >> "$GITHUB_OUTPUT" + echo "Job URL: $JOB_URL" diff --git a/.github/scripts/detect-app-cache.sh b/.github/scripts/detect-app-cache.sh index 293383c31cfb0..8af75b833ad5b 100644 --- a/.github/scripts/detect-app-cache.sh +++ b/.github/scripts/detect-app-cache.sh @@ -17,7 +17,8 @@ set -o pipefail # Exit if any command in pipeline fails : "${FORCE_REBUILD:?FORCE_REBUILD not set}" : "${ARTIFACTORY_REPOSITORY_SNAPSHOT:?ARTIFACTORY_REPOSITORY_SNAPSHOT not set}" -# Optional JFrog variables +# Optional variables +APPS_TO_REBUILD="${APPS_TO_REBUILD:-}" JF_URL="${JF_URL:-}" JF_USER="${JF_USER:-}" JF_ACCESS_TOKEN="${JF_ACCESS_TOKEN:-}" @@ -34,7 +35,11 @@ JF_ACCESS_TOKEN="${JF_ACCESS_TOKEN:-}" # - has_apps_to_restore: boolean flag echo "Collecting app SHAs and checking cache status..." +echo "Cache version: $CACHE_VERSION" echo "Force rebuild mode: $FORCE_REBUILD" +if [ -n "$APPS_TO_REBUILD" ]; then + echo "Apps to rebuild: $APPS_TO_REBUILD" +fi echo "" # Setup JFrog CLI if credentials are available @@ -92,10 +97,16 @@ echo "" echo "### 📦 Cache Status Report for ($GITHUB_REF)" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" +echo "**Cache Version:** \`$CACHE_VERSION\`" >> "$GITHUB_STEP_SUMMARY" +echo "" >> "$GITHUB_STEP_SUMMARY" if [ "$FORCE_REBUILD" == "true" ]; then echo "**🔄 FORCE REBUILD MODE ENABLED** - All caches bypassed" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" fi +if [ -n "$APPS_TO_REBUILD" ]; then + echo "**🔨 Specific apps to rebuild:** \`$APPS_TO_REBUILD\`" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi if [ "$JFROG_AVAILABLE" == "true" ]; then echo "**🎯 JFrog Artifact Cache**: Enabled for all branches" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" @@ -103,6 +114,17 @@ fi echo "| App | SHA | Cache Key | Status |" >> "$GITHUB_STEP_SUMMARY" echo "|-----|-----|-----------|--------|" >> "$GITHUB_STEP_SUMMARY" +# Convert comma-separated apps list to array for easier checking +if [ -n "$APPS_TO_REBUILD" ]; then + IFS=',' read -ra REBUILD_APPS_ARRAY <<< "$APPS_TO_REBUILD" + # Trim whitespace from each app name using xargs for readability + for i in "${!REBUILD_APPS_ARRAY[@]}"; do + REBUILD_APPS_ARRAY[$i]=$(echo "${REBUILD_APPS_ARRAY[$i]}" | xargs) + done +else + REBUILD_APPS_ARRAY=() +fi + # Iterate through each app in the matrix while IFS= read -r app_json; do APP_NAME=$(echo "$app_json" | jq -r '.name') @@ -148,6 +170,23 @@ while IFS= read -r app_json; do continue fi + # Check if this specific app should be rebuilt (ignore cache) + REBUILD_THIS_APP=false + for rebuild_app in "${REBUILD_APPS_ARRAY[@]}"; do + if [ "$APP_NAME" == "$rebuild_app" ]; then + REBUILD_THIS_APP=true + break + fi + done + + if [ "$REBUILD_THIS_APP" == "true" ]; then + echo "🔨 rebuild requested" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔨 Rebuild requested |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + # Check JFrog first before GitHub cache (available for all branches) if [ "$JFROG_AVAILABLE" == "true" ]; then JFROG_PATH="${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${CURRENT_SHA}.tar.gz" diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 81e553cf82091..88b8b3244831f 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -38,6 +38,16 @@ on: required: false type: boolean default: false + cache_version_suffix: + description: 'Optional cache version suffix (e.g., "test", "debug") - creates separate cache namespace' + required: false + type: string + default: '' + apps_to_rebuild: + description: 'Comma-separated list of specific apps to rebuild (e.g., "app1,app2") - ignores cache for these apps only' + required: false + type: string + default: '' concurrency: group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/ionos-dev' && github.run_id || github.event.pull_request.number || github.ref }} @@ -66,7 +76,16 @@ jobs: apps_sha_map: ${{ steps.detect.outputs.apps_sha_map }} has_apps_to_build: ${{ steps.detect.outputs.has_apps_to_build }} has_apps_to_restore: ${{ steps.detect.outputs.has_apps_to_restore }} + effective_cache_version: ${{ steps.compute_cache_version.outputs.effective_cache_version }} steps: + - name: Compute effective cache version + id: compute_cache_version + run: | + # Compute cache version with optional suffix to create a single source of truth + EFFECTIVE_VERSION="${{ env.CACHE_VERSION }}${{ github.event.inputs.cache_version_suffix && format('-{0}', github.event.inputs.cache_version_suffix) || '' }}" + echo "effective_cache_version=$EFFECTIVE_VERSION" >> "$GITHUB_OUTPUT" + echo "Effective cache version: $EFFECTIVE_VERSION" + - name: Checkout repository uses: actions/checkout@v5 with: @@ -182,14 +201,21 @@ jobs: id: detect env: GH_TOKEN: ${{ github.token }} - CACHE_VERSION: ${{ env.CACHE_VERSION }} + CACHE_VERSION: ${{ steps.compute_cache_version.outputs.effective_cache_version }} FORCE_REBUILD: ${{ github.event.inputs.force_rebuild || 'false' }} + APPS_TO_REBUILD: ${{ github.event.inputs.apps_to_rebuild || '' }} JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} ARTIFACTORY_REPOSITORY_SNAPSHOT: ${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }} GITHUB_REF: ${{ github.ref }} run: | + # Show effective cache version + echo "Base CACHE_VERSION: ${{ env.CACHE_VERSION }}" + echo "Cache version suffix: ${{ github.event.inputs.cache_version_suffix || '(none)' }}" + echo "Effective CACHE_VERSION: ${CACHE_VERSION}" + echo "Apps to rebuild: ${{ github.event.inputs.apps_to_rebuild || '(none)' }}" + echo "" bash .github/scripts/detect-app-cache.sh '${{ steps.set_matrix.outputs.matrix }}' build-external-apps: @@ -287,7 +313,7 @@ jobs: uses: actions/cache/save@v4 with: path: ${{ steps.app-config.outputs.path }} - key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app_info.name }}-${{ matrix.app_info.sha }} + key: ${{ needs.prepare-matrix.outputs.effective_cache_version }}-app-build-${{ matrix.app_info.name }}-${{ matrix.app_info.sha }} # Push to JFrog for ionos-dev branch builds - name: Setup JFrog CLI @@ -302,6 +328,16 @@ jobs: # Ping the server jf rt ping + - name: Get Job data + id: get_job_data + continue-on-error: true + uses: ./.github/actions/get-job-data + with: + job-name: 'build-external-apps (${{ matrix.app_info.name }}, ${{ matrix.app_info.sha }})' + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} + - name: Push ${{ matrix.app_info.name }} to JFrog run: | set -e @@ -309,11 +345,15 @@ jobs: APP_SHA="${{ matrix.app_info.sha }}" APP_PATH="${{ steps.app-config.outputs.path }}" + # Use precomputed effective cache version + EFFECTIVE_CACHE_VERSION="${{ needs.prepare-matrix.outputs.effective_cache_version }}" + echo "=== JFrog Upload Debug Info ===" echo "📦 Packaging $APP_NAME for JFrog upload..." echo "App Name: $APP_NAME" echo "App SHA: $APP_SHA" echo "App Path: $APP_PATH" + echo "Cache Version: $EFFECTIVE_CACHE_VERSION" echo "Repository: ${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}" echo "===============================" @@ -345,16 +385,33 @@ jobs: # Upload to JFrog - store in snapshot repo under dev/apps/ # Include CACHE_VERSION in path to enable complete cache invalidation - JFROG_PATH="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/apps/${{ env.CACHE_VERSION }}/${APP_NAME}/${ARCHIVE_NAME}" + JFROG_PATH="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/apps/${EFFECTIVE_CACHE_VERSION}/${APP_NAME}/${ARCHIVE_NAME}" + + # Build properties for artifact metadata + JFROG_PROPS_LIST=() + JFROG_PROPS_LIST+=("app.name=${APP_NAME}") + JFROG_PROPS_LIST+=("app.sha=${APP_SHA}") + JFROG_PROPS_LIST+=("vcs.branch=${{ github.ref_name }}") + JFROG_PROPS_LIST+=("vcs.revision=${{ github.sha }}") + + # Add job URL if available + JOB_URL="${{ steps.get_job_data.outputs.job_html_url }}" + if [ -n "$JOB_URL" ]; then + JFROG_PROPS_LIST+=("job.html_url=${JOB_URL}") + fi + + # Join properties into a single semicolon-separated string using IFS and array expansion + # This bash technique temporarily sets IFS (Internal Field Separator) to ';' and expands + # the array with [*] which joins elements using the first character of IFS + JFROG_PROPS=$(IFS=';'; printf '%s' "${JFROG_PROPS_LIST[*]}") echo "" echo "Uploading to JFrog..." echo "Target Path: $JFROG_PATH" - echo "Properties: app.name=${APP_NAME};app.sha=${APP_SHA};vcs.branch=${{ github.ref_name }};vcs.revision=${{ github.sha }}" + echo "Properties: $JFROG_PROPS" echo "Running: jf rt upload \"$ARCHIVE_NAME\" \"$JFROG_PATH\" --target-props \"...\"" - if jf rt upload "$ARCHIVE_NAME" "$JFROG_PATH" \ - --target-props "app.name=${APP_NAME};app.sha=${APP_SHA};vcs.branch=${{ github.ref_name }};vcs.revision=${{ github.sha }}"; then + if jf rt upload "$ARCHIVE_NAME" "$JFROG_PATH" --target-props "$JFROG_PROPS"; then echo "✅ Successfully uploaded $APP_NAME to JFrog" echo "" echo "Verifying upload..." @@ -637,6 +694,12 @@ jobs: BUILD_NAME: "nextcloud-workspace-snapshot" steps: + # Checkout is required to access the local composite action at ./.github/actions/get-job-data + - name: Checkout repository + uses: actions/checkout@v5 + with: + fetch-depth: 1 + - name: Check prerequisites run: | # count the number of secrets that are set @@ -682,6 +745,16 @@ jobs: # Ping the server jf rt ping + - name: Get Job data + id: get_job_data + continue-on-error: true + uses: ./.github/actions/get-job-data + with: + job-name: 'Push to artifactory' + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} + - name: Upload build to artifactory id: artifactory_upload run: | @@ -704,6 +777,23 @@ jobs: export PATH_TO_LATEST_ARTIFACT="${PATH_TO_DIRECTORY}/${PATH_TO_FILE}" + # Build properties for artifact metadata + JFROG_PROPS_LIST=() + JFROG_PROPS_LIST+=("build.nc_version=${{ needs.build-artifact.outputs.NC_VERSION }}") + JFROG_PROPS_LIST+=("vcs.branch=${{ github.ref_name }}") + JFROG_PROPS_LIST+=("vcs.revision=${{ github.sha }}") + + # Add job URL if available + JOB_URL="${{ steps.get_job_data.outputs.job_html_url }}" + if [ -n "$JOB_URL" ]; then + JFROG_PROPS_LIST+=("job.html_url=${JOB_URL}") + fi + + # Join properties into a single semicolon-separated string using IFS and array expansion + # This bash technique temporarily sets IFS (Internal Field Separator) to ';' and expands + # the array with [*] which joins elements using the first character of IFS + JFROG_PROPS=$(IFS=';'; printf '%s' "${JFROG_PROPS_LIST[*]}") + # Upload with retry logic (3 attempts with 30s delay) MAX_ATTEMPTS=3 ATTEMPT=1 @@ -716,7 +806,7 @@ jobs: if jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ --build-name "${{ env.BUILD_NAME }}" \ --build-number ${{ github.run_number }} \ - --target-props "build.nc_version=${{ needs.build-artifact.outputs.NC_VERSION }};vcs.branch=${{ github.ref }};vcs.revision=${{ github.sha }}" \ + --target-props "$JFROG_PROPS" \ $PATH_TO_LATEST_ARTIFACT; then UPLOAD_SUCCESS=true echo "✅ Upload successful on attempt $ATTEMPT" @@ -843,6 +933,10 @@ jobs: BUILD_TYPE="stable" fi + # Construct source build URL for traceability + SOURCE_BUILD_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "Source Build URL: $SOURCE_BUILD_URL" + # Trigger GitLab pipeline via webhook with retry logic (3 attempts with 30s delay) MAX_ATTEMPTS=3 ATTEMPT=1 @@ -865,6 +959,7 @@ jobs: --form "variables[NC_VERSION]=${{ needs.build-artifact.outputs.NC_VERSION }}" \ --form "variables[BUILD_ID]=${{ github.run_id }}" \ --form "variables[BUILD_TYPE]=${BUILD_TYPE}" \ + --form "variables[SOURCE_BUILD_URL]=${SOURCE_BUILD_URL}" \ "${{ secrets.GITLAB_TRIGGER_URL }}"; then TRIGGER_SUCCESS=true echo "✅ Trigger successful on attempt $ATTEMPT" @@ -1111,9 +1206,12 @@ jobs: echo "" echo "### Workflow Inputs (if workflow_dispatch)" echo "Force Rebuild: ${{ github.event.inputs.force_rebuild || 'N/A' }}" + echo "Cache Version Suffix: ${{ github.event.inputs.cache_version_suffix || 'N/A' }}" + echo "Apps to Rebuild: ${{ github.event.inputs.apps_to_rebuild || 'N/A' }}" echo "" echo "### Environment Variables" - echo "CACHE_VERSION: ${{ env.CACHE_VERSION }}" + echo "Base CACHE_VERSION: ${{ env.CACHE_VERSION }}" + echo "Effective CACHE_VERSION: ${{ needs.prepare-matrix.outputs.effective_cache_version }}" echo "TARGET_PACKAGE_NAME: ${{ env.TARGET_PACKAGE_NAME }}" echo "ARTIFACTORY_REPOSITORY_SNAPSHOT: ${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}" echo "" diff --git a/.gitmodules b/.gitmodules index 8011d8c0460ec..a7c0d60e7e858 100644 --- a/.gitmodules +++ b/.gitmodules @@ -88,3 +88,6 @@ [submodule "apps-external/twofactor_totp"] path = apps-external/twofactor_totp url = git@github.com:nextcloud/twofactor_totp.git +[submodule "apps-external/ncw_tools"] + path = apps-external/ncw_tools + url = git@github.com:IONOS-Productivity/ncw-tools.git diff --git a/IONOS b/IONOS index 3d870c64bb27b..5823634d2b872 160000 --- a/IONOS +++ b/IONOS @@ -1 +1 @@ -Subproject commit 3d870c64bb27b778439108ddc3ddb524432f1be2 +Subproject commit 5823634d2b872b8c116350bacc3ccc08836436db diff --git a/apps-external/assistant b/apps-external/assistant index 772c2006b5af3..a53bb2a710cc8 160000 --- a/apps-external/assistant +++ b/apps-external/assistant @@ -1 +1 @@ -Subproject commit 772c2006b5af3cc7bb802e1d54c1685806d7ed07 +Subproject commit a53bb2a710cc834720bc8532d8a80c3d0d4b0257 diff --git a/apps-external/mail b/apps-external/mail index 71abf4ac19487..da047107868b3 160000 --- a/apps-external/mail +++ b/apps-external/mail @@ -1 +1 @@ -Subproject commit 71abf4ac194871f024f765167716cc490ab242e4 +Subproject commit da047107868b328ceaf08f9ecc2e1ab84327125c diff --git a/apps-external/ncw_tools b/apps-external/ncw_tools new file mode 160000 index 0000000000000..3af182e92b7c5 --- /dev/null +++ b/apps-external/ncw_tools @@ -0,0 +1 @@ +Subproject commit 3af182e92b7c529e3b42f4d0dce090945efbcb46