Merge branch 'revert-cache-changes' into 'master'

Revert recently introduced cache changes

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7521

Merged-by: Andrew Patterson <[email protected]>
Approved-by: Jenny Kim <[email protected]>
Approved-by: Andrew Patterson <[email protected]>
Co-authored-by: Robert Marshall <[email protected]>

Author: Andrew Patterson

.gitlab-ci.yml

@@ -60,7 +60,6 @@ variables:
   DOCS_GITLAB_REPO_SUFFIX: "omnibus"
   CACHE_KEY_SUFFIX: '-v3'
   CACHE_EDITION: "CE"
-  CACHE_POLICY: 'pull-push'
   ISSUE_BOT_LABELS_EXTRA: "group::distribution"
   BUNDLER_VERSION: "2.5.6"
   # NOTE: When `NEXT_RUBY_VERSION` is updated, flip

gitlab-ci-config/dev-gitlab-org.yml

@@ -34,13 +34,11 @@
 .branch-cache: &branch-cache
   key: "${CI_JOB_NAME}-${BUILDER_IMAGE_REVISION}-${CACHE_EDITION}${CACHE_KEY_SUFFIX}"
   paths: !reference [.cache-paths]
-  policy: $CACHE_POLICY
 
 .tag-cache: &tag-cache
   # Use the same cache key as branch builds, by manually appending `-branch` substring.
   key: "${CI_JOB_NAME}-branch-${BUILDER_IMAGE_REVISION}-${CACHE_EDITION}${CACHE_KEY_SUFFIX}"
   paths: !reference [.cache-paths]
-  policy: $CACHE_POLICY
 
 .branch_template: &branch_build
   stage: package
@@ -284,11 +282,6 @@ Ubuntu-20.04-branch:
 Ubuntu-22.04-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
   extends: .branch_template
-  rules:
-    - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
 
 Debian-10-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/debian_10:${BUILDER_IMAGE_REVISION}"
@@ -314,12 +307,6 @@ Ubuntu-20.04-arm64-branch:
 Ubuntu-22.04-arm64-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04_arm64:${BUILDER_IMAGE_REVISION}"
   extends: .arm64_branch_template
-  rules:
-    - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-      when: manual
-      allow_failure: true
 CentOS-7-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/centos_7:${BUILDER_IMAGE_REVISION}"
   extends: .branch_template
@@ -381,15 +368,6 @@ Ubuntu-18.04-fips-branch:
 Ubuntu-20.04-fips-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04_fips:${BUILDER_IMAGE_REVISION}"
   extends: .fips_branch_template
-  rules:
-    - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE == "EE_NIGHTLY_BUILD_PIPELINE"'
-    - if: '$PIPELINE_TYPE == "EE_BRANCH_BUILD_PIPELINE"'
-      when: manual
-      allow_failure: true
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-      when: manual
-      allow_failure: true
 
 Docker-branch:
   extends: .docker_job
@@ -410,8 +388,6 @@ Docker-branch:
   rules:
     - !reference [.default_rules, rules]
     - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
 
 QA-branch:
   extends: .docker_job

gitlab-ci-config/gitlab-com.yml

@@ -42,7 +42,7 @@
       - gems
       - assets_cache
       - node_modules
-    policy: $CACHE_POLICY
+    policy: pull
 
 .trigger-fips-package-cache:
   cache:
@@ -52,7 +52,7 @@
       - gems
       - assets_cache
       - node_modules
-    policy: $CACHE_POLICY
+    policy: pull
 
 .install-gems: &install-gems
   - gem install bundler:${BUNDLER_VERSION}
@@ -133,8 +133,6 @@
       junit: junit_rspec.xml
 
 .base-trigger-job-variables:
-  # In trigger jobs, we don't want to pollute cache
-  CACHE_POLICY: 'pull'
   ALTERNATIVE_SOURCES: 'true'
   SECURITY_SOURCES: ${SECURITY_SOURCES}
   BUILDER_IMAGE_REVISION: ${BUILDER_IMAGE_REVISION}
@@ -180,6 +178,17 @@
       when: manual
       variables: !reference [.ce-trigger-job-variables]
 
+.trigger-pipeline:
+  stage: trigger-qa
+  image: "${RUBY_IMAGE}"
+  allow_failure: true
+  rules:
+    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
+      when: manual
+  needs:
+    - job: Trigger:package
+      artifacts: false
+
 .review-docs:
   image: "${RUBY_IMAGE}-alpine"
   stage: post-test
@@ -250,6 +259,22 @@ update-gems-cache-for-docker-jobs:
     - update-gems-cache
     - .docker_job
 
+update-trigger-package-cache:
+  extends: .trigger-package-cache
+  stage: update-cache
+  image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
+  script:
+    - !reference [.build-package]
+    - echo "Cache is up to date!"
+  cache:
+    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+  tags: !reference [.distribution-amd64-tags]
+  rules:
+    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
+  needs:
+    - job: fetch-assets
+      optional: true
+
 ###########################
 #     Branch pipeline     #
 ###########################
@@ -501,6 +526,25 @@ update-knapsack:
 #     Trigger Pipeline     #
 ############################
 
+Trigger:package:
+  extends: .trigger-package-common
+  variables:
+    BASE_OS: "ubuntu_22.04"
+    PACKAGE_DIRECTORY: "pkg/ubuntu-jammy"
+
+Trigger:package:fips:
+  extends:
+    - .trigger-package-common
+    - .trigger-fips-package-cache
+  variables:
+    USE_SYSTEM_SSL: "true"
+    BASE_OS: "ubuntu_20.04_fips"
+    PACKAGE_DIRECTORY: "pkg/ubuntu-focal_fips"
+  allow_failure: true
+  rules:
+    - if: '$PIPELINE_TYPE == "TRIGGERED_EE_PIPELINE"'
+      when: manual
+
 package_size_check:
   extends: .trigger-package-cache
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
@@ -509,11 +553,29 @@ package_size_check:
     - bundle exec rake build:package:generate_sizefile
     - bundle exec rake check:package_size
   needs:
-    - job: Ubuntu-22.04-branch
+    - job: Trigger:package
       artifacts: false
   rules:
     - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
 
+Trigger:gitlab-docker:
+  extends:
+    - .docker_job
+    - .gems-cache
+  stage: trigger-docker
+  script:
+    - bundle exec rake docker:build:image
+    - bundle exec rake docker:push:triggered
+  rules:
+    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
+    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
+  needs:
+    - job: Trigger:package
+      artifacts: false
+    - job: generate-facts
+      optional: true
+      artifacts: true
+
 qa-subset-test:
   extends:
     - .qa-template
@@ -540,10 +602,9 @@ qa-subset-test:
   needs:
     - job: generate-facts
       artifacts: true
-    - job: Ubuntu-22.04-branch
+    - job: Trigger:package
       artifacts: false
-    - job: Docker-branch
-      optional: true
+    - job: Trigger:gitlab-docker
       artifacts: false
 
 qa-remaining-test-manual:
@@ -568,10 +629,9 @@ qa-remaining-test-manual:
   needs:
     - job: generate-facts
       artifacts: true
-    - job: Ubuntu-22.04-branch
+    - job: Trigger:package
       artifacts: false
-    - job: Docker-branch
-      optional: true
+    - job: Trigger:gitlab-docker
       artifacts: false
 
 letsencrypt-test:
@@ -587,7 +647,7 @@ letsencrypt-test:
       when: never
     - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
   needs:
-    - job: Docker-branch
+    - job: Trigger:gitlab-docker
       artifacts: false
 
 RAT:
@@ -612,6 +672,10 @@ RAT:
   needs:
     - job: Ubuntu-22.04-branch
       artifacts: false
+      optional: true
+    - job: Trigger:package
+      artifacts: false
+      optional: true
     - job: generate-facts
       artifacts: true
 
@@ -623,6 +687,10 @@ RAT:FIPS:
   needs:
     - job: Ubuntu-20.04-fips-branch
       artifacts: false
+      optional: true
+    - job: Trigger:package:fips
+      artifacts: false
+      optional: true
     - job: generate-facts
       artifacts: true
 
@@ -644,7 +712,7 @@ GET:Geo:
       when: manual
       allow_failure: true
   needs:
-    - job: Ubuntu-22.04-branch
+    - job: Trigger:package
       artifacts: false
     - job: generate-facts
       artifacts: true
@@ -725,7 +793,7 @@ pages:
 
 .build-package-on-all-os-vars:
   extends: .base-trigger-job-variables
-  SKIP_JOB_REGEX: '/Ubuntu-22.04|Ubuntu-20.04-fips|Docker|QA/'
+  SKIP_JOB_REGEX: '/Ubuntu-22.04|Docker|QA/'
   PIPELINE_TYPE: "${EDITION}_BRANCH_BUILD_PIPELINE"
   CACHE_EDITION: ${EDITION}
   ee: ${ee}

gitlab-ci-config/workflow-rules.yml

@@ -28,8 +28,6 @@ workflow:
       variables:
         PIPELINE_TYPE: TRIGGER_CACHE_UPDATE_PIPELINE
         CACHE_EDITION: "GITLAB"
-        # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
-        CACHE_POLICY: 'push'
     - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $QA_PROJECT_PATH && $CACHE_UPDATE == "true"'
       variables:
         PIPELINE_TYPE: TRIGGER_CACHE_UPDATE_PIPELINE

lib/gitlab/build/facts.rb

@@ -91,8 +91,8 @@ def qa_trigger_vars
           GITLAB_SEMVER_VERSION=#{Build::Info::Git.latest_stable_tag.tr('+', '-')}
           RAT_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs'}
           RAT_FIPS_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_FIPS_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs-fips-ubuntu'}
-          RAT_PACKAGE_URL=#{Build::Info::CI.package_download_url(fips: false)}
-          RAT_FIPS_PACKAGE_URL=#{Build::Info::CI.package_download_url(job_name: 'Ubuntu-20.04', fips: true)}
+          RAT_PACKAGE_URL=#{Build::Info::CI.package_download_url}
+          RAT_FIPS_PACKAGE_URL=#{Build::Info::CI.fips_package_download_url}
         ]
       end
 

lib/gitlab/build/info/ci.rb

@@ -38,6 +38,26 @@ def commit_ref_slug
           Gitlab::Util.get_env('CI_COMMIT_REF_SLUG')
         end
 
+        def package_download_url
+          return Gitlab::Util.get_env('PACKAGE_URL') if Gitlab::Util.get_env('PACKAGE_URL')
+
+          # For builds running in dev.gitlab.org, or nightly pipelines in .com,
+          # use the artifact from `Ubuntu-22.04-branch` job. For the other build,
+          # which is essentially in triggered pipeline, use artifact from
+          # `Trigger:package` job.
+          if /dev.gitlab.org/.match?(Build::Info::CI.api_v4_url) || Build::Check.is_nightly?
+            Build::Info::CI.branch_build_package_download_url
+          else
+            Build::Info::CI.triggered_package_download_url(fips: false)
+          end
+        end
+
+        def fips_package_download_url
+          return Gitlab::Util.get_env('FIPS_PACKAGE_URL') if Gitlab::Util.get_env('FIPS_PACKAGE_URL')
+
+          Build::Info::CI.triggered_package_download_url(fips: false)
+        end
+
         def artifact_url(job_name, file_path)
           client = Gitlab::APIClient.new
           target_job_id = client.get_job_id(job_name)
@@ -47,10 +67,15 @@ def artifact_url(job_name, file_path)
           URI("#{api_v4_url}/projects/#{project_id}/jobs/#{target_job_id}/artifacts/#{file_path}")
         end
 
-        def package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64', fips: Build::Check.use_system_ssl?)
-          return Gitlab::Util.get_env('FIPS_PACKAGE_URL') if fips && Gitlab::Util.get_env('FIPS_PACKAGE_URL')
-          return Gitlab::Util.get_env('PACKAGE_URL') if !fips && Gitlab::Util.get_env('PACKAGE_URL')
+        def triggered_package_download_url(fips: Build::Check.use_system_ssl?)
+          folder = fips ? 'ubuntu-focal_fips' : 'ubuntu-jammy'
+          job_name = fips ? 'Trigger:package:fips' : 'Trigger:package'
+          package_path = "pkg/#{folder}/gitlab.deb"
+
+          artifact_url(job_name, package_path)
+        end
 
+        def branch_build_package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64')
           case job_name
           when /AlmaLinux-8/
             # In EL world, amd64 is called x86_64
@@ -70,8 +95,6 @@ def package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64', fips: Build::C
             folder = "#{folder}_aarch64"
           end
 
-          job_name = "#{job_name}-fips" if fips
-
           job_name = "#{job_name}-branch" unless Build::Info::CI.tag_name
 
           package_path = "pkg/#{folder}/#{package_file_name}"

lib/gitlab/package_size.rb

@@ -13,7 +13,7 @@ class PackageSizeCheck
 
   class << self
     def fetch_sizefile
-      sizefile_url = URI("#{Build::Info::CI.package_download_url}.size")
+      sizefile_url = Build::Info::CI.artifact_url("Trigger:package", "pkg/ubuntu-jammy/gitlab.deb.size")
 
       # We have to use net/http here because `gitlab` gem's `download_job_artifact_file`
       # method doesn't support plain text files. It has to be either binary or valid JSON.

spec/lib/gitlab/build/image_spec.rb

@@ -159,7 +159,7 @@ def self.dockerhub_image_name
             "PACKAGECLOUD_REPO=download-package",
             "RELEASE_PACKAGE=gitlab-ce",
             "RELEASE_VERSION=12.121.12-ce.0",
-            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab-ce_12.121.12-ce.0_amd64.deb",
+            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab.deb",
             "CI_JOB_TOKEN=NOT-CI-JOB-TOKEN\n"
           ].join("\n")
         end
@@ -180,7 +180,7 @@ def self.dockerhub_image_name
             "PACKAGECLOUD_REPO=download-package",
             "RELEASE_PACKAGE=gitlab-ee",
             "RELEASE_VERSION=12.121.12-ee.0",
-            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab-ee_12.121.12-ee.0_amd64.deb",
+            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab.deb",
             "CI_JOB_TOKEN=NOT-CI-JOB-TOKEN\n"
           ].join("\n")
         end