Merge branch 'deduplicate-package-build-jobs' into 'master'

Drop trigger build jobs in favor of normal branch build jobs

Closes #6333 and #7187

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7497

Merged-by: Andrew Patterson <[email protected]>
Approved-by: Clemens Beck <[email protected]>
Approved-by: Andrew Patterson <[email protected]>
Reviewed-by: Balasankar 'Balu' C <[email protected]>
Reviewed-by: Clemens Beck <[email protected]>
Co-authored-by: Balasankar 'Balu' C <[email protected]>

Author: Andrew Patterson

.gitlab-ci.yml

@@ -60,6 +60,7 @@ variables:
   DOCS_GITLAB_REPO_SUFFIX: "omnibus"
   CACHE_KEY_SUFFIX: '-v3'
   CACHE_EDITION: "CE"
+  CACHE_POLICY: 'pull-push'
   ISSUE_BOT_LABELS_EXTRA: "group::distribution"
   BUNDLER_VERSION: "2.5.6"
   # NOTE: When `NEXT_RUBY_VERSION` is updated, flip

gitlab-ci-config/dev-gitlab-org.yml

@@ -34,11 +34,13 @@
 .branch-cache: &branch-cache
   key: "${CI_JOB_NAME}-${BUILDER_IMAGE_REVISION}-${CACHE_EDITION}${CACHE_KEY_SUFFIX}"
   paths: !reference [.cache-paths]
+  policy: $CACHE_POLICY
 
 .tag-cache: &tag-cache
   # Use the same cache key as branch builds, by manually appending `-branch` substring.
   key: "${CI_JOB_NAME}-branch-${BUILDER_IMAGE_REVISION}-${CACHE_EDITION}${CACHE_KEY_SUFFIX}"
   paths: !reference [.cache-paths]
+  policy: $CACHE_POLICY
 
 .branch_template: &branch_build
   stage: package
@@ -282,6 +284,11 @@ Ubuntu-20.04-branch:
 Ubuntu-22.04-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
   extends: .branch_template
+  rules:
+    - !reference [.default_rules, rules]
+    - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
+    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
+    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
 
 Debian-10-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/debian_10:${BUILDER_IMAGE_REVISION}"
@@ -307,6 +314,12 @@ Ubuntu-20.04-arm64-branch:
 Ubuntu-22.04-arm64-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04_arm64:${BUILDER_IMAGE_REVISION}"
   extends: .arm64_branch_template
+  rules:
+    - !reference [.default_rules, rules]
+    - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
+    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
+      when: manual
+      allow_failure: true
 CentOS-7-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/centos_7:${BUILDER_IMAGE_REVISION}"
   extends: .branch_template
@@ -368,6 +381,15 @@ Ubuntu-18.04-fips-branch:
 Ubuntu-20.04-fips-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04_fips:${BUILDER_IMAGE_REVISION}"
   extends: .fips_branch_template
+  rules:
+    - !reference [.default_rules, rules]
+    - if: '$PIPELINE_TYPE == "EE_NIGHTLY_BUILD_PIPELINE"'
+    - if: '$PIPELINE_TYPE == "EE_BRANCH_BUILD_PIPELINE"'
+      when: manual
+      allow_failure: true
+    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
+      when: manual
+      allow_failure: true
 
 Docker-branch:
   extends: .docker_job
@@ -388,6 +410,7 @@ Docker-branch:
   rules:
     - !reference [.default_rules, rules]
     - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
+    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
 
 QA-branch:
   extends: .docker_job

gitlab-ci-config/gitlab-com.yml

@@ -42,7 +42,7 @@
       - gems
       - assets_cache
       - node_modules
-    policy: pull
+    policy: $CACHE_POLICY
 
 .trigger-fips-package-cache:
   cache:
@@ -52,7 +52,7 @@
       - gems
       - assets_cache
       - node_modules
-    policy: pull
+    policy: $CACHE_POLICY
 
 .install-gems: &install-gems
   - gem install bundler:${BUNDLER_VERSION}
@@ -133,6 +133,8 @@
       junit: junit_rspec.xml
 
 .base-trigger-job-variables:
+  # In trigger jobs, we don't want to pollute cache
+  CACHE_POLICY: 'pull'
   ALTERNATIVE_SOURCES: 'true'
   SECURITY_SOURCES: ${SECURITY_SOURCES}
   BUILDER_IMAGE_REVISION: ${BUILDER_IMAGE_REVISION}
@@ -178,17 +180,6 @@
       when: manual
       variables: !reference [.ce-trigger-job-variables]
 
-.trigger-pipeline:
-  stage: trigger-qa
-  image: "${RUBY_IMAGE}"
-  allow_failure: true
-  rules:
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-      when: manual
-  needs:
-    - job: Trigger:package
-      artifacts: false
-
 .review-docs:
   image: "${RUBY_IMAGE}-alpine"
   stage: post-test
@@ -259,22 +250,6 @@ update-gems-cache-for-docker-jobs:
     - update-gems-cache
     - .docker_job
 
-update-trigger-package-cache:
-  extends: .trigger-package-cache
-  stage: update-cache
-  image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
-  script:
-    - !reference [.build-package]
-    - echo "Cache is up to date!"
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
-  tags: !reference [.distribution-amd64-tags]
-  rules:
-    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
-  needs:
-    - job: fetch-assets
-      optional: true
-
 ###########################
 #     Branch pipeline     #
 ###########################
@@ -526,25 +501,6 @@ update-knapsack:
 #     Trigger Pipeline     #
 ############################
 
-Trigger:package:
-  extends: .trigger-package-common
-  variables:
-    BASE_OS: "ubuntu_22.04"
-    PACKAGE_DIRECTORY: "pkg/ubuntu-jammy"
-
-Trigger:package:fips:
-  extends:
-    - .trigger-package-common
-    - .trigger-fips-package-cache
-  variables:
-    USE_SYSTEM_SSL: "true"
-    BASE_OS: "ubuntu_20.04_fips"
-    PACKAGE_DIRECTORY: "pkg/ubuntu-focal_fips"
-  allow_failure: true
-  rules:
-    - if: '$PIPELINE_TYPE == "TRIGGERED_EE_PIPELINE"'
-      when: manual
-
 package_size_check:
   extends: .trigger-package-cache
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
@@ -553,29 +509,11 @@ package_size_check:
     - bundle exec rake build:package:generate_sizefile
     - bundle exec rake check:package_size
   needs:
-    - job: Trigger:package
+    - job: Ubuntu-22.04-branch
       artifacts: false
   rules:
     - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
 
-Trigger:gitlab-docker:
-  extends:
-    - .docker_job
-    - .gems-cache
-  stage: trigger-docker
-  script:
-    - bundle exec rake docker:build:image
-    - bundle exec rake docker:push:triggered
-  rules:
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
-  needs:
-    - job: Trigger:package
-      artifacts: false
-    - job: generate-facts
-      optional: true
-      artifacts: true
-
 qa-subset-test:
   extends:
     - .qa-template
@@ -602,9 +540,10 @@ qa-subset-test:
   needs:
     - job: generate-facts
       artifacts: true
-    - job: Trigger:package
+    - job: Ubuntu-22.04-branch
       artifacts: false
-    - job: Trigger:gitlab-docker
+    - job: Docker-branch
+      optional: true
       artifacts: false
 
 qa-remaining-test-manual:
@@ -629,9 +568,10 @@ qa-remaining-test-manual:
   needs:
     - job: generate-facts
       artifacts: true
-    - job: Trigger:package
+    - job: Ubuntu-22.04-branch
       artifacts: false
-    - job: Trigger:gitlab-docker
+    - job: Docker-branch
+      optional: true
       artifacts: false
 
 letsencrypt-test:
@@ -647,7 +587,7 @@ letsencrypt-test:
       when: never
     - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
   needs:
-    - job: Trigger:gitlab-docker
+    - job: Docker-branch
       artifacts: false
 
 RAT:
@@ -672,10 +612,6 @@ RAT:
   needs:
     - job: Ubuntu-22.04-branch
       artifacts: false
-      optional: true
-    - job: Trigger:package
-      artifacts: false
-      optional: true
     - job: generate-facts
       artifacts: true
 
@@ -687,10 +623,6 @@ RAT:FIPS:
   needs:
     - job: Ubuntu-20.04-fips-branch
       artifacts: false
-      optional: true
-    - job: Trigger:package:fips
-      artifacts: false
-      optional: true
     - job: generate-facts
       artifacts: true
 
@@ -712,7 +644,7 @@ GET:Geo:
       when: manual
       allow_failure: true
   needs:
-    - job: Trigger:package
+    - job: Ubuntu-22.04-branch
       artifacts: false
     - job: generate-facts
       artifacts: true
@@ -793,7 +725,7 @@ pages:
 
 .build-package-on-all-os-vars:
   extends: .base-trigger-job-variables
-  SKIP_JOB_REGEX: '/Ubuntu-22.04|Docker|QA/'
+  SKIP_JOB_REGEX: '/Ubuntu-22.04|Ubuntu-20.04-fips|Docker|QA/'
   PIPELINE_TYPE: "${EDITION}_BRANCH_BUILD_PIPELINE"
   CACHE_EDITION: ${EDITION}
   ee: ${ee}

gitlab-ci-config/workflow-rules.yml

@@ -28,6 +28,8 @@ workflow:
       variables:
         PIPELINE_TYPE: TRIGGER_CACHE_UPDATE_PIPELINE
         CACHE_EDITION: "GITLAB"
+        # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+        CACHE_POLICY: 'push'
     - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $QA_PROJECT_PATH && $CACHE_UPDATE == "true"'
       variables:
         PIPELINE_TYPE: TRIGGER_CACHE_UPDATE_PIPELINE

lib/gitlab/build/facts.rb

@@ -91,8 +91,8 @@ def qa_trigger_vars
           GITLAB_SEMVER_VERSION=#{Build::Info::Git.latest_stable_tag.tr('+', '-')}
           RAT_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs'}
           RAT_FIPS_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_FIPS_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs-fips-ubuntu'}
-          RAT_PACKAGE_URL=#{Build::Info::CI.package_download_url}
-          RAT_FIPS_PACKAGE_URL=#{Build::Info::CI.fips_package_download_url}
+          RAT_PACKAGE_URL=#{Build::Info::CI.package_download_url(fips: false)}
+          RAT_FIPS_PACKAGE_URL=#{Build::Info::CI.package_download_url(job_name: 'Ubuntu-20.04', fips: true)}
         ]
       end
 

lib/gitlab/build/info/ci.rb

@@ -38,26 +38,6 @@ def commit_ref_slug
           Gitlab::Util.get_env('CI_COMMIT_REF_SLUG')
         end
 
-        def package_download_url
-          return Gitlab::Util.get_env('PACKAGE_URL') if Gitlab::Util.get_env('PACKAGE_URL')
-
-          # For builds running in dev.gitlab.org, or nightly pipelines in .com,
-          # use the artifact from `Ubuntu-22.04-branch` job. For the other build,
-          # which is essentially in triggered pipeline, use artifact from
-          # `Trigger:package` job.
-          if /dev.gitlab.org/.match?(Build::Info::CI.api_v4_url) || Build::Check.is_nightly?
-            Build::Info::CI.branch_build_package_download_url
-          else
-            Build::Info::CI.triggered_package_download_url(fips: false)
-          end
-        end
-
-        def fips_package_download_url
-          return Gitlab::Util.get_env('FIPS_PACKAGE_URL') if Gitlab::Util.get_env('FIPS_PACKAGE_URL')
-
-          Build::Info::CI.triggered_package_download_url(fips: false)
-        end
-
         def artifact_url(job_name, file_path)
           client = Gitlab::APIClient.new
           target_job_id = client.get_job_id(job_name)
@@ -67,15 +47,10 @@ def artifact_url(job_name, file_path)
           URI("#{api_v4_url}/projects/#{project_id}/jobs/#{target_job_id}/artifacts/#{file_path}")
         end
 
-        def triggered_package_download_url(fips: Build::Check.use_system_ssl?)
-          folder = fips ? 'ubuntu-focal_fips' : 'ubuntu-jammy'
-          job_name = fips ? 'Trigger:package:fips' : 'Trigger:package'
-          package_path = "pkg/#{folder}/gitlab.deb"
-
-          artifact_url(job_name, package_path)
-        end
+        def package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64', fips: Build::Check.use_system_ssl?)
+          return Gitlab::Util.get_env('FIPS_PACKAGE_URL') if fips && Gitlab::Util.get_env('FIPS_PACKAGE_URL')
+          return Gitlab::Util.get_env('PACKAGE_URL') if !fips && Gitlab::Util.get_env('PACKAGE_URL')
 
-        def branch_build_package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64')
           case job_name
           when /AlmaLinux-8/
             # In EL world, amd64 is called x86_64
@@ -95,6 +70,8 @@ def branch_build_package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64')
             folder = "#{folder}_aarch64"
           end
 
+          job_name = "#{job_name}-fips" if fips
+
           job_name = "#{job_name}-branch" unless Build::Info::CI.tag_name
 
           package_path = "pkg/#{folder}/#{package_file_name}"

lib/gitlab/package_size.rb

@@ -13,7 +13,7 @@ class PackageSizeCheck
 
   class << self
     def fetch_sizefile
-      sizefile_url = Build::Info::CI.artifact_url("Trigger:package", "pkg/ubuntu-jammy/gitlab.deb.size")
+      sizefile_url = URI("#{Build::Info::CI.package_download_url}.size")
 
       # We have to use net/http here because `gitlab` gem's `download_job_artifact_file`
       # method doesn't support plain text files. It has to be either binary or valid JSON.

spec/lib/gitlab/build/image_spec.rb

@@ -159,7 +159,7 @@ def self.dockerhub_image_name
             "PACKAGECLOUD_REPO=download-package",
             "RELEASE_PACKAGE=gitlab-ce",
             "RELEASE_VERSION=12.121.12-ce.0",
-            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab.deb",
+            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab-ce_12.121.12-ce.0_amd64.deb",
             "CI_JOB_TOKEN=NOT-CI-JOB-TOKEN\n"
           ].join("\n")
         end
@@ -180,7 +180,7 @@ def self.dockerhub_image_name
             "PACKAGECLOUD_REPO=download-package",
             "RELEASE_PACKAGE=gitlab-ee",
             "RELEASE_VERSION=12.121.12-ee.0",
-            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab.deb",
+            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab-ee_12.121.12-ee.0_amd64.deb",
             "CI_JOB_TOKEN=NOT-CI-JOB-TOKEN\n"
           ].join("\n")
         end