Merge branch 'run-nightly-in-com' into 'master'

Enable running nightly pipelines in Canonical repo

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7434

Merged-by: Andrew Patterson <[email protected]>
Approved-by: Clemens Beck <[email protected]>
Approved-by: Andrew Patterson <[email protected]>
Reviewed-by: Balasankar 'Balu' C <[email protected]>
Co-authored-by: Balasankar 'Balu' C <[email protected]>

Author: Andrew Patterson

.gitlab-ci.yml

@@ -72,6 +72,8 @@ variables:
   QA_PROJECT_PATH: 'gitlab-org/build/omnibus-gitlab-mirror'
   ARM64_RUNNER_TAG: 'arm64'
   PACKAGE_PROMOTION_RUNNER_TAG: 'promotion'
+  NIGHTLY_REPO: 'nightly-builds'
+  NIGHTLY_FIPS_REPO: 'nightly-fips-builds'
 
 .distribution-amd64-tags:
   - distribution-runner
@@ -154,7 +156,10 @@ before_script:
       chmod -R 0600 ~/.ssh/;
     fi
   - bash scripts/ci/prepare_bundle.sh
-  - if [ -n "$NIGHTLY" ]; then export STAGING_REPO=nightly-builds; export FIPS_STAGING_REPO=nightly-fips-builds; fi
+  - if [ -n "$NIGHTLY" ]; then
+      export STAGING_REPO=${NIGHTLY_REPO};
+      export FIPS_STAGING_REPO=${NIGHTLY_FIPS_REPO};
+    fi
 
 fetch-assets:
   extends: .docker_job

gitlab-ci-config/workflow-rules.yml

@@ -46,10 +46,7 @@ workflow:
     - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $CANONICAL_PROJECT_PATH && $CI_COMMIT_REF_PROTECTED == "true"'
       variables:
         PIPELINE_TYPE: PROTECTED_TEST_PIPELINE
-    - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $CANONICAL_PROJECT_PATH && $CI_COMMIT_BRANCH'
-      variables:
-        PIPELINE_TYPE: GITLAB_BRANCH_TEST_PIPELINE
-    - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $DEV_PROJECT_PATH && $NIGHTLY && $CI_COMMIT_BRANCH && ($ee == "true" || $GITLAB_VERSION =~ /-ee$/)'
+    - if: '$PIPELINE_TYPE == null && $NIGHTLY && $CI_COMMIT_BRANCH && ($ee == "true" || $GITLAB_VERSION =~ /-ee$/)'
       variables:
         PIPELINE_TYPE: EE_NIGHTLY_BUILD_PIPELINE
         CACHE_EDITION: "EE"
@@ -62,7 +59,7 @@ workflow:
       variables:
         PIPELINE_TYPE: EE_BRANCH_BUILD_PIPELINE
         CACHE_EDITION: "EE"
-    - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $DEV_PROJECT_PATH && $NIGHTLY && $CI_COMMIT_BRANCH'
+    - if: '$PIPELINE_TYPE == null && $NIGHTLY && $CI_COMMIT_BRANCH'
       variables:
         PIPELINE_TYPE: CE_NIGHTLY_BUILD_PIPELINE
         CACHE_EDITION: "CE"
@@ -103,6 +100,9 @@ workflow:
     - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $SECURITY_PROJECT_PATH && $CI_COMMIT_REF_PROTECTED == "true"'
       variables:
         PIPELINE_TYPE: PROTECTED_TEST_PIPELINE
+    - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $CANONICAL_PROJECT_PATH && $CI_COMMIT_BRANCH'
+      variables:
+        PIPELINE_TYPE: GITLAB_BRANCH_TEST_PIPELINE
     - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $SECURITY_PROJECT_PATH && $CI_COMMIT_BRANCH'
       variables:
         PIPELINE_TYPE: GITLAB_BRANCH_TEST_PIPELINE

lib/gitlab/build/info/docker.rb

@@ -14,7 +14,7 @@ def tag
         def release_file_contents
           repo = Gitlab::Util.get_env('PACKAGECLOUD_REPO') # Target repository
 
-          download_url = if /dev.gitlab.org/.match?(Build::Info::CI.api_v4_url)
+          download_url = if /dev.gitlab.org/.match?(Build::Info::CI.api_v4_url) || Build::Check.is_nightly?
                            Build::Info::CI.package_download_url
                          else
                            Build::Info::CI.triggered_package_download_url

lib/gitlab/package_repository.rb

@@ -35,7 +35,7 @@ def validate(dry_run)
 
   def upload(repository = nil, dry_run = false)
     if upload_user.nil?
-      puts "User for uploading to package server not specified!"
+      puts "Owner of the repository to which packages are being uploaded not specified! Set `PACKAGECLOUD_USER` environment variable."
       return
     end
 
@@ -51,20 +51,21 @@ def upload(repository = nil, dry_run = false)
       cmd = "LC_ALL='en_US.UTF-8' bin/package_cloud push #{upload_user}/#{pkg} --url=https://packages.gitlab.com"
       puts "Uploading...\n"
 
-      if dry_run
-        puts cmd
-      else
-        Retriable.with_context(:package_publish, on: PackageUploadError) do
-          result = `#{cmd}`
+      puts "Running the command: #{cmd}"
 
-          if child_process_status == 1
-            unless /filename: has already been taken/.match?(result)
-              puts 'Upload to package server failed!.'
-              raise PackageUploadError, "Upload to package server failed!."
-            end
+      next if dry_run
 
-            puts "Package #{pkg} has already been uploaded, skipping.\n"
+      Retriable.with_context(:package_publish, on: PackageUploadError) do
+        result = `#{cmd}`
+
+        if child_process_status == 1
+          unless /filename: has already been taken/.match?(result)
+            puts 'Upload to package server failed!.'
+            puts "The command returned the output: #{result}"
+            raise PackageUploadError, "Upload to package server failed!."
           end
+
+          puts "Package #{pkg} has already been uploaded, skipping.\n"
         end
       end
     end
@@ -119,6 +120,10 @@ def package_list(repository)
   end
 
   def upload_user
+    # Even though the variable says "upload user", this is acutally the user
+    # who owns the repository to which packages are being uploaded. This
+    # information is only used to generate the path to the repository - the
+    # user who actually does the upload is known by the token.
     Gitlab::Util.get_env('PACKAGECLOUD_USER') if Gitlab::Util.get_env('PACKAGECLOUD_USER') && !Gitlab::Util.get_env('PACKAGECLOUD_USER').empty?
   end
 

spec/lib/gitlab/package_repository_spec.rb

@@ -59,6 +59,7 @@
 
     context 'on non-stable branch' do
       before do
+        unset_all_env_variables
         allow(IO).to receive(:popen).with(%w[git describe]).and_return("8.1.0+rc1.ce.0-1685-gd2a2c51\n")
       end
 
@@ -72,6 +73,7 @@
 
     context 'on a stable branch' do
       before do
+        unset_all_env_variables
         allow(IO).to receive(:popen).with(%w[git describe]).and_return("8.12.8+ce.0-1-gdac92d4\n")
       end
 
@@ -135,8 +137,12 @@
   describe '#upload' do
     describe 'with staging repository' do
       context 'when upload user is not specified' do
+        before do
+          unset_all_env_variables
+        end
+
         it 'prints a message and aborts' do
-          expect { repo.upload('my-staging-repository', true) }.to output(%r{User for uploading to package server not specified!\n}).to_stdout
+          expect { repo.upload('my-staging-repository', true) }.to output(%r{Owner of the repository to which packages are being uploaded not specified}).to_stdout
         end
       end
 
@@ -278,6 +284,14 @@
     end
   end
 
+  def unset_all_env_variables
+    stub_env_var('PACKAGECLOUD_TOKEN', nil)
+    stub_env_var('PACKAGECLOUD_USER', nil)
+    stub_env_var('PACKAGECLOUD_REPO', nil)
+    stub_env_var('RASPBERRY_REPO', nil)
+    stub_env_var('STAGING_REPO', nil)
+  end
+
   def set_all_env_variables
     stub_env_var("PACKAGECLOUD_REPO", "super-stable-1234")
     stub_env_var("RASPBERRY_REPO", "raspi")