Merge branch 'make-artifact-fetches-use-library' into 'master'

Use GitLab API interface for fetching artifacts

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7007

Merged-by: Jason Plum <[email protected]>
Approved-by: Dustin Collins <[email protected]>
Approved-by: Jason Plum <[email protected]>
Reviewed-by: Jason Plum <[email protected]>
Co-authored-by: Balasankar "Balu" C <[email protected]>

Author: WarheadsSE

lib/gitlab/build/facts.rb

@@ -77,8 +77,8 @@ def qa_trigger_vars
           GITLAB_SEMVER_VERSION=#{Build::Info.latest_stable_tag.tr('+', '-')}
           RAT_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs'}
           RAT_FIPS_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_FIPS_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs-fips-ubuntu'}
-          RAT_PACKAGE_URL=#{Gitlab::Util.get_env('PACKAGE_URL') || Build::Info.triggered_build_package_url(fips: false)}
-          RAT_FIPS_PACKAGE_URL=#{Gitlab::Util.get_env('FIPS_PACKAGE_URL') || Build::Info.triggered_build_package_url(fips: true)}
+          RAT_PACKAGE_URL=#{Gitlab::Util.get_env('PACKAGE_URL') || Build::Info::CI.triggered_package_download_url(fips: false)}
+          RAT_FIPS_PACKAGE_URL=#{Gitlab::Util.get_env('FIPS_PACKAGE_URL') || Build::Info::CI.triggered_package_download_url(fips: true)}
         ]
       end
 

lib/gitlab/build/info.rb

@@ -1,6 +1,4 @@
 require 'omnibus'
-require 'net/http'
-require 'json'
 
 require_relative '../build_iteration'
 require_relative "../util.rb"
@@ -236,15 +234,6 @@ def log_level
         end
       end
 
-      # Fetch the package from an S3 bucket
-      def deb_package_download_url(arch: 'amd64')
-        folder = 'ubuntu-jammy'
-        folder = "#{folder}_aarch64" if arch == 'arm64'
-
-        package_filename_url_safe = Info.release_version.gsub("+", "%2B")
-        "https://#{Info.release_bucket}.#{Info.release_bucket_s3_endpoint}/#{folder}/#{Info.package}_#{package_filename_url_safe}_#{arch}.deb"
-      end
-
       # Fetch the package used in AWS AMIs from an S3 bucket
       def ami_deb_package_download_url(arch: 'amd64')
         folder = 'ubuntu-focal'
@@ -254,53 +243,6 @@ def ami_deb_package_download_url(arch: 'amd64')
         "https://#{Info.release_bucket}.#{Info.release_bucket_s3_endpoint}/#{folder}/#{Info.package}_#{package_filename_url_safe}_#{arch}.deb"
       end
 
-      def rpm_package_download_url(arch: 'x86_64')
-        folder = 'el-8'
-        folder = "#{folder}_aarch64" if arch == 'arm64'
-        folder = "#{folder}_fips" if Build::Check.use_system_ssl?
-
-        package_filename_url_safe = Info.release_version.gsub("+", "%2B")
-        "https://#{Info.release_bucket}.#{Info.release_bucket_s3_endpoint}/#{folder}/#{Info.package}-#{package_filename_url_safe}.el8.#{arch}.rpm"
-      end
-
-      def get_api(path, token: nil)
-        uri = URI("https://gitlab.com/api/v4/#{path}")
-        req = Net::HTTP::Get.new(uri)
-        req['PRIVATE-TOKEN'] = token || Gitlab::Util.get_env('TRIGGER_PRIVATE_TOKEN')
-        http = Net::HTTP.new(uri.hostname, uri.port)
-        http.use_ssl = true
-        res = http.request(req)
-        JSON.parse(res.body) if res.code == '200'
-      end
-
-      def fetch_artifact_url(project_id, pipeline_id, fips: Build::Check.use_system_ssl?)
-        job_name = 'Trigger:package'
-        job_name = "#{job_name}:fips" if fips
-
-        output = get_api("projects/#{project_id}/pipelines/#{pipeline_id}/jobs") || {}
-        output.map { |job| job['id'] if job['name'] == job_name }.compact.max
-      end
-
-      def fetch_pipeline_jobs(project_id, pipeline_id, token)
-        get_api("projects/#{project_id}/pipelines/#{pipeline_id}/jobs")
-      end
-
-      def triggered_build_package_url(fips: Build::Check.use_system_ssl?)
-        project_id = Gitlab::Util.get_env('CI_PROJECT_ID')
-        pipeline_id = Gitlab::Util.get_env('CI_PIPELINE_ID')
-        return unless project_id && !project_id.empty? && pipeline_id && !pipeline_id.empty?
-
-        id = fetch_artifact_url(project_id, pipeline_id, fips: fips)
-
-        return unless id
-
-        # Ubuntu 22.04 is still not FIPS compliant, so we use 20.04 packages
-        # there.
-        folder = fips ? 'ubuntu-focal_fips' : 'ubuntu-jammy'
-
-        "https://gitlab.com/api/v4/projects/#{Gitlab::Util.get_env('CI_PROJECT_ID')}/jobs/#{id}/artifacts/pkg/#{folder}/gitlab.deb"
-      end
-
       def tag_match_pattern
         return '*[+.]ee.*' if Check.is_ee?
 

lib/gitlab/package_size.rb

@@ -1,6 +1,7 @@
 require_relative "util"
 require_relative "ohai_helper"
 require_relative "build/check"
+require_relative "build/info/secrets"
 
 require 'gitlab'
 
@@ -12,21 +13,13 @@ class PackageSizeCheck
 
   class << self
     def fetch_sizefile
-      api_url = Gitlab::Util.get_env('CI_API_V4_URL')
-      project_id = Gitlab::Util.get_env('CI_PROJECT_ID')
-      pipeline_id = Gitlab::Util.get_env('CI_PIPELINE_ID')
-      token = Gitlab::Util.get_env('PACKAGE_SIZE_CHECK_OMNIBUS_GITLAB_MIRROR_TOKEN')
-
-      gitlab_client = ::Gitlab.client(endpoint: api_url, private_token: token)
-      pipeline_jobs = gitlab_client.pipeline_jobs(project_id, pipeline_id)
-      trigger_package_job = pipeline_jobs.find { |j| j.name == 'Trigger:package' }
+      sizefile_url = Build::Info::CI.artifact_url("Trigger:package", "pkg/ubuntu-jammy/gitlab.deb.size")
 
       # We have to use net/http here because `gitlab` gem's `download_job_artifact_file`
       # method doesn't support plain text files. It has to be either binary or valid JSON.
       # https://github.com/NARKOZ/gitlab/issues/621
-      sizefile_url = URI("#{api_url}/projects/#{project_id}/jobs/#{trigger_package_job.id}/artifacts/pkg/ubuntu-jammy/gitlab.deb.size")
       req = Net::HTTP::Get.new(sizefile_url)
-      req['PRIVATE-TOKEN'] = token
+      req['PRIVATE-TOKEN'] = Build::Info::Secrets.api_token
       res = Net::HTTP.start(sizefile_url.hostname, sizefile_url.port, use_ssl: true) do |http|
         http.request(req)
       end