Merge branch 'use-google-storage-gem-instead-of-gcloud' into 'master'

Use google-cloud-storage gem for uploading packages to GCS

Closes #8088

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7052

Merged-by: Robert Marshall <[email protected]>
Approved-by: Ryan Egesdahl <[email protected]>
Approved-by: Robert Marshall <[email protected]>
Reviewed-by: Robert Marshall <[email protected]>
Reviewed-by: Ryan Egesdahl <[email protected]>
Co-authored-by: Balasankar "Balu" C <[email protected]>

Author: Robert Marshall

Gemfile

@@ -41,7 +41,8 @@ gem 'gitlab'
 gem 'yard'
 gem 'toml-rb'
 gem 'retriable'
-gem "tomlib", "~> 0.6.0"
+gem 'tomlib', '~> 0.6.0'
+gem 'google-cloud-storage'
 
 group :packagecloud, optional: true do
   gem 'package_cloud'

Gemfile.lock

@@ -192,7 +192,10 @@ GEM
       irb (>= 1.3.6)
       reline (>= 0.3.1)
     debug_inspector (1.1.0)
+    declarative (0.0.20)
     diff-lcs (1.3)
+    digest-crc (0.6.5)
+      rake (>= 12.0.0, < 14.0.0)
     docile (1.4.0)
     docker-api (2.0.0)
       excon (>= 0.47.0)
@@ -249,6 +252,40 @@ GEM
       rubocop-performance (~> 1.14)
       rubocop-rails (~> 2.15)
       rubocop-rspec (~> 2.12)
+    google-apis-core (0.11.0)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (>= 0.16.2, < 2.a)
+      httpclient (>= 2.8.1, < 3.a)
+      mini_mime (~> 1.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.a)
+      rexml
+      webrick
+    google-apis-iamcredentials_v1 (0.17.0)
+      google-apis-core (>= 0.11.0, < 2.a)
+    google-apis-storage_v1 (0.19.0)
+      google-apis-core (>= 0.9.0, < 2.a)
+    google-cloud-core (1.6.0)
+      google-cloud-env (~> 1.0)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.6.0)
+      faraday (>= 0.17.3, < 3.0)
+    google-cloud-errors (1.3.1)
+    google-cloud-storage (1.44.0)
+      addressable (~> 2.8)
+      digest-crc (~> 0.4)
+      google-apis-iamcredentials_v1 (~> 0.1)
+      google-apis-storage_v1 (~> 0.19.0)
+      google-cloud-core (~> 1.6)
+      googleauth (>= 0.16.2, < 2.a)
+      mini_mime (~> 1.0)
+    googleauth (1.7.0)
+      faraday (>= 0.17.3, < 3.a)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (>= 0.16, < 2.a)
     gssapi (1.3.1)
       ffi (>= 1.0.1)
     gyoku (1.4.0)
@@ -280,6 +317,7 @@ GEM
     jmespath (1.6.2)
     json (2.6.1)
     json_pure (2.3.1)
+    jwt (2.7.1)
     knapsack (1.17.1)
       rake
     kramdown (2.4.0)
@@ -299,10 +337,12 @@ GEM
     logging (2.3.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.14)
+    memoist (0.16.2)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
+    mini_mime (1.1.2)
     minitar (0.9)
     minitest (5.14.4)
     mixlib-archive (1.1.7)
@@ -345,6 +385,7 @@ GEM
       wmi-lite (~> 1.0)
     omnibus-ctl (0.3.6)
     open4 (1.3.4)
+    os (1.1.4)
     package_cloud (0.3.09)
       highline (~> 2.0.0)
       json_pure (~> 2.3.0)
@@ -381,6 +422,10 @@ GEM
     regexp_parser (2.1.1)
     reline (0.3.1)
       io-console (~> 0.5)
+    representable (3.2.0)
+      declarative (< 0.1.0)
+      trailblazer-option (>= 0.1.1, < 0.2.0)
+      uber (< 0.2.0)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -450,6 +495,11 @@ GEM
       addressable (>= 2.3.5)
       faraday (>= 0.17.3, < 3)
     semverse (3.0.0)
+    signet (0.17.0)
+      addressable (~> 2.8)
+      faraday (>= 0.17.5, < 3.a)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -476,6 +526,7 @@ GEM
       citrus (~> 3.0, > 3.0)
     tomlib (0.6.0)
     tomlrb (1.3.0)
+    trailblazer-option (0.1.2)
     train-core (3.9.2)
       addressable (~> 2.5)
       ffi (!= 1.13.0)
@@ -507,6 +558,7 @@ GEM
       tty-screen (~> 0.8)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
+    uber (0.1.0)
     unf (0.1.4)
       unf_ext
     unicode-display_width (1.8.0)
@@ -558,6 +610,7 @@ DEPENDENCIES
   gitlab
   gitlab-dangerfiles (~> 3.0)
   gitlab-styles (~> 9.0)
+  google-cloud-storage
   http
   json
   knapsack

lib/gitlab/gcloud_helper.rb

@@ -1,61 +1,40 @@
-require 'retriable'
-require 'open3'
+require 'google/cloud/storage'
 require_relative './build/info.rb'
 
 class GCloudHelper
-  GCSSyncError = Class.new(StandardError)
-  SAFileNotSetError = Class.new(StandardError)
-  SAActivationError = Class.new(StandardError)
-
   class << self
-    def activate_sa!
-      raise SAFileNotSetError, 'Service account file not set in environment!' unless sa_file
-
-      out, status = Open3.capture2e("gcloud auth activate-service-account --key-file #{sa_file}")
-      raise SAActivationError, "Service account activation failed! ret=#{status.exitstatus} out=#{out}" unless status.success?
-    end
-
-    def sa_file
-      Build::Info.gcp_release_bucket_sa_file
-    end
-
-    def pkgs_bucket
-      Build::Info.gcp_release_bucket
-    end
-
-    def gcs_sync!(dir)
-      begin
-        activate_sa!
-      rescue SAFileNotSetError, SAActivationError => e
-        return e.message
+    def upload_packages_and_print_urls(dir)
+      if sa_file.nil? || !File.exist?(sa_file)
+        warn "Error finding service account file. Can not upload packages to bucket."
+        return
       end
 
-      out = ""
+      storage = Google::Cloud::Storage.new(credentials: sa_file)
+      bucket = storage.bucket(pkgs_bucket)
+
+      signed_urls = []
+      dir_path = File.absolute_path(dir)
 
-      Retriable.retriable(tries: 4, max_elapsed_time: 900, on: GCSSyncError) do
-        out, status = Open3.capture2e("gsutil -o GSUtil:parallel_composite_upload_threshold=150M -m rsync -r #{dir} gs://#{pkgs_bucket}")
-        break if status.success?
+      puts "Syncing packages to GCS bucket."
+      Dir.glob("#{dir_path}/*/**").each do |source|
+        destination = source.delete_prefix("#{dir_path}/")
 
-        raise GCSSyncError, "Gsutil rsync failed! ret=#{status.exitstatus} out=#{out}"
+        puts "\tUploading #{destination}"
+        bucket.upload_file(source, destination)
+        signed_urls << bucket.signed_url(destination, version: :v4)
       end
 
-      out
+      puts signed_urls
     end
 
-    def signed_urls(paths)
-      begin
-        activate_sa!
-      rescue SAFileNotSetError, SAActivationError => e
-        return e.message
-      end
+    private
 
-      gs_uris = paths.map { |p| "gs://#{pkgs_bucket}/#{p}" }
-      # 12 hours is the maximum duration allowed for a signed url that
-      # uses a service account
-      out, status = Open3.capture2e("gsutil signurl -r us-east1 --use-service-account -d 12h #{gs_uris.join(' ')}")
-      return "Unable to generate signed URL for #{gs_uris.join(' ')}! ret=#{status.exitstatus} out=#{out}" unless status.success?
+    def sa_file
+      Build::Info.gcp_release_bucket_sa_file
+    end
 
-      out
+    def pkgs_bucket
+      Build::Info.gcp_release_bucket
     end
   end
 end

lib/gitlab/tasks/build.rake

@@ -74,10 +74,7 @@ namespace :build do
     desc "Sync packages to gcp"
     task :sync do
       Gitlab::Util.section('build:package:sync', collapsed: Build::Check.on_tag?) do
-        puts '---- Syncing packages to GCP'
-        puts GCloudHelper.gcs_sync!('pkg/')
-        paths = Dir.glob('pkg/**/*').select { |f| File.file?(f) }.map { |p| p.gsub(%r[^pkg/], '') }
-        puts GCloudHelper.signed_urls(paths)
+        GCloudHelper.upload_packages_and_print_urls('pkg/')
       end
     end
 

spec/lib/gitlab/gcloud_helper_spec.rb

@@ -1,64 +1,56 @@
 require 'spec_helper'
 require 'gitlab/gcloud_helper'
 
-RSpec.describe GCloudHelper do
-  let(:status_failed) { double("status", success?: false, exitstatus: 1) }
-  let(:status_success) { double("status", success?: true, exitstatus: 0) }
+class StubGoogleCloudBucket
+  def upload_file(source, destination); end
 
+  def signed_url(path, version: :v4)
+    "signed-#{path}"
+  end
+end
+
+RSpec.describe GCloudHelper do
   before do
     allow(ENV).to receive(:[]).and_call_original
-    allow(ENV).to receive(:[]).with('GITLAB_COM_PKGS_SA_FILE').and_return('/path/to/sa')
     allow(Build::Check).to receive(:on_tag?).and_return(false)
+    allow(Google::Cloud::Storage).to receive(:new).and_return(double(bucket: StubGoogleCloudBucket.new))
   end
 
-  describe '.gcs_sync!' do
-    it 'exits with error when activation fails' do
-      expect(Open3).to receive(:capture2e).once.and_return(['sa activation', status_failed])
-
-      expect(described_class.gcs_sync!('some-dir/')).to eq('Service account activation failed! ret=1 out=sa activation')
-    end
-
-    context 'with successful activation' do
+  describe '.upload_packages_and_print_urls' do
+    context 'when SA file variable not defined' do
       before do
-        allow(Open3).to receive(:capture2e).with('gcloud auth activate-service-account --key-file /path/to/sa').and_return(['sa activation', status_success])
-      end
-
-      it 'raises an error if rsync fails after retries' do
-        expect(Open3).to receive(:capture2e).with('gsutil -o GSUtil:parallel_composite_upload_threshold=150M -m rsync -r some-dir/ gs://gitlab-com-pkgs-builds').exactly(4).times.and_return(['gcs rsync output', status_failed])
-
-        expect { described_class.gcs_sync!('some-dir/') }.to raise_error(GCloudHelper::GCSSyncError)
+        stub_env_var('GITLAB_COM_PKGS_SA_FILE', '')
       end
 
-      it 'rsyncs packages to the package bucket' do
-        allow(Open3).to receive(:capture2e).with('gsutil -o GSUtil:parallel_composite_upload_threshold=150M -m rsync -r some-dir/ gs://gitlab-com-pkgs-builds').and_return(['gcs rsync output', status_success])
+      it 'prints error message' do
+        expect(described_class).to receive(:warn).with(/Error finding service account file./)
 
-        expect(described_class.gcs_sync!('some-dir/')).to eq('gcs rsync output')
+        described_class.upload_packages_and_print_urls('pkg/')
       end
     end
-  end
 
-  describe '.signed_urls' do
-    it 'exits with error when activation fails' do
-      expect(Open3).to receive(:capture2e).once.and_return(['sa activation', status_failed])
-
-      expect(described_class.signed_urls(%w[pkg/1 pkg/2 pkg/3])).to eq('Service account activation failed! ret=1 out=sa activation')
-    end
-
-    context 'with successful activation' do
+    context 'when file mentioned in SA variable does not exist' do
       before do
-        allow(Open3).to receive(:capture2e).with('gcloud auth activate-service-account --key-file /path/to/sa').and_return(['sa activation', status_success])
+        stub_env_var('GITLAB_COM_PKGS_SA_FILE', 'a-dummy-file')
+        allow(::File).to receive(:exist?).with('a-dummy-file').and_return(false)
       end
 
-      it 'exits with error when signed URLs fail to generate' do
-        allow(Open3).to receive(:capture2e).with('gsutil signurl -r us-east1 --use-service-account -d 12h gs://gitlab-com-pkgs-builds/pkg/1 gs://gitlab-com-pkgs-builds/pkg/2 gs://gitlab-com-pkgs-builds/pkg/3').once.and_return(['signurl output', status_failed])
+      it 'prints error message' do
+        expect(described_class).to receive(:warn).with(/Error finding service account file./)
 
-        expect(described_class.signed_urls(%w[pkg/1 pkg/2 pkg/3])).to eq('Unable to generate signed URL for gs://gitlab-com-pkgs-builds/pkg/1 gs://gitlab-com-pkgs-builds/pkg/2 gs://gitlab-com-pkgs-builds/pkg/3! ret=1 out=signurl output')
+        described_class.upload_packages_and_print_urls('pkg/')
       end
+    end
 
-      it 'generates signed urls' do
-        allow(Open3).to receive(:capture2e).with('gsutil signurl -r us-east1 --use-service-account -d 12h gs://gitlab-com-pkgs-builds/pkg/1 gs://gitlab-com-pkgs-builds/pkg/2 gs://gitlab-com-pkgs-builds/pkg/3').once.and_return(['signurl output', status_success])
+    context 'when service account file exists' do
+      before do
+        stub_env_var('GITLAB_COM_PKGS_SA_FILE', 'a-dummy-file')
+        allow(::File).to receive(:exist?).with('a-dummy-file').and_return(true)
+        allow(::Dir).to receive(:glob).with(/pkg/).and_return(%w[one two three])
+      end
 
-        expect(described_class.signed_urls(%w[pkg/1 pkg/2 pkg/3])).to eq('signurl output')
+      it 'prints signed URLs' do
+        expect { described_class.upload_packages_and_print_urls('pkg/') }.to output(/signed-one\nsigned-two\nsigned-three/).to_stdout
       end
     end
   end