Merge branch 'stop-syncing-packages-to-s3' into 'master'

Stop syncing packages to S3 bucket

Closes https://gitlab.com/gitlab-org/security/omnibus-gitlab/-/issues/3

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7057

Merged-by: Robert Marshall <[email protected]>
Approved-by: Andrew Patterson <[email protected]>
Co-authored-by: Balasankar "Balu" C <[email protected]>

Author: Robert Marshall

.gitlab-ci.yml

@@ -108,6 +108,14 @@ variables:
     paths:
       - gems
 
+.build-package: &build-package
+  - bundle exec rake cache:populate
+  - bundle exec rake cache:restore
+  - bundle exec rake build:project
+  - bundle exec rake build:package:sync
+  - bundle exec rake cache:bundle
+  - bundle exec rake build:component_shas
+
 before_script:
   - echo "PIPELINE_TYPE detected as ${PIPELINE_TYPE}"
   # Exit early if building on an OS for which we don't provide the specified

gitlab-ci-config/dev-gitlab-org.yml

@@ -44,11 +44,7 @@
   stage: package
   script:
     - support/import_gpg_secrets.sh
-    - bundle exec rake cache:populate
-    - bundle exec rake cache:restore
-    - bundle exec rake build:project build:package:sync
-    - bundle exec rake cache:bundle
-    - bundle exec rake build:component_shas
+    - !reference [.build-package]
     # Run package size checks on nightlies. The job will create alerts, so we
     # don't need it to fail and block the pipeline. Hence the `|| true`.
     - if [ -n "$NIGHTLY" ]; then

gitlab-ci-config/gitlab-com.yml

@@ -62,13 +62,6 @@
   - bundle install -j $(nproc)
   - bundle binstubs --all
 
-.build-package: &build-package
-  - bundle exec rake cache:populate
-  - bundle exec rake cache:restore
-  - bundle exec rake build:project
-  - bundle exec rake cache:bundle
-  - bundle exec rake build:component_shas
-
 .trigger-package-common:
   extends: .trigger-package-cache
   variables:

lib/gitlab/build/info.rb

@@ -23,22 +23,6 @@ def qa_image
         Gitlab::Util.get_env('QA_IMAGE') || "#{Gitlab::Util.get_env('CI_REGISTRY')}/#{Build::Info::Components::GitLabRails.project_path}/#{Build::Info::Package.name}-qa:#{Build::Info::Components::GitLabRails.ref(prepend_version: false)}"
       end
 
-      def release_bucket
-        # Tag builds are releases and they get pushed to a specific S3 bucket
-        # whereas regular branch builds use a separate one
-        downloads_bucket = Gitlab::Util.get_env('RELEASE_BUCKET') || "downloads-packages"
-        builds_bucket = Gitlab::Util.get_env('BUILDS_BUCKET') || "omnibus-builds"
-        Check.on_tag? ? downloads_bucket : builds_bucket
-      end
-
-      def release_bucket_region
-        Gitlab::Util.get_env('RELEASE_BUCKET_REGION') || "eu-west-1"
-      end
-
-      def release_bucket_s3_endpoint
-        Gitlab::Util.get_env('RELEASE_BUCKET_S3_ENDPOINT') || "s3.amazonaws.com"
-      end
-
       def gcp_release_bucket
         # All tagged builds are pushed to the release bucket
         # whereas regular branch builds use a separate one

lib/gitlab/tasks/build.rake

@@ -71,24 +71,13 @@ namespace :build do
       end
     end
 
-    desc "Sync packages to aws and gcp"
+    desc "Sync packages to gcp"
     task :sync do
       Gitlab::Util.section('build:package:sync', collapsed: Build::Check.on_tag?) do
         puts '---- Syncing packages to GCP'
         puts GCloudHelper.gcs_sync!('pkg/')
         paths = Dir.glob('pkg/**/*').select { |f| File.file?(f) }.map { |p| p.gsub(%r[^pkg/], '') }
         puts GCloudHelper.signed_urls(paths)
-
-        puts '---- Syncing packages to AWS'
-        release_bucket = Build::Info.release_bucket
-        release_bucket_region = Build::Info.release_bucket_region
-        release_bucket_s3_endpoint = Build::Info.release_bucket_s3_endpoint
-        puts "AWS S3 Sync: Copying pkg/ contents to #{release_bucket_s3_endpoint}"
-        system(*%W[aws s3 --endpoint-url https://#{release_bucket_s3_endpoint} sync pkg/ s3://#{release_bucket} --no-progress --acl public-read --region #{release_bucket_region}])
-        files = Dir.glob('pkg/**/*').select { |f| File.file? f }
-        files.each do |file|
-          puts file.gsub('pkg', "https://#{release_bucket}.#{release_bucket_s3_endpoint}").gsub('+', '%2B')
-        end
       end
     end