Merge branch 'registry-import-command' into 'master'

Add registry-database import command to gitlab-ctl

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7265

Merged-by: Balasankar 'Balu' C <[email protected]>
Approved-by: Andrew Patterson <[email protected]>
Approved-by: Balasankar 'Balu' C <[email protected]>
Reviewed-by: João Pereira <[email protected]>
Co-authored-by: João Pereira <[email protected]>
Co-authored-by: Jaime Martinez <[email protected]>

Author: 3 people

files/gitlab-ctl-commands/lib/registry/import.rb

@@ -0,0 +1,88 @@
+require 'optparse'
+
+module Import
+  CMD_NAME = 'import'.freeze
+  SUMMARY_WIDTH = 40
+  DESC_INDENT = 45
+
+  def self.indent(str, len)
+    str.gsub(/%%/, ' ' * len)
+  end
+
+  USAGE = <<~EOS.freeze
+  Import filesystem metadata into the database
+
+  See documentation at https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/database-import-tool.md
+
+  Usage:
+    gitlab-ctl registry-database import [options]
+
+  Options:
+    -B, --common-blobs                Import all blob metadata from common storage
+    -c, --row-count                   Count and log number of rows across relevant database tables on (pre)import completion
+    -d, --dry-run                     Do not commit changes to the database
+    -e, --require-empty-database      Abort import if the database is not empty
+    -p, --pre-import                  Import immutable repository-scoped data to speed up a following import
+    -r, --all-repositories            Import all repository-scoped data
+    -h, --help                        Help for import
+    -1, --step-one pre-import         Perform step one of a multi-step import: alias for pre-import
+    -2, --step-two all-repositories   Perform step two of a multi-step import: alias for all-repositories
+    -3, --step-three common-blobs     Perform step three of a multi-step import: alias for common-blobs
+  EOS
+
+  def self.parse_options!(args, parser, options)
+    return unless args.include? CMD_NAME
+
+    loop do
+      break if args.shift == CMD_NAME
+    end
+
+    parser.on('-h', '--help', 'Usage help') do
+      parser.set_summary_width(SUMMARY_WIDTH)
+      Kernel.puts USAGE
+      Kernel.exit 0
+    end
+
+    parser.on('-B', '--common-blobs', indent('import all blob metadata from common storage', DESC_INDENT)) do
+      options[:common_blobs] = '--common-blobs'
+    end
+
+    parser.on('-c', '--row-count', indent('count and log number of rows across relevant database tables on (pre)import completion', DESC_INDENT)) do
+      options[:row_count] = '--row-count'
+    end
+
+    parser.on('-d', '--dry-run', indent('do not commit changes to the database', DESC_INDENT)) do
+      options[:dry_run] = '--dry-run'
+    end
+
+    parser.on('-e', '--require-empty-database', indent('abort import if the database is not empty', DESC_INDENT)) do
+      options[:empty] = '--require-empty-database'
+    end
+
+    parser.on('-p', '--pre-import', indent('import immutable repository-scoped data to speed up a following import', DESC_INDENT)) do
+      options[:pre_import] = '--pre-import'
+    end
+
+    parser.on('-r', '--all-repositories', indent('import all repository-scoped data', DESC_INDENT)) do
+      options[:all_repositories] = '--all-repositories'
+      options[:needs_read_only] = true
+    end
+
+    parser.on('-1', '--step-one', indent('perform step one of a multi-step import: alias for pre-import', DESC_INDENT)) do
+      options[:step_one] = '--step-one'
+    end
+
+    parser.on('-2', '--step-two', indent('perform step two of a multi-step import: alias for all-repositories', DESC_INDENT)) do
+      options[:step_two] = '--step-two'
+      options[:needs_read_only] = true
+    end
+
+    parser.on('-3', '--step-three', indent('perform step three of a multi-step import: alias for common-blobs', DESC_INDENT)) do
+      options[:step_three] = '--step-three'
+    end
+
+    parser.order!(args)
+
+    options
+  end
+end

files/gitlab-ctl-commands/lib/registry/migrate.rb

@@ -105,7 +105,7 @@ def self.parse_options!(args, options)
   end
 
   def self.populate_subcommands(options)
-    database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-migrations.md?ref_type=heads#administration'
+    database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/database-migrations.md#administration'
 
     {
       'up' => OptionParser.new do |opts|

files/gitlab-ctl-commands/lib/registry/registry_database.rb

@@ -2,12 +2,13 @@
 require 'optparse'
 
 require_relative './migrate'
+require_relative './import'
 
 module RegistryDatabase
   EXEC_PATH = '/opt/gitlab/embedded/bin/registry'.freeze
   CONFIG_PATH = '/var/opt/gitlab/registry/config.yml'.freeze
 
-  USAGE ||= <<~EOS.freeze
+  USAGE = <<~EOS.freeze
     Usage:
       gitlab-ctl registry-database command subcommand [options]
 
@@ -16,6 +17,7 @@ module RegistryDatabase
 
     COMMANDS:
       migrate                Manage schema migrations
+      import                 Import filesystem metadata into the database
   EOS
 
   def self.parse_options!(ctl, args)
@@ -52,7 +54,7 @@ def self.parse_options!(ctl, args)
   end
 
   def self.populate_commands(options)
-    database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-migrations.md?ref_type=heads#administration'
+    database_docs_url = 'https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/database-migrations.md#administration'
 
     {
       'migrate' => OptionParser.new do |opts|
@@ -64,6 +66,13 @@ def self.populate_commands(options)
           exit 128
         end
       end,
+
+      'import' => OptionParser.new do |parser|
+        Import.parse_options!(ARGV, parser, options)
+      rescue OptionParser::ParseError => e
+        warn "#{e}\n\n#{Import::USAGE}"
+        exit 128
+      end,
     }
   end
 
@@ -85,23 +94,30 @@ def self.execute(options)
 
     command = set_command(options)
 
+    puts "Executing command:\n#{command.join(' ')}\n"
+
     begin
       status = Kernel.system(*command)
       Kernel.exit!(1) unless status
     ensure
-      start!
+      start! unless running?
     end
   end
 
   def self.set_command(options)
-    command = [EXEC_PATH, "database", options[:command], options[:subcommand]]
-
+    command = [EXEC_PATH, "database", options[:command]]
     options.delete(:command)
+
+    command += [options[:subcommand]] unless options[:subcommand].nil?
     options.delete(:subcommand)
+
     needs_stop = options[:needs_stop]
     options.delete(:needs_stop)
 
-    continue?(needs_stop)
+    needs_read_only = options[:needs_read_only]
+    options.delete(:needs_read_only)
+
+    continue?(needs_stop, needs_read_only)
 
     command += ["-n", options[:limit]] unless options[:limit].nil?
     options.delete(:limit)
@@ -124,6 +140,11 @@ def self.running?
     !GitlabCtl::Util.run_command("gitlab-ctl status #{service_name}").error?
   end
 
+  def self.read_only?
+    config = YAML.load_file(CONFIG_PATH)
+    config.dig('storage', 'maintenance', 'readonly', 'enabled') == true
+  end
+
   def self.start!
     puts "Starting service #{service_name}"
 
@@ -148,17 +169,22 @@ def self.service_name
     "registry"
   end
 
-  def self.continue?(needs_stop)
-    return unless needs_stop && running?
-
-    puts 'WARNING: Migrations cannot be applied while the container registry is running. '\
-         'Stop the registry before proceeding? (y/n)'.color(:yellow)
+  def self.continue?(needs_stop, needs_read_only)
+    if needs_stop && running?
+      puts 'WARNING: Command cannot run while the container registry is running. ' \
+           'Stop the registry before proceeding? (y/n)'.color(:yellow)
 
-    if $stdin.gets.chomp.casecmp('y').zero?
-      stop!
-    else
-      puts "Exiting..."
-      exit 1
+      if $stdin.gets.chomp.casecmp('y').zero?
+        stop!
+      else
+        puts "Exiting..."
+        exit 1
+      end
     end
+
+    return unless running? && needs_read_only && !read_only?
+
+    puts 'Command requires the container registry to be in read-only mode. Exiting...'
+    exit 1
   end
 end

spec/chef/gitlab-ctl-commands/lib/registry/import_spec.rb

@@ -0,0 +1,33 @@
+require 'optparse'
+
+require_relative('../../../../../files/gitlab-ctl-commands/lib/registry/import')
+
+RSpec.describe Import do
+  describe '.parse_options!' do
+    before do
+      allow(Kernel).to receive(:exit) { |code| raise "Kernel.exit(#{code})" }
+    end
+
+    options_data = [
+      [:common_blobs, 'common-blobs', false],
+      [:row_count, 'row-count', false],
+      [:dry_run, 'dry-run', false],
+      [:empty, 'require-empty-database', false],
+      [:pre_import, 'pre-import', false],
+      [:all_repositories, 'all-repositories', true],
+      [:step_one, 'step-one', false],
+      [:step_two, 'step-two', true],
+      [:step_three, 'step-three', false]
+    ]
+
+    options_data.each do |option, option_name, read_only|
+      it "correctly parses the #{option_name} option#{' with read-only mode' if read_only}" do
+        expected_options = { option => "--#{option_name}" }
+        expected_options[:needs_read_only] = true if read_only
+
+        result = Import.parse_options!(%W[import --#{option_name}], OptionParser.new, {})
+        expect(result).to eq(expected_options)
+      end
+    end
+  end
+end

spec/chef/gitlab-ctl-commands/lib/registry/registry_database_spec.rb

@@ -42,5 +42,12 @@
         expect(received).to have_key(:command)
       end
     end
+
+    context 'when command is import' do
+      let(:command) { 'import' }
+
+      it_behaves_like 'unknown option is specified'
+      it_behaves_like 'parses command options'
+    end
   end
 end