Add gitlab-ctl generate-secrets command

Add the generate-secrets command to gitlab-ctl which creates a JSON file
containing secrets from the gitlab.rb file. The -f|--file option can
be used to send the output to a specified file instead of the default
/etc/gitlab/gitlab-secrets.json.

Changelog: added

Author: Andrew Patterson

files/gitlab-config-template/gitlab.rb.template

@@ -2746,7 +2746,7 @@ external_url 'GENERATED_EXTERNAL_URL'
 ##! Note: We do not recommend changing these values unless absolutely necessary
 ##! Set to false to only parse secrets from `gitlab-secrets.json` file but not generate them.
 # package['generate_default_secrets'] = true
-##! Set to false to prevent creating `gitlab-secrets.json` file
+##! Set to false to prevent creating the default gitlab-secrets.json` file
 # package['generate_secrets_json_file'] = true
 ################################################################################
 ################################################################################

files/gitlab-cookbooks/gitlab-pages/libraries/gitlab_pages.rb

@@ -27,7 +27,8 @@ class << self
     def parse_variables
       parse_pages_external_url
       parse_gitlab_pages_daemon
-      parse_secrets
+      # Only call parse_secrets when not generating a defaults secrets file.
+      parse_secrets unless Gitlab['node'][SecretsHelper::SECRETS_FILE_CHEF_ATTR]
       parse_automatic_oauth_registration
     end
 

files/gitlab-cookbooks/gitlab/recipes/generate_secrets.rb

@@ -0,0 +1,32 @@
+#
+# Copyright:: Copyright (c) 2018 GitLab Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative '../../package/libraries/helpers/secrets_helper'
+
+Gitlab[:node] = node
+
+if node['package']['generate_secrets_json_file'] == true
+  warning_message = <<~EOS
+    You have enabled writing to the default secrets file location with package['generate_secrets_json_file'] in the gitlab.rb file which is not compatible with this command.
+    Use 'gitlab-ctl reconfigure' to generate secrets instead and copy the resulting #{SecretsHelper::SECRETS_FILE} file.
+  EOS
+  LoggingHelper.warning(warning_message)
+  return
+end
+
+secrets_file = node[SecretsHelper::SECRETS_FILE_CHEF_ATTR] || SecretsHelper::SECRETS_FILE
+node.override[SecretsHelper::SKIP_GENERATE_SECRETS_CHEF_ATTR] = false
+Gitlab.generate_secrets(node['fqdn'], secrets_file)

files/gitlab-cookbooks/package/libraries/helpers/secrets_helper.rb

@@ -1,6 +1,10 @@
 require 'openssl'
 
 class SecretsHelper
+  SECRETS_FILE = '/etc/gitlab/gitlab-secrets.json'.freeze
+  SECRETS_FILE_CHEF_ATTR = '_gitlab_secrets_file_path'.freeze
+  SKIP_GENERATE_SECRETS_CHEF_ATTR = '_skip_generate_secrets'.freeze
+
   def self.generate_hex(chars)
     SecureRandom.hex(chars)
   end
@@ -40,17 +44,17 @@ def self.generate_keypair(bits:, subject:, validity:)
   # Load the secrets from disk
   #
   # @return [Hash]  empty if no secrets
-  def self.load_gitlab_secrets
+  def self.load_gitlab_secrets(path = SecretsHelper::SECRETS_FILE)
     existing_secrets = {}
 
-    existing_secrets = Chef::JSONCompat.from_json(File.read("/etc/gitlab/gitlab-secrets.json")) if File.exist?("/etc/gitlab/gitlab-secrets.json")
+    existing_secrets = Chef::JSONCompat.from_json(File.read(path)) if File.exist?(path)
 
     existing_secrets
   end
 
   # Reads the secrets into the Gitlab config singleton
-  def self.read_gitlab_secrets
-    existing_secrets = load_gitlab_secrets
+  def self.read_gitlab_secrets(path = SecretsHelper::SECRETS_FILE)
+    existing_secrets = load_gitlab_secrets(path)
 
     existing_secrets.each do |k, v|
       if Gitlab[k]
@@ -59,7 +63,7 @@ def self.read_gitlab_secrets
           Gitlab[k][pk] ||= p
         end
       else
-        warn("Ignoring section #{k} in /etc/gitlab/gitlab-secrets.json, does not exist in gitlab.rb")
+        warn("Ignoring section #{k} in #{path}, does not exist in gitlab.rb")
       end
     end
   end
@@ -130,11 +134,11 @@ def self.gather_gitlab_secrets # rubocop:disable Metrics/AbcSize
     secret_tokens
   end
 
-  def self.write_to_gitlab_secrets
+  def self.write_to_gitlab_secrets(path = SECRETS_FILE)
     secret_tokens = gather_gitlab_secrets
 
-    if File.directory?('/etc/gitlab')
-      File.open('/etc/gitlab/gitlab-secrets.json', 'w', 0600) do |f|
+    if File.directory?(File.dirname(path))
+      File.open(path, 'w', 0600) do |f|
         f.puts(Chef::JSONCompat.to_json_pretty(secret_tokens))
         f.chmod(0600)
       end

files/gitlab-cookbooks/package/libraries/settings_dsl.rb

@@ -162,9 +162,18 @@ def load_roles
     end
   end
 
-  def generate_secrets(node_name)
+  def generate_secrets(node_name, path = SecretsHelper::SECRETS_FILE)
+    # Gitlab['node'][SecretsHelper::SKIP_GENERATE_SECRETS_CHEF_ATTR] is set to
+    # true  if we are calling from the 'gitlab-ctrl generate-secrets' command
+    # and we are running the 'config(-ee)' recipe in which case we will generate
+    # secrets in the 'generate_secrets' recipe where it will then be set to
+    # false.
+    return if Gitlab['node'][SecretsHelper::SKIP_GENERATE_SECRETS_CHEF_ATTR] == true
+
+    force_write_secrets = !Gitlab['node'][SecretsHelper::SECRETS_FILE_CHEF_ATTR].nil?
+
     # guards against creating secrets on non-bootstrap node
-    SecretsHelper.read_gitlab_secrets
+    SecretsHelper.read_gitlab_secrets(path)
     generate_default_secrets = Gitlab['package']['generate_default_secrets'] != false
 
     Chef::Log.info("Generating default secrets") if generate_default_secrets
@@ -175,18 +184,18 @@ def generate_secrets(node_name)
       handler.validate_secrets if handler.respond_to?(:validate_secrets)
     end
 
-    if Gitlab['package']['generate_secrets_json_file'] == false
+    if Gitlab['package']['generate_secrets_json_file'] == false && !force_write_secrets
       return unless generate_default_secrets
 
       warning_message = <<~EOS
-        You've enabled generating default secrets but have disabled writing them to gitlab-secrets.json file.
+        You've enabled generating default secrets but have disabled writing them to #{path} file.
         This results in secrets not persisting across `gitlab-ctl reconfigure` runs and can cause issues with functionality.
       EOS
 
       LoggingHelper.warning(warning_message)
     else
-      Chef::Log.info("Generating gitlab-secrets.json file")
-      SecretsHelper.write_to_gitlab_secrets
+      Chef::Log.info("Generating #{path} file")
+      SecretsHelper.write_to_gitlab_secrets(path)
     end
   end
 

files/gitlab-ctl-commands/generate_secrets.rb

@@ -0,0 +1,43 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "#{base_path}/embedded/service/omnibus-ctl/lib/gitlab_ctl/generate_secrets"
+require "#{base_path}/embedded/service/omnibus-ctl/lib/gitlab_ctl/util"
+
+add_command 'generate-secrets', 'Generates secrets used in gitlab.rb', 2 do |cmd_name|
+  begin
+    options = GitlabCtl::GenerateSecrets.parse_options(ARGV)
+  rescue OptionParser::ParseError => e
+    warn "#{e}\n\n#{GitlabCtl::GenerateSecrets::USAGE}"
+    exit 128
+  end
+
+  json_attributes = %(
+  {
+    "run_list": [
+      "recipe[#{GitlabCtl::Util.master_cookbook}::config]",
+      "recipe[gitlab::generate_secrets]"
+    ],
+    "_gitlab_secrets_file_path": "#{options[:secrets_path]}",
+    "_skip_generate_secrets": true
+  }).strip
+  command = %W( cinc-client
+                --log_level info
+                --local-mode
+                --config #{base_path}/embedded/cookbooks/solo.rb
+                --json-attributes /dev/stdin)
+
+  cmd = GitlabCtl::Util.run_command(command.join(" "), live: true, input: json_attributes)
+  remove_old_node_state
+  Kernel.exit 1 unless cmd.status.success?
+end

files/gitlab-ctl-commands/lib/gitlab_ctl/generate_secrets.rb

@@ -0,0 +1,48 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# For testing purposes, if the first path cannot be found load the second
+begin
+  require_relative '../../../../cookbooks/package/libraries/helpers/secrets_helper'
+rescue LoadError
+  require_relative '../../../gitlab-cookbooks/package/libraries/helpers/secrets_helper'
+end
+
+module GitlabCtl
+  class GenerateSecrets
+    USAGE ||= <<~EOS.freeze
+    Usage:
+      gitlab-ctl generate-secrets -f|--file FILE
+
+        f, --file=FILE                  Output secrets to file
+    EOS
+    class << self
+      def parse_options(args)
+        options = {}
+
+        OptionParser.new do |opts|
+          opts.on('-fFILE', '--file=FILE', "Output secrets to file )") do |f|
+            options[:secrets_path] = f
+          end
+          opts.on('-h', '--help', 'Usage help') do
+            Kernel.puts USAGE
+            Kernel.exit 0
+          end
+        end.parse!(args)
+        raise OptionParser::ParseError, "Option --file must be specified." unless options.key?(:secrets_path)
+
+        options
+      end
+    end
+  end
+end

files/gitlab-ctl-commands/lib/gitlab_ctl/util.rb

@@ -23,10 +23,11 @@ def get_command_output(command, user = nil, timeout = nil)
         shell_out.stdout
       end
 
-      def run_command(command, live: false, user: nil, timeout: nil, env: {})
+      def run_command(command, live: false, user: nil, timeout: nil, env: {}, input: nil)
         timeout = Mixlib::ShellOut::DEFAULT_READ_TIMEOUT if timeout.nil?
         shell_out = Mixlib::ShellOut.new(command, timeout: timeout, environment: env)
         shell_out.user = user unless user.nil?
+        shell_out.input = input if input
         shell_out.live_stdout = $stdout if live
         shell_out.live_stderr = $stderr if live
         shell_out.run_command
@@ -172,6 +173,10 @@ def public_attributes_broken?(attribute_key = "gitlab")
 
         !get_public_node_attributes.key?(attribute_key)
       end
+
+      def master_cookbook
+        File.directory?('/opt/gitlab/embedded/cookbooks/gitlab-ee') ? 'gitlab-ee' : 'gitlab'
+      end
     end
   end
 end

spec/chef/cookbooks/gitlab/recipes/generate_secrets_spec.rb

@@ -0,0 +1,124 @@
+# These tests confirm that calling the generate_secrets recipe works correctly
+# both when using the default path and and when using anoptional path to the
+# secrets file. It tests that the file is created and contains secrets. It does
+# not exhaustively test the resulting secrets file contents. Those tests are
+# left to secrets_helper_spec.rb.
+
+require 'chef_helper'
+require_relative '../../../../../files/gitlab-cookbooks/package/libraries/helpers/secrets_helper'
+
+RSpec.describe 'generate_secrets' do
+  let(:chef_runner) { ChefSpec::SoloRunner.new }
+  let(:chef_run) { chef_runner.converge('gitlab::generate_secrets') }
+  let(:node) { chef_runner.node }
+
+  optional_path = '/etc/mygitlab/mysecrets.json'.freeze
+  hex_key = /\h{128}/.freeze
+  rsa_key = /\A-----BEGIN RSA PRIVATE KEY-----\n.+\n-----END RSA PRIVATE KEY-----\n\Z/m.freeze
+  default_secrets_error_regexp = %r{You have enabled writing to the default secrets file location with package\['generate_secrets_json_file'].*}
+
+  def stub_gitlab_secrets_json(secrets)
+    allow(File).to receive(:read).with(SecretsHelper::SECRETS_FILE).and_return(JSON.generate(secrets))
+  end
+
+  def stub_check_secrets
+    rails_keys = new_secrets['gitlab_rails']
+    hex_keys = rails_keys.values_at('db_key_base', 'otp_key_base', 'secret_key_base', 'encrypted_settings_key_base')
+    rsa_keys = rails_keys.values_at('openid_connect_signing_key', 'ci_jwt_signing_key')
+
+    expect(rails_keys.to_a.uniq).to eq(rails_keys.to_a)
+    expect(hex_keys).to all(match(hex_key))
+    expect(rsa_keys).to all(match(rsa_key))
+  end
+
+  before do
+    allow(File).to receive(:directory?).and_call_original
+    allow(File).to receive(:exist?).and_call_original
+    allow(File).to receive(:read).and_call_original
+    allow(File).to receive(:open).and_call_original
+  end
+
+  context 'when default directory does not exist' do
+    it 'does not write secrets to the file' do
+      allow(File).to receive(:directory?).with(File.dirname(SecretsHelper::SECRETS_FILE)).and_return(false)
+      expect(File).not_to receive(:open).with(SecretsHelper::SECRETS_FILE, 'w')
+
+      chef_run
+    end
+  end
+
+  context 'when optional path directory does not exist' do
+    it 'does not write secrets to the file' do
+      allow(File).to receive(:directory?).with(File.dirname(optional_path)).and_return(false)
+      expect(File).not_to receive(:open).with(optional_path, 'w')
+
+      node.normal[SecretsHelper::SECRETS_FILE_CHEF_ATTR] = optional_path
+      chef_run
+    end
+  end
+
+  context 'when optional path directory does exists and we request the optional secret path' do
+    let(:file) { double(:file) }
+    let(:new_secrets) { @new_secrets }
+    before do
+      allow(SecretsHelper).to receive(:system)
+      allow(File).to receive(:directory?).with(File.dirname(optional_path)).and_return(true)
+
+      allow(file).to receive(:puts) { |json| @new_secrets = JSON.parse(json) }
+      allow(file).to receive(:chmod).and_return(true)
+    end
+
+    context 'when there are no existing secrets and generate_secrets_json_file = false' do
+      before do
+        allow(File).to receive(:open).with(optional_path, 'w', 0600).and_yield(file).once
+
+        node.normal[SecretsHelper::SECRETS_FILE_CHEF_ATTR] = optional_path
+        node.normal['package']['generate_secrets_json_file'] = false
+        chef_run
+      end
+
+      it 'writes new secrets to the file, with different values for each' do
+        rails_keys = new_secrets['gitlab_rails']
+        hex_keys = rails_keys.values_at('db_key_base', 'otp_key_base', 'secret_key_base', 'encrypted_settings_key_base')
+        rsa_keys = rails_keys.values_at('openid_connect_signing_key', 'ci_jwt_signing_key')
+
+        expect(rails_keys.to_a.uniq).to eq(rails_keys.to_a)
+        expect(hex_keys).to all(match(hex_key))
+        expect(rsa_keys).to all(match(rsa_key))
+      end
+    end
+  end
+
+  context 'when there are no existing secrets and generate_secrets_json_file = true' do
+    before do
+      allow(File).to receive(:directory?).with(optional_path).and_return(false)
+      allow(File).to receive(:exist?).with(File.dirname(optional_path)).and_return(true)
+      node.normal[SecretsHelper::SECRETS_FILE_CHEF_ATTR] = optional_path
+      node.normal['package']['generate_secrets_json_file'] = true
+    end
+
+    it 'does not write secrets to the file' do
+      expect(File).not_to receive(:directory?).with(File.dirname(optional_path))
+      expect(File).not_to receive(:open).with(optional_path, 'w')
+      expect(LoggingHelper).to receive(:warning).with(default_secrets_error_regexp)
+
+      chef_run
+    end
+  end
+
+  context 'when there are no existing secrets and generate_secrets_json_file is not set' do
+    before do
+      allow(File).to receive(:directory?).with(optional_path).and_return(false)
+      allow(File).to receive(:exist?).with(File.dirname(optional_path)).and_return(true)
+      node.normal[SecretsHelper::SECRETS_FILE_CHEF_ATTR] = optional_path
+    end
+
+    it 'does not write secrets to the file' do
+      expect(File).not_to receive(:directory?).with(File.dirname(optional_path))
+      expect(File).not_to receive(:open).with(optional_path, 'w')
+      expect(LoggingHelper).to receive(:warning).with(default_secrets_error_regexp)
+
+      chef_run
+    end
+  end
+end