Merge branch 'vshushlin/clickhouse-databases' into 'master'

Add support for ClickHouse databases

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7179

Merged-by: Balasankar 'Balu' C <[email protected]>
Approved-by: Adam Hegyi <[email protected]>
Approved-by: Clemens Beck <[email protected]>
Approved-by: Balasankar 'Balu' C <[email protected]>
Reviewed-by: Balasankar 'Balu' C <[email protected]>
Reviewed-by: Vladimir Shushlin <[email protected]>
Reviewed-by: Clemens Beck <[email protected]>
Co-authored-by: Vladimir Shushlin <[email protected]>

Author: balasankarc

files/gitlab-config-template/gitlab.rb.template

@@ -765,6 +765,13 @@ external_url 'GENERATED_EXTERNAL_URL'
 # gitlab_rails['databases']['ci']['db_database'] = 'gitlabhq_production'
 # gitlab_rails['databases']['ci']['database_tasks'] = false
 
+### GitLab ClickHouse connection settings
+###! EXPERIMENTAL
+# gitlab_rails['clickhouse_databases']['main']['database'] = 'dbname'
+# gitlab_rails['clickhouse_databases']['main']['url'] = 'https://example.com/path'
+# gitlab_rails['clickhouse_databases']['main']['username'] = 'gitlab'
+# gitlab_rails['clickhouse_databases']['main']['password'] = 'password'
+
 ### GitLab Redis settings
 ###! Connect to your own Redis instance
 ###! Docs: https://docs.gitlab.com/omnibus/settings/redis.html

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -427,6 +427,7 @@
 default['gitlab']['gitlab_rails']['db_application_name'] = nil
 
 default['gitlab']['gitlab_rails']['databases'] = {}
+default['gitlab']['gitlab_rails']['clickhouse_databases'] = {}
 
 # Automatic Database Reindexing
 # See https://docs.gitlab.com/omnibus/settings/database.html#automatic-database-reindexing

files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb

@@ -196,6 +196,18 @@
   sensitive true
 end
 
+templatesymlink "Create a clickhouse.yml and create a symlink to Rails root" do
+  link_from File.join(gitlab_rails_source_dir, "config/click_house.yml")
+  link_to File.join(gitlab_rails_etc_dir, "click_house.yml")
+  source "click_house.yml.erb"
+  owner "root"
+  group gitlab_group
+  mode "0640"
+  variables node['gitlab']['gitlab_rails'].to_hash
+  dependent_services.each { |svc| notifies :restart, svc }
+  sensitive true
+end
+
 redis_url = redis_helper.redis_url
 redis_sentinels = node['gitlab']['gitlab_rails']['redis_sentinels']
 redis_sentinels_password = node['gitlab']['gitlab_rails']['redis_sentinels_password']

files/gitlab-cookbooks/gitlab/templates/default/click_house.yml.erb

@@ -0,0 +1,16 @@
+# This file is managed by gitlab-ctl. Manual changes will be
+# erased! To change the contents below, edit /etc/gitlab/gitlab.rb
+# and run `sudo gitlab-ctl reconfigure`.
+
+production:
+  <% @clickhouse_databases.each do |database, attributes| %>
+  <%= database %>:
+    database: <%= quote(attributes['database']) %>
+    url: <%= quote(attributes['url']) %>
+    username: <%= quote(attributes['username']) %>
+    password: <%= quote(attributes['password']) %>
+    variables:
+      enable_http_compression: 1
+      date_time_input_format: basic # needed for CH cloud
+  <% end %>
+

spec/chef/cookbooks/gitlab/recipes/gitlab-rails/click_house_yml_spec.rb

@@ -0,0 +1,71 @@
+require 'chef_helper'
+
+RSpec.describe 'gitlab::gitlab-rails' do
+  describe 'ClickHouse database settings' do
+    let(:chef_run) { ChefSpec::SoloRunner.new(step_into: 'templatesymlink').converge('gitlab::default') }
+    let(:clickhouse_yml_template) { chef_run.template('/var/opt/gitlab/gitlab-rails/etc/click_house.yml') }
+    let(:clickhouse_yml_file_content) { ChefSpec::Renderer.new(chef_run, clickhouse_yml_template).content }
+    let(:clickhouse_yml) { YAML.safe_load(clickhouse_yml_file_content, [], [], true, symbolize_names: true) }
+
+    before do
+      allow(Gitlab).to receive(:[]).and_call_original
+      allow(File).to receive(:symlink?).and_call_original
+    end
+
+    context 'with default settings' do
+      it 'renders empty clickhouse.yml' do
+        expect(clickhouse_yml[:production]).to eq(nil)
+      end
+    end
+
+    context 'with databases setup' do
+      before do
+        stub_gitlab_rb(
+          gitlab_rails: {
+            clickhouse_databases: {
+              main: {
+                database: 'production',
+                url: 'https://example.com/path',
+                username: 'gitlab',
+                password: 'password'
+              },
+              main2: {
+                database: 'production2',
+                url: 'https://example.com/path2',
+                username: 'gitlab2',
+                password: 'password2'
+              }
+            }
+          }
+        )
+      end
+
+      it 'renders clickhouse.yml using these settings' do
+        expect(clickhouse_yml[:production]).to eq(
+          {
+            main: {
+              database: 'production',
+              url: 'https://example.com/path',
+              username: 'gitlab',
+              password: 'password',
+              variables: {
+                enable_http_compression: 1,
+                date_time_input_format: "basic"
+              }
+            },
+            main2: {
+              database: 'production2',
+              url: 'https://example.com/path2',
+              username: 'gitlab2',
+              password: 'password2',
+              variables: {
+                enable_http_compression: 1,
+                date_time_input_format: "basic"
+              }
+            }
+          }
+        )
+      end
+    end
+  end
+end