Merge branch 'pedropombeiro/421201/implement-ch-worker' into 'master'

Add cronjob to sync finished builds to ClickHouse

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7169

Merged-by: Balasankar 'Balu' C <[email protected]>
Approved-by: Clemens Beck <[email protected]>
Approved-by: Balasankar 'Balu' C <[email protected]>
Reviewed-by: Pedro Pombeiro <[email protected]>
Reviewed-by: Clemens Beck <[email protected]>
Co-authored-by: Pedro Pombeiro <[email protected]>

Author: balasankarc

files/gitlab-config-template/gitlab.rb.template

@@ -935,7 +935,7 @@ external_url 'GENERATED_EXTERNAL_URL'
 
 ### Registry database
 ###! Docs: https://docs.gitlab.com/ee/administration/packages/container_registry.html?tab=Linux+package+%28Omnibus%29#configure-a-metadata-database-for-the-container-registry
-# registry['database'] = { 
+# registry['database'] = {
 #   'enabled' => true,
 #   'host' => 'localhost',
 #   'port' => 5432,
@@ -953,7 +953,7 @@ external_url 'GENERATED_EXTERNAL_URL'
 #       'maxidle' => 25,
 #       'maxopen' => 25,
 #       'maxlifetime' => '5m'
-#   } 
+#   }
 # }
 
 ### Registry notifications endpoints
@@ -2704,6 +2704,8 @@ external_url 'GENERATED_EXTERNAL_URL'
 # gitlab_rails['elastic_index_bulk_cron'] = "*/1 * * * *"
 # gitlab_rails['analytics_devops_adoption_create_all_snapshots_worker_cron'] = "0 4 * * 0"
 # gitlab_rails['ci_runners_stale_group_runners_prune_worker_cron'] = "30 * * * *"
+# gitlab_rails['click_house_ci_finished_builds_sync_worker_cron'] = "*/3 * * * *"
+# gitlab_rails['click_house_ci_finished_builds_sync_worker_args'] = [1]
 
 ################################################################################
 ## Kerberos (EE Only)

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -149,6 +149,8 @@
 default['gitlab']['gitlab_rails']['incoming_email_inbox_options'] = nil
 default['gitlab']['gitlab_rails']['incoming_email_delivery_method'] = "webhook"
 default['gitlab']['gitlab_rails']['incoming_email_auth_token'] = nil
+default['gitlab']['gitlab_rails']['click_house_ci_finished_builds_sync_worker_cron'] = nil
+default['gitlab']['gitlab_rails']['click_house_ci_finished_builds_sync_worker_args'] = nil
 
 default['gitlab']['gitlab_rails']['service_desk_email_enabled'] = false
 default['gitlab']['gitlab_rails']['service_desk_email_address'] = nil

files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb

@@ -700,6 +700,15 @@ production: &base
       cron: "<%= @ci_runners_stale_group_runners_prune_worker_cron %>"
     <% end %>
 
+    # Periodically sync finished builds from p_ci_finished_build_ch_sync_events to ClickHouse
+    <% unless @click_house_ci_finished_builds_sync_worker_cron.nil? %>
+    click_house_ci_finished_builds_sync_worker:
+      cron: "<%= @click_house_ci_finished_builds_sync_worker_cron %>"
+      <% unless @click_house_ci_finished_builds_sync_worker_args.nil? %>
+      args: <%= @click_house_ci_finished_builds_sync_worker_args %>
+      <% end %>
+    <% end %>
+
   ## Geo
   # NOTE: These settings will only take effect if Geo is enabled
   geo:

spec/chef/cookbooks/gitlab/recipes/gitlab-rails/gitlab_yml/cron_jobs_spec.rb

@@ -52,6 +52,7 @@
         'ci_runners_stale_group_runners_prune_worker_cron'       | 'ci_runners_stale_group_runners_prune_worker_cron'
         'ci_runner_versions_reconciliation_worker'               | 'ci_runner_versions_reconciliation_worker_cron'
         'ci_runners_stale_machines_cleanup_worker'               | 'ci_runners_stale_machines_cleanup_worker_cron'
+        'click_house_ci_finished_builds_sync_worker'             | 'click_house_ci_finished_builds_sync_worker_cron'
       end
 
       with_them do