Merge pull request #2057 from gooddata/GRIF-510

GRIF-510: Allow configure increase parallel processing for each step on LCM Rollout Brick

Author: hung-nguyen-hoang

VERSION

@@ -1 +1 @@
-3.7.99
+3.7.100

lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb

@@ -57,7 +57,7 @@ def migrate_date_dimension(params, segment_info)
           results = []
           client = params.gdc_gd_client
           number_of_threads = Integer(params.number_of_threads_migrate_dates || '10')
-          GoodData.logger.info "Number of threads using migration dates #{number_of_threads}"
+          GoodData.logger.info "Number of threads using migration dates #{number_of_threads}" if number_of_threads != 10
           latest_blueprint = segment_info[:from_blueprint]
           # don't migrate when latest master doesn't contain custom v2 date.
           return results unless contain_v2?(latest_blueprint)

lib/gooddata/lcm/actions/synchronize_cas.rb

@@ -41,6 +41,9 @@ class SynchronizeComputedAttributes < BaseAction
 
         description 'Sync failed list'
         param :sync_failed_list, instance_of(Type::HashType), required: false
+
+        description 'Number Of Threads'
+        param :number_of_threads_synchronize_cas, instance_of(Type::StringType), required: false, default: '10'
       end
 
       class << self
@@ -57,14 +60,16 @@ def call(params)
           client = params.gdc_gd_client
           collect_synced_status = collect_synced_status(params)
           failed_projects = ThreadSafe::Array.new
+          number_of_threads = Integer(params.number_of_threads_synchronize_cas || '10')
+          GoodData.logger.info "Number of threads using synchronize computed attributes #{number_of_threads}" if number_of_threads != 10
 
           params.synchronize.each do |info|
             from = info.from
             to_projects = info.to
 
             params.gdc_logger.info "Synchronize Computed Attributes from project pid: #{from}"
 
-            to_projects.peach do |entry|
+            to_projects.peach(number_of_threads) do |entry|
               ca_scripts = entry[:ca_scripts]
               next unless ca_scripts
 

lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb

@@ -53,6 +53,9 @@ class SynchronizeETLsInSegment < BaseAction
 
         description 'Sync failed list'
         param :sync_failed_list, instance_of(Type::HashType), required: false
+
+        description 'Number Of Threads'
+        param :number_of_threads_synchronize_etls, instance_of(Type::StringType), required: false, default: '10'
       end
 
       # will be updated later based on the way etl synchronization
@@ -73,6 +76,8 @@ def call(params)
 
           schedule_additional_params = params.schedule_additional_params || params.additional_params
           schedule_additional_hidden_params = params.schedule_additional_hidden_params || params.additional_hidden_params
+          number_of_threads = Integer(params.number_of_threads_synchronize_etls || '10')
+          GoodData.logger.info "Number of threads using synchronize ETL processes #{number_of_threads}" if number_of_threads != 10
 
           synchronize_segments = params.synchronize.group_by do |info|
             info[:segment_id]
@@ -132,7 +137,7 @@ def call(params)
             next if delete_extra_process_schedule && from_project_etl_names.nil?
 
             to_projects = info.to
-            to_projects.peach do |entry|
+            to_projects.peach(number_of_threads) do |entry|
               pid = entry[:pid]
               next if collect_synced_status && sync_failed_project(pid, params)
 

lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb

@@ -40,6 +40,9 @@ class SynchronizeKDDashboardPermissions < BaseAction
 
         description 'Sync failed list'
         param :sync_failed_list, instance_of(Type::HashType), required: false
+
+        description 'Number Of Threads'
+        param :number_of_threads_synchronize_kd_dashboard_permissions, instance_of(Type::StringType), required: false, default: '10'
       end
 
       class << self
@@ -49,6 +52,8 @@ def call(params)
           failed_projects = ThreadSafe::Array.new
 
           disable_kd_dashboard_permission = GoodData::Helpers.to_boolean(params.disable_kd_dashboard_permission)
+          number_of_threads = Integer(params.number_of_threads_synchronize_kd_dashboard_permissions || '10')
+          GoodData.logger.info "Number of threads using synchronize KD dashboard permissions #{number_of_threads}" if number_of_threads != 10
 
           # rubocop:disable Style/UnlessElse
           unless disable_kd_dashboard_permission
@@ -69,7 +74,7 @@ def call(params)
               from_dashboards = from_project.analytical_dashboards
 
               params.gdc_logger.info "Transferring #{dashboard_type} Dashboard permission, from project: '#{from_project.title}', PID: '#{from_project.pid}' for dashboard(s): #{from_dashboards.map { |d| "#{d.title.inspect}" }.join(', ')}" # rubocop:disable Metrics/LineLength
-              to_projects_info.peach do |item|
+              to_projects_info.peach(number_of_threads) do |item|
                 to_project_pid = item[:pid]
                 next if sync_failed_project(to_project_pid, params)
 

lib/gooddata/lcm/actions/synchronize_ldm.rb

@@ -95,7 +95,7 @@ def sync_segment_ldm(params, segment_info)
           include_deprecated = params.include_deprecated.to_b
           update_preference = params[:update_preference]
           number_of_threads = Integer(params.number_of_threads_synchronize_ldm || '10')
-          GoodData.logger.info "Number of threads using synchronize logical data model #{number_of_threads}"
+          GoodData.logger.info "Number of threads using synchronize logical data model #{number_of_threads}" if number_of_threads != 10
 
           exist_fallback_to_hard_sync_config = !update_preference.nil? && !update_preference[:fallback_to_hard_sync].nil?
           include_maql_fallback_hard_sync = exist_fallback_to_hard_sync_config && Helpers.to_bool('fallback_to_hard_sync', update_preference[:fallback_to_hard_sync])

lib/gooddata/lcm/actions/synchronize_ldm_layout.rb

@@ -33,6 +33,9 @@ class SynchronizeLdmLayout < BaseAction
 
         description 'Sync failed list'
         param :sync_failed_list, instance_of(Type::HashType), required: false
+
+        description 'Number Of Threads'
+        param :number_of_threads_synchronize_ldm_layout, instance_of(Type::StringType), required: false, default: '10'
       end
 
       RESULT_HEADER = %i[from to status]
@@ -45,6 +48,8 @@ def call(params)
           development_client = params.development_client
           gdc_logger = params.gdc_logger
           collect_synced_status = collect_synced_status(params)
+          number_of_threads = Integer(params.number_of_threads_synchronize_ldm_layout || '10')
+          GoodData.logger.info "Number of threads using synchronize ldm layout #{number_of_threads}" if number_of_threads != 10
           failed_projects = ThreadSafe::Array.new
 
           params.synchronize.peach do |info|
@@ -64,7 +69,7 @@ def call(params)
             if from_ldm_layout&.dig('ldmLayout', 'layout').nil? || from_ldm_layout['ldmLayout']['layout'].empty?
               gdc_logger.info "Project: '#{from_title}', PID: '#{from_pid}' has no ldm layout, skip synchronizing ldm layout."
             else
-              to_projects.peach do |to|
+              to_projects.peach(number_of_threads) do |to|
                 pid = to[:pid]
                 to_project = client.projects(pid)
                 unless to_project

lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb

@@ -42,13 +42,18 @@ class SynchronizePPDashboardPermissions < BaseAction
 
         description 'Sync failed list'
         param :sync_failed_list, instance_of(Type::HashType), required: false
+
+        description 'Number Of Threads'
+        param :number_of_threads_synchronize_pp_dashboard_permissions, instance_of(Type::StringType), required: false, default: '10'
       end
 
       class << self
         def call(params)
           results = []
           disable_pp_dashboard_permission = GoodData::Helpers.to_boolean(params.disable_pp_dashboard_permission)
           collect_synced_status = collect_synced_status(params)
+          number_of_threads = Integer(params.number_of_threads_synchronize_pp_dashboard_permissions || '10')
+          GoodData.logger.info "Number of threads using synchronize pixel perfect dashboard permissions #{number_of_threads}" if number_of_threads != 10
           failed_projects = ThreadSafe::Array.new
 
           if disable_pp_dashboard_permission
@@ -72,7 +77,7 @@ def call(params)
               source_dashboards = from.dashboards
 
               params.gdc_logger.info "Transferring Pixel Perfect Dashboard permission, from project: '#{from.title}', PID: '#{from.pid}' for dashboard(s): #{source_dashboards.map { |d| "#{d.title.inspect}" }.join(', ')}" # rubocop:disable Metrics/LineLength
-              to_projects.peach do |entry|
+              to_projects.peach(number_of_threads) do |entry|
                 pid = entry[:pid]
                 next if sync_failed_project(pid, params)
 

lib/gooddata/lcm/actions/synchronize_user_groups.rb

@@ -36,6 +36,9 @@ class SynchronizeUserGroups < BaseAction
 
         description 'Sync failed list'
         param :sync_failed_list, instance_of(Type::HashType), required: false
+
+        description 'Number Of Threads'
+        param :number_of_threads_synchronize_user_groups, instance_of(Type::StringType), required: false, default: '10'
       end
 
       class << self
@@ -46,6 +49,8 @@ def call(params)
 
           client = params.gdc_gd_client
           development_client = params.development_client
+          number_of_threads = Integer(params.number_of_threads_synchronize_user_groups || '10')
+          GoodData.logger.info "Number of threads using synchronize user groups #{number_of_threads}" if number_of_threads != 10
 
           params.synchronize.peach do |info|
             from_project = info.from
@@ -57,7 +62,7 @@ def call(params)
               next
             end
 
-            to_projects.peach do |entry|
+            to_projects.peach(number_of_threads) do |entry|
               pid = entry[:pid]
               next if sync_failed_project(pid, params)