Fix airbyte deploy job and stop apply policy tagging for final tables only

Author: asatwal

lib/dfe/analytics/jobs/airbyte_deploy_job.rb

@@ -37,10 +37,10 @@ def perform
 
           Rails.logger.info('Finished WaitForSync')
 
-          # Trigger policy tagging
-          BigQueryApplyPolicyTagsJob.perform_later
+          # Trigger policy tagging for final tables
+          DfE::Analytics::Services::ApplyAirbyteFinalTablesPolicyTags.call
 
-          Rails.logger.info('Finished BigQueryApplyPolicyTagsJob')
+          Rails.logger.info('Finished AirbyteDeployJob')
         rescue StandardError => e
           Rails.logger.error(e.message)
           raise "AirbyteDeployJob failed: #{e.message}"

lib/dfe/analytics/tasks/big_query_apply_policy_tags.rake

@@ -7,8 +7,9 @@ namespace :dfe do
       puts "Calling DfE::Analytics::Services::ApplyAirbyteFinalTablesPolicyTags.call(delay_in_minutes: #{delay})"
       DfE::Analytics::Services::ApplyAirbyteFinalTablesPolicyTags.call(delay_in_minutes: delay)
 
-      puts "Calling DfE::Analytics::Services::ApplyAirbyteInternalTablesPolicyTags.call(delay_in_minutes: #{delay})"
-      DfE::Analytics::Services::ApplyAirbyteInternalTablesPolicyTags.call(delay_in_minutes: delay)
+      # Internal tables no longer created by airbyte - remove following code following testing
+      #puts "Calling DfE::Analytics::Services::ApplyAirbyteInternalTablesPolicyTags.call(delay_in_minutes: #{delay})"
+      #DfE::Analytics::Services::ApplyAirbyteInternalTablesPolicyTags.call(delay_in_minutes: delay)
     end
   end
 end

spec/dfe/analytics/jobs/airbyte_deploy_job_spec.rb

@@ -31,7 +31,7 @@
     allow(Services::Airbyte::JobLast).to receive(:call).and_return(nil)
     allow(Services::Airbyte::StartSync).to receive(:call).and_return(job_id)
     allow(Services::Airbyte::WaitForSync).to receive(:call).and_return('succeeded')
-    allow(DfE::Analytics::Jobs::BigQueryApplyPolicyTagsJob).to receive(:perform_later)
+    allow(DfE::Analytics::Services::ApplyAirbyteFinalTablesPolicyTags).to receive(:call)
   end
 
   it 'calls all Airbyte orchestration steps in order' do
@@ -54,7 +54,7 @@
       connection_id: connection_id,
       job_id: job_id
     )
-    expect(DfE::Analytics::Jobs::BigQueryApplyPolicyTagsJob).to have_received(:perform_later)
+    expect(DfE::Analytics::Services::ApplyAirbyteFinalTablesPolicyTags).to have_received(:call)
   end
 
   context 'when last job is running' do

spec/dfe/analytics/tasks/big_query_apply_policy_tag_spec.rb

@@ -1,8 +1,5 @@
 require 'rake'
 
-require_relative '../../../../lib/dfe/analytics/services/apply_airbyte_final_tables_policy_tags'
-require_relative '../../../../lib/dfe/analytics/services/apply_airbyte_internal_tables_policy_tags'
-
 RSpec.describe 'dfe:analytics:big_query_apply_policy_tags' do
   before :all do
     Rake.application.rake_require('big_query_apply_policy_tags', [File.expand_path('../../../../lib/dfe/analytics/tasks', __dir__)])
@@ -15,7 +12,6 @@
 
   before do
     allow(DfE::Analytics::Services::ApplyAirbyteFinalTablesPolicyTags).to receive(:call)
-    allow(DfE::Analytics::Services::ApplyAirbyteInternalTablesPolicyTags).to receive(:call)
     task.reenable # allow re-invocation in the same test run
   end
 
@@ -24,7 +20,6 @@
       task.invoke('15')
 
       expect(DfE::Analytics::Services::ApplyAirbyteFinalTablesPolicyTags).to have_received(:call).with(delay_in_minutes: 15)
-      expect(DfE::Analytics::Services::ApplyAirbyteInternalTablesPolicyTags).to have_received(:call).with(delay_in_minutes: 15)
     end
   end
 
@@ -33,7 +28,6 @@
       task.invoke
 
       expect(DfE::Analytics::Services::ApplyAirbyteFinalTablesPolicyTags).to have_received(:call).with(delay_in_minutes: 0)
-      expect(DfE::Analytics::Services::ApplyAirbyteInternalTablesPolicyTags).to have_received(:call).with(delay_in_minutes: 0)
     end
   end
 end