Make keep_unprocessed_records feature configurable

joyvuu-dave · joyvuu-dave · commit 82339b07ea6c · 2025-11-03T13:21:11.000-06:00
Keep it off by default
diff --git a/app/jobs/runtime/app_usage_events_cleanup.rb b/app/jobs/runtime/app_usage_events_cleanup.rb
@@ -4,10 +4,11 @@ module VCAP::CloudController
   module Jobs
     module Runtime
       class AppUsageEventsCleanup < VCAP::CloudController::Jobs::CCJob
-        attr_accessor :cutoff_age_in_days, :threshold_for_keeping_unprocessed_records
+        attr_accessor :cutoff_age_in_days, :keep_unprocessed_records, :threshold_for_keeping_unprocessed_records
 
-        def initialize(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+        def initialize(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
           @cutoff_age_in_days = cutoff_age_in_days
+          @keep_unprocessed_records = keep_unprocessed_records
           @threshold_for_keeping_unprocessed_records = threshold_for_keeping_unprocessed_records
         end
 
@@ -16,7 +17,7 @@ def perform
           logger.info('Cleaning up old AppUsageEvent rows')
 
           repository = Repositories::AppUsageEventRepository.new
-          repository.delete_events_older_than(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+          repository.delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
         end
 
         def job_name_in_configuration
diff --git a/app/jobs/v2/services/service_usage_events_cleanup.rb b/app/jobs/v2/services/service_usage_events_cleanup.rb
@@ -4,10 +4,11 @@ module VCAP::CloudController
   module Jobs
     module Services
       class ServiceUsageEventsCleanup < VCAP::CloudController::Jobs::CCJob
-        attr_accessor :cutoff_age_in_days, :threshold_for_keeping_unprocessed_records
+        attr_accessor :cutoff_age_in_days, :keep_unprocessed_records, :threshold_for_keeping_unprocessed_records
 
-        def initialize(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+        def initialize(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
           @cutoff_age_in_days = cutoff_age_in_days
+          @keep_unprocessed_records = keep_unprocessed_records
           @threshold_for_keeping_unprocessed_records = threshold_for_keeping_unprocessed_records
         end
 
@@ -16,7 +17,7 @@ def perform
           logger.info('Cleaning up old ServiceUsageEvent rows')
 
           repository = Repositories::ServiceUsageEventRepository.new
-          repository.delete_events_older_than(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+          repository.delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
         end
 
         def job_name_in_configuration
diff --git a/app/repositories/app_usage_event_repository.rb b/app/repositories/app_usage_event_repository.rb
@@ -151,9 +151,10 @@ def purge_and_reseed_started_apps!
         AppUsageEvent.insert(column_map.keys, usage_query)
       end
 
-      def delete_events_older_than(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+      def delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
         Database::OldRecordCleanup.new(AppUsageEvent, cutoff_age_in_days: cutoff_age_in_days, keep_at_least_one_record: true, keep_running_records: true,
-                                                      keep_unprocessed_records: true, threshold_for_keeping_unprocessed_records: threshold_for_keeping_unprocessed_records).delete
+                                                      keep_unprocessed_records: keep_unprocessed_records,
+                                                      threshold_for_keeping_unprocessed_records: threshold_for_keeping_unprocessed_records).delete
       end
 
       private
diff --git a/app/repositories/service_usage_event_repository.rb b/app/repositories/service_usage_event_repository.rb
@@ -91,9 +91,9 @@ def purge_and_reseed_service_instances!
         ServiceUsageEvent.insert(column_map.keys, usage_query)
       end
 
-      def delete_events_older_than(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+      def delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
         Database::OldRecordCleanup.new(ServiceUsageEvent, cutoff_age_in_days: cutoff_age_in_days, keep_at_least_one_record: true, keep_running_records: true,
-                                                          keep_unprocessed_records: true,
+                                                          keep_unprocessed_records: keep_unprocessed_records,
                                                           threshold_for_keeping_unprocessed_records: threshold_for_keeping_unprocessed_records).delete
       end
     end
diff --git a/config/bosh-lite.yml b/config/bosh-lite.yml
@@ -21,10 +21,12 @@ jobs:
 
 app_usage_events:
   cutoff_age_in_days: 31
+  keep_unprocessed_records: false
   threshold_for_keeping_unprocessed_records: 5000000
 
 service_usage_events:
   cutoff_age_in_days: 31
+  keep_unprocessed_records: false
   threshold_for_keeping_unprocessed_records: 5000000
 
 audit_events:
diff --git a/config/cloud_controller.yml b/config/cloud_controller.yml
@@ -38,10 +38,12 @@ jobs:
 
 app_usage_events:
   cutoff_age_in_days: 31
+  keep_unprocessed_records: false
   threshold_for_keeping_unprocessed_records: 5000000
 
 service_usage_events:
   cutoff_age_in_days: 31
+  keep_unprocessed_records: false
   threshold_for_keeping_unprocessed_records: 5000000
 
 audit_events:
diff --git a/lib/cloud_controller/clock/scheduler.rb b/lib/cloud_controller/clock/scheduler.rb
@@ -6,10 +6,12 @@ module VCAP::CloudController
   class Scheduler
     CLEANUPS = [
       { name: 'app_usage_events', class: Jobs::Runtime::AppUsageEventsCleanup, time: '18:00',
-        arg_from_config: [%i[app_usage_events cutoff_age_in_days], %i[app_usage_events threshold_for_keeping_unprocessed_records]] },
+        arg_from_config: [%i[app_usage_events cutoff_age_in_days], %i[app_usage_events keep_unprocessed_records],
+                          %i[app_usage_events threshold_for_keeping_unprocessed_records]] },
       { name: 'audit_events', class: Jobs::Runtime::EventsCleanup, time: '20:00', arg_from_config: %i[audit_events cutoff_age_in_days] },
       { name: 'service_usage_events', class: Jobs::Services::ServiceUsageEventsCleanup, time: '22:00',
-        arg_from_config: [%i[service_usage_events cutoff_age_in_days], %i[service_usage_events threshold_for_keeping_unprocessed_records]] },
+        arg_from_config: [%i[service_usage_events cutoff_age_in_days], %i[service_usage_events keep_unprocessed_records],
+                          %i[service_usage_events threshold_for_keeping_unprocessed_records]] },
       { name: 'completed_tasks', class: Jobs::Runtime::PruneCompletedTasks, time: '23:00', arg_from_config: %i[completed_tasks cutoff_age_in_days] },
       { name: 'expired_blob_cleanup', class: Jobs::Runtime::ExpiredBlobCleanup, time: '00:00' },
       { name: 'expired_resource_cleanup', class: Jobs::Runtime::ExpiredResourceCleanup, time: '00:30' },
diff --git a/lib/cloud_controller/config_schemas/clock_schema.rb b/lib/cloud_controller/config_schemas/clock_schema.rb
@@ -16,6 +16,7 @@ class ClockSchema < VCAP::Config
           },
           app_usage_events: {
             cutoff_age_in_days: Integer,
+            keep_unprocessed_records: bool,
             threshold_for_keeping_unprocessed_records: Integer
           },
           audit_events: {
@@ -209,6 +210,7 @@ class ClockSchema < VCAP::Config
 
           service_usage_events: {
             cutoff_age_in_days: Integer,
+            keep_unprocessed_records: bool,
             threshold_for_keeping_unprocessed_records: Integer
           },
           default_app_ssh_access: bool,
diff --git a/spec/fixtures/config/port_8181_config.yml b/spec/fixtures/config/port_8181_config.yml
@@ -30,6 +30,7 @@ jobs:
 
 app_usage_events:
   cutoff_age_in_days: 31
+  keep_unprocessed_records: true
   threshold_for_keeping_unprocessed_records: 5000000
 
 audit_events:
diff --git a/spec/unit/jobs/runtime/app_usage_events_cleanup_spec.rb b/spec/unit/jobs/runtime/app_usage_events_cleanup_spec.rb
@@ -4,13 +4,14 @@ module VCAP::CloudController
   module Jobs::Runtime
     RSpec.describe AppUsageEventsCleanup, job_context: :worker do
       let(:cutoff_age_in_days) { 30 }
+      let(:keep_unprocessed_records) { true }
       let(:threshold_for_keeping_unprocessed_records) { 5_000_000 }
       let(:logger) { double(Steno::Logger, info: nil) }
       let!(:event_before_threshold) { AppUsageEvent.make(created_at: (cutoff_age_in_days + 1).days.ago, state: 'STOPPED') }
       let!(:event_after_threshold) { AppUsageEvent.make(created_at: (cutoff_age_in_days - 1).days.ago) }
 
       subject(:job) do
-        AppUsageEventsCleanup.new(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+        AppUsageEventsCleanup.new(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
       end
 
       before do
diff --git a/spec/unit/jobs/services/service_usage_events_cleanup_spec.rb b/spec/unit/jobs/services/service_usage_events_cleanup_spec.rb
@@ -4,13 +4,14 @@ module VCAP::CloudController
   module Jobs::Services
     RSpec.describe ServiceUsageEventsCleanup, job_context: :worker do
       let(:cutoff_age_in_days) { 30 }
+      let(:keep_unprocessed_records) { true }
       let(:threshold_for_keeping_unprocessed_records) { 5_000_000 }
       let(:logger) { double(Steno::Logger, info: nil) }
       let!(:event_before_threshold) { ServiceUsageEvent.make(created_at: (cutoff_age_in_days + 1).days.ago, state: 'DELETED') }
       let!(:event_after_threshold) { ServiceUsageEvent.make(created_at: (cutoff_age_in_days - 1).days.ago) }
 
       subject(:job) do
-        ServiceUsageEventsCleanup.new(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+        ServiceUsageEventsCleanup.new(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
       end
 
       before do
diff --git a/spec/unit/lib/cloud_controller/clock/scheduler_spec.rb b/spec/unit/lib/cloud_controller/clock/scheduler_spec.rb
@@ -18,6 +18,7 @@ module VCAP::CloudController
           },
           app_usage_events: {
             cutoff_age_in_days: 1,
+            keep_unprocessed_records: true,
             threshold_for_keeping_unprocessed_records: 5_000_000
           },
           audit_events: { cutoff_age_in_days: 3 },
@@ -26,6 +27,7 @@ module VCAP::CloudController
           service_operations_initial_cleanup: { frequency_in_seconds: 600 },
           service_usage_events: {
             cutoff_age_in_days: 5,
+            keep_unprocessed_records: true,
             threshold_for_keeping_unprocessed_records: 5_000_000
           },
           completed_tasks: { cutoff_age_in_days: 6 },
@@ -67,7 +69,7 @@ module VCAP::CloudController
 
         expect(clock).to receive(:schedule_daily_job) do |args, &block|
           expect(args).to eql(name: 'app_usage_events', at: '18:00', priority: 0)
-          expect(Jobs::Runtime::AppUsageEventsCleanup).to receive(:new).with(1, 5_000_000).and_call_original
+          expect(Jobs::Runtime::AppUsageEventsCleanup).to receive(:new).with(1, true, 5_000_000).and_call_original
           expect(block.call).to be_instance_of(Jobs::Runtime::AppUsageEventsCleanup)
         end
 
@@ -79,7 +81,7 @@ module VCAP::CloudController
 
         expect(clock).to receive(:schedule_daily_job) do |args, &block|
           expect(args).to eql(name: 'service_usage_events', at: '22:00', priority: 0)
-          expect(Jobs::Services::ServiceUsageEventsCleanup).to receive(:new).with(5, 5_000_000).and_call_original
+          expect(Jobs::Services::ServiceUsageEventsCleanup).to receive(:new).with(5, true, 5_000_000).and_call_original
           expect(block.call).to be_instance_of(Jobs::Services::ServiceUsageEventsCleanup)
         end
 
diff --git a/spec/unit/repositories/app_usage_event_repository_spec.rb b/spec/unit/repositories/app_usage_event_repository_spec.rb
@@ -680,6 +680,7 @@ module Repositories
 
       describe '#delete_events_older_than' do
         let(:cutoff_age_in_days) { 1 }
+        let(:keep_unprocessed_records) { true }
         let(:threshold_for_keeping_unprocessed_records) { 5_000_000 }
 
         before do
@@ -699,15 +700,15 @@ module Repositories
           repository.create_from_process(process)
 
           expect do
-            repository.delete_events_older_than(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+            repository.delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
           end.to change(AppUsageEvent, :count).to(1)
 
           expect(AppUsageEvent.last).to match_app(process)
         end
 
         it 'keeps the last record even if before the cutoff age' do
           expect do
-            repository.delete_events_older_than(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+            repository.delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
           end.to change(AppUsageEvent, :count).to(1)
 
           expect(AppUsageEvent.last.created_at).to be < cutoff_age_in_days.days.ago
diff --git a/spec/unit/repositories/service_usage_event_repository_spec.rb b/spec/unit/repositories/service_usage_event_repository_spec.rb
@@ -157,6 +157,7 @@ module Repositories
       describe '#delete_events_older_than' do
         let!(:service_instance) { ManagedServiceInstance.make }
         let(:cutoff_age_in_days) { 1 }
+        let(:keep_unprocessed_records) { true }
         let(:threshold_for_keeping_unprocessed_records) { 5_000_000 }
 
         before do
@@ -175,15 +176,15 @@ module Repositories
           new_event = repository.create_from_service_instance(service_instance, 'SOME-STATE')
 
           expect do
-            repository.delete_events_older_than(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+            repository.delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
           end.to change(ServiceUsageEvent, :count).to(1)
 
           expect(ServiceUsageEvent.last).to eq(new_event.reload)
         end
 
         it 'keeps the last record even if before the cutoff age' do
           expect do
-            repository.delete_events_older_than(cutoff_age_in_days, threshold_for_keeping_unprocessed_records)
+            repository.delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
           end.to change(ServiceUsageEvent, :count).to(1)
 
           expect(ServiceUsageEvent.last.created_at).to be < cutoff_age_in_days.days.ago
