[COST-7202] Don't summarize vm tables 4 previous month finalization (#5888)

Co-authored-by: lcouzens <lcouzens@redhat.com>

Author: myersCody

koku/api/report/test/util/model_bakery_loader.py

@@ -34,6 +34,7 @@
 from masu.database.ocp_report_db_accessor import OCPReportDBAccessor
 from masu.processor.tasks import update_cost_model_costs
 from masu.util.aws.insert_aws_org_tree import InsertAwsOrgTree
+from masu.util.common import SummaryRangeConfig
 from reporting.models import AWSAccountAlias
 from reporting.models import AWSOrganizationalUnit
 
@@ -466,7 +467,9 @@ def load_openshift_data(self, cluster_id, on_cloud=False):
                     report_period_ids, self.first_start_date, self.last_end_date, provider.uuid
                 )
             with OCPReportDBAccessor(self.schema) as accessor:
-                accessor.populate_ui_summary_tables(self.first_start_date, self.last_end_date, provider.uuid)
+                accessor.populate_ui_summary_tables(
+                    SummaryRangeConfig(start_date=self.first_start_date, end_date=self.last_end_date), provider.uuid
+                )
 
         populate_ocp_topology(self.schema, provider, cluster_id)
 

koku/masu/database/ocp_report_db_accessor.py

@@ -95,11 +95,11 @@ def get_report_periods_before_date(self, date):
         """Get the report periods with report period before provided date."""
         return OCPUsageReportPeriod.objects.filter(report_period_start__lte=date)
 
-    def populate_ui_summary_tables(self, start_date, end_date, source_uuid, tables=UI_SUMMARY_TABLES):
+    def populate_ui_summary_tables(self, summary_range: SummaryRangeConfig, source_uuid, tables=UI_SUMMARY_TABLES):
         """Populate our UI summary tables (formerly materialized views)."""
         sql_params = {
-            "start_date": start_date,
-            "end_date": end_date,
+            "start_date": summary_range.start_date,
+            "end_date": summary_range.end_date,
             "schema": self.schema,
             "source_uuid": source_uuid,
         }
@@ -114,6 +114,9 @@ def populate_ui_summary_tables(self, start_date, end_date, source_uuid, tables=U
         sql_params["month"] = start_date.strftime("%m")
 
         self._populate_gpu_ui_summary_table_with_usage_only(sql_params)
+        if summary_range.summarize_previous_month and not summary_range.is_current_month:
+            # Don't resummarize virtualization UI table if we are summarizing previous month
+            return
         self._populate_virtualization_ui_summary_table(sql_params)
 
     def _populate_gpu_ui_summary_table_with_usage_only(self, params):
@@ -529,9 +532,6 @@ def populate_distributed_cost_sql(
             }
             # Handle distributions that require full month data
             if config.requires_full_month:
-                # Skip full-month distributions on subsequent days when iterating day-by-day
-                if summary_range.skip_full_month:
-                    continue
                 sql_params["start_date"] = summary_range.start_of_month
                 sql_params["end_date"] = summary_range.end_of_month
                 if summary_range.is_current_month:
@@ -540,6 +540,7 @@ def populate_distributed_cost_sql(
                     if dh.now_utc.day in [1, 2, 3]:
                         sql_params["start_date"] = summary_range.start_of_previous_month
                         sql_params["end_date"] = summary_range.end_of_previous_month
+                        summary_range.summarize_previous_month = True
                     else:
                         msg = f"Skipping {cost_model_key} distribution requires full month"
                         LOG.info(log_json(msg=msg, context={"schema": self.schema, "cost_model_key": cost_model_key}))

koku/masu/processor/__init__.py

@@ -141,12 +141,6 @@ def is_feature_cost_4403_ec2_compute_cost_enabled(schema):  # pragma: no cover
     return UNLEASH_CLIENT.is_enabled(unleash_flag, context, fallback_development_true)
 
 
-def is_customer_cost_model_large(schema):  # pragma: no cover
-    """Flag the customer as having a large amount of data for cost model updates."""
-    context = {"schema": schema}
-    return UNLEASH_CLIENT.is_enabled("cost-management.backend.large-customer-cost-model", context)
-
-
 def is_tag_processing_disabled(schema):  # pragma: no cover
     """Flag the customer as tag processing disabled."""
     context = {"schema": schema}

koku/masu/processor/cost_model_cost_updater.py

@@ -10,7 +10,6 @@
 from koku.cache import invalidate_cache_for_tenant_and_cache_key
 from koku.cache import invalidate_view_cache_for_tenant_and_source_type
 from koku.cache import TAG_MAPPING_PREFIX
-from masu.processor import is_customer_cost_model_large
 from masu.processor.aws.aws_cost_model_cost_updater import AWSCostModelCostUpdater
 from masu.processor.azure.azure_cost_model_cost_updater import AzureCostModelCostUpdater
 from masu.processor.gcp.gcp_cost_model_cost_updater import GCPCostModelCostUpdater
@@ -93,11 +92,7 @@ def update_cost_model_costs(self, start_date=None, end_date=None):
             end_date = dh.today.date()
         summary_range = SummaryRangeConfig(start_date=start_date, end_date=end_date)
         if self._updater:
-            if is_customer_cost_model_large(self._schema):
-                for day_range in summary_range.iter_days():
-                    self._updater.update_summary_cost_model_costs(day_range)
-            else:
-                self._updater.update_summary_cost_model_costs(summary_range)
+            self._updater.update_summary_cost_model_costs(summary_range)
             invalidate_view_cache_for_tenant_and_source_type(self._schema, self._provider.type)
             # Invalidate the tag_rate_map for tag mapping
             invalidate_cache_for_tenant_and_cache_key(self._schema, TAG_MAPPING_PREFIX)

koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py

@@ -24,6 +24,7 @@
 from masu.util.aws.common import get_bills_from_provider as aws_get_bills_from_provider
 from masu.util.azure.common import get_bills_from_provider as azure_get_bills_from_provider
 from masu.util.common import date_range_pair
+from masu.util.common import SummaryRangeConfig
 from masu.util.gcp.common import get_bills_from_provider as gcp_get_bills_from_provider
 from masu.util.ocp.common import get_cluster_alias_from_cluster_id
 from masu.util.ocp.common import get_cluster_id_from_provider
@@ -170,7 +171,9 @@ def update_summary_tables(self, start_date, end_date, ocp_provider_uuid, infra_p
 
         # Update the UI tables for the OpenShift provider
         with OCPReportDBAccessor(self._schema) as ocp_accessor:
-            ocp_accessor.populate_ui_summary_tables(start_date, end_date, ocp_provider_uuid)
+            ocp_accessor.populate_ui_summary_tables(
+                SummaryRangeConfig(start_date=start_date, end_date=end_date), ocp_provider_uuid
+            )
 
     def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, start_date, end_date):
         """Update operations specifically for OpenShift on AWS."""

koku/masu/processor/ocp/ocp_cost_model_cost_updater.py

@@ -541,9 +541,7 @@ def distribute_costs_and_update_ui_summary(self, summary_range: SummaryRangeConf
                 summary_range, self._provider_uuid, self._distribution_info
             )
             for month_range in summary_range.iter_summary_range_by_month():
-                accessor.populate_ui_summary_tables(
-                    month_range.summary_start, month_range.summary_end, self._provider.uuid
-                )
+                accessor.populate_ui_summary_tables(month_range, self._provider.uuid)
 
                 if report_period := accessor.report_periods_for_provider_uuid(
                     self._provider_uuid, month_range.summary_start

koku/masu/processor/ocp/ocp_report_parquet_summary_updater.py

@@ -17,6 +17,7 @@
 from masu.database.ocp_report_db_accessor import OCPReportDBAccessor
 from masu.processor.ocp.ocp_cloud_updater_base import OCPCloudUpdaterBase
 from masu.util.common import date_range_pair
+from masu.util.common import SummaryRangeConfig
 from masu.util.ocp.common import get_cluster_alias_from_cluster_id
 from masu.util.ocp.common import get_cluster_id_from_provider
 from reporting.provider.ocp.models import UI_SUMMARY_TABLES
@@ -130,7 +131,9 @@ def update_summary_tables(self, start_date, end_date, **kwargs):
                 accessor.populate_line_item_daily_summary_table_trino(
                     start, end, report_period_id, self._cluster_id, self._cluster_alias, self._provider.uuid
                 )
-                accessor.populate_ui_summary_tables(start, end, self._provider.uuid)
+                accessor.populate_ui_summary_tables(
+                    SummaryRangeConfig(start_date=start, end_date=end), self._provider.uuid
+                )
 
             # This will process POD and STORAGE together
             LOG.info(

koku/masu/test/database/test_ocp_report_db_accessor.py

@@ -1342,6 +1342,23 @@ def test_no_tag_rates(self, mock_psql):
             )
             mock_psql.assert_not_called()
 
+    @patch("masu.database.ocp_report_db_accessor.OCPReportDBAccessor._populate_virtualization_ui_summary_table")
+    @patch("masu.database.ocp_report_db_accessor.OCPReportDBAccessor._populate_gpu_ui_summary_table_with_usage_only")
+    @patch("masu.database.ocp_report_db_accessor.OCPReportDBAccessor._prepare_and_execute_raw_sql_query")
+    def test_populate_ui_summary_tables_skips_virtualization_for_previous_month(
+        self, mock_psql, mock_gpu_populate, mock_virt_populate
+    ):
+        """Test that virtualization UI table is not populated when summarizing previous month (not current month)."""
+        start_date = self.dh.last_month_start.date()
+        end_date = self.dh.last_month_end.date()
+        summary_range = SummaryRangeConfig(start_date=start_date, end_date=end_date, summarize_previous_month=True)
+        source_uuid = self.ocp_provider_uuid
+        with self.accessor as acc:
+            acc.populate_ui_summary_tables(summary_range, source_uuid, tables=[])
+        mock_psql.assert_not_called()
+        mock_gpu_populate.assert_called_once()
+        mock_virt_populate.assert_not_called()
+
     @patch("masu.database.ocp_report_db_accessor.OCPReportDBAccessor.schema_exists_trino", return_value=False)
     @patch("masu.database.ocp_report_db_accessor.OCPReportDBAccessor._prepare_and_execute_raw_sql_query")
     def test__populate_virtualization_ui_summary_table_no_trino_schema(self, mock_psql, mock_schema_exists):

koku/masu/test/util/test_common.py

@@ -664,14 +664,14 @@ def test_iter_summary_range_by_month_with_extended_summary_dates(self):
         self.assertEqual(ranges[1].summary_start, date(2025, 2, 1))
         self.assertEqual(ranges[1].summary_end, date(2025, 2, 3))
 
-    def test_iter_summary_range_by_month_preserves_skip_full_month(self):
-        """Test iter_summary_range_by_month preserves skip_full_month flag."""
+    def test_iter_summary_range_by_month_preserves_summarize_previous_month(self):
+        """Test iter_summary_range_by_month preserves summarize_previous_month flag."""
         config = common_utils.SummaryRangeConfig(
-            start_date=date(2025, 1, 1), end_date=date(2025, 2, 3), skip_full_month=True
+            start_date=date(2025, 1, 1), end_date=date(2025, 2, 3), summarize_previous_month=True
         )
 
         for range_config in config.iter_summary_range_by_month():
-            self.assertTrue(range_config.skip_full_month)
+            self.assertTrue(range_config.summarize_previous_month)
 
 
 class NamedTemporaryGZipTests(TestCase):

koku/masu/util/common.py

@@ -532,10 +532,10 @@ class SummaryRangeConfig(BaseModel):
     end_date: date
     summary_starts: list[date] = Field(default_factory=list)
     summary_ends: list[date] = Field(default_factory=list)
-    skip_full_month: bool = Field(
+    summarize_previous_month: bool = Field(
         default=False,
-        description="When True, skip distributions that require full month data. "
-        "Set automatically by iter_days() for days after the first.",
+        description="When True, indicates we are summarizing previous month data (e.g., GPU finalization). "
+        "Used to skip certain UI table updates that don't need to be reprocessed.",
     )
 
     @field_validator("start_date", "end_date", mode="before")
@@ -591,18 +591,6 @@ def summary_start(self) -> date:
     def summary_end(self) -> date:
         return max(self.summary_ends)
 
-    def iter_days(self) -> Iterator[Self]:
-        """Yield a SummaryRangeConfig for each day in the range.
-
-        Useful for splitting processing into individual days for large customers.
-        The first day will have skip_full_month=False (runs full-month distributions),
-        subsequent days will have skip_full_month=True (skips full-month distributions).
-        """
-        first_day = True
-        for day in DateHelper().list_days(self.start_date, self.end_date):
-            yield SummaryRangeConfig(start_date=day, end_date=day, skip_full_month=not first_day)
-            first_day = False
-
     def iter_summary_range_by_month(self) -> Iterator[Self]:
         """Yield a SummaryRangeConfig for each month in the summary range.
 
@@ -616,4 +604,6 @@ def iter_summary_range_by_month(self) -> Iterator[Self]:
             - SummaryRangeConfig(2025-02-01, 2025-02-03)
         """
         for start, end in DateHelper().list_month_tuples(self.summary_start, self.summary_end):
-            yield SummaryRangeConfig(start_date=start, end_date=end, skip_full_month=self.skip_full_month)
+            yield SummaryRangeConfig(
+                start_date=start, end_date=end, summarize_previous_month=self.summarize_previous_month
+            )