Merge pull request #39 from DataKitchen/release/4.0.12

Release/4.0.12

Author: datakitchen-devops

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "4.0.9"
+version = "4.0.12"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "[email protected]" },

testgen/commands/run_refresh_score_cards_results.py

@@ -2,7 +2,7 @@
 import logging
 import time
 
-from testgen.common.models import with_database_session
+from testgen.common.models import get_current_session, with_database_session
 from testgen.common.models.scores import (
     SCORE_CATEGORIES,
     ScoreCard,
@@ -36,6 +36,7 @@ def run_refresh_score_cards_results(
         LOG.exception("CurrentStep: Stopping scorecards results refresh after unexpected error")
         return
 
+    db_session = get_current_session()
     for definition in definitions:
         LOG.info(
             "CurrentStep: Refreshing results for scorecard %s in project %s",
@@ -45,6 +46,10 @@ def run_refresh_score_cards_results(
 
         try:
             fresh_score_card = definition.as_score_card()
+            definition.results = []
+            definition.breakdown = []
+            db_session.flush([definition])
+
             definition.results = _score_card_to_results(fresh_score_card)
             definition.breakdown = _score_definition_to_results_breakdown(definition)
             if add_history_entry:

testgen/common/logs.py

@@ -33,7 +33,7 @@ def configure_logging(
 
             file_handler = ConcurrentTimedRotatingFileHandler(
                 get_log_full_path(),
-                when="D",
+                when="MIDNIGHT",
                 interval=1,
                 backupCount=int(settings.LOG_FILE_MAX_QTY),
             )

testgen/common/models/scores.py

@@ -361,17 +361,22 @@ def recalculate_scores_history(self) -> None:
         )
         overall_scores = get_current_session().execute(query).mappings().all()
         current_history: dict[tuple[datetime, str, str], ScoreDefinitionResultHistoryEntry] = {}
-        for entry in self.history:
-            current_history[(entry.last_run_time, entry.category,)] = entry
-
         renewed_history: dict[tuple[datetime, str, str], float] = {}
+
         for scores in overall_scores:
             renewed_history[(scores["last_run_time"], "score",)] = scores["score"]
             renewed_history[(scores["last_run_time"], "cde_score",)] = scores["cde_score"]
 
+        for entry in self.history:
+            entry_key = (entry.last_run_time, entry.category,)
+            if entry_key in renewed_history:
+                current_history[entry_key] = entry
+
         for key, entry in current_history.items():
             entry.score = renewed_history[key]
 
+        self.history = list(current_history.values())
+
     def _get_raw_query_filters(self, cde_only: bool = False, prefix: str | None = None) -> list[str]:
         values_by_field = defaultdict(list)
         for filter_ in self.filters:

testgen/common/process_service.py

@@ -13,13 +13,13 @@ def get_current_process_id():
 
 
 def kill_profile_run(process_id):
-    keywords = ["run-profile"]
+    keywords = ["/dk/bin/testgen", "run-profile"]
     status, message = kill_process(process_id, keywords)
     return status, message
 
 
 def kill_test_run(process_id):
-    keywords = ["run-tests"]
+    keywords = ["/dk/bin/testgen", "run-tests"]
     status, message = kill_process(process_id, keywords)
     return status, message
 
@@ -31,7 +31,7 @@ def kill_process(process_id, keywords=None):
         return False, msg
     try:
         process = psutil.Process(process_id)
-        if process.name().lower() != "testgen":
+        if process.name().lower() not in ["testgen", "python3"]:
             message = f"The process was not killed because the process_id {process_id} is not a testgen process. Details: {process.name()}"
             LOG.error(f"kill_process: {message}")
             return False, message

testgen/template/dbupgrade/0137_incremental_upgrade.sql

@@ -0,0 +1,5 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+UPDATE job_schedules
+    SET kwargs = kwargs - 'project_code' || jsonb_build_object('project_key', kwargs->'project_code')
+WHERE key = 'run-tests';

testgen/ui/views/score_explorer.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from io import BytesIO
 from typing import ClassVar
 
@@ -246,7 +247,7 @@ def save_score_definition(_) -> None:
         latest_run = max(
             profiling_queries.get_latest_run_date(project_code),
             test_run_queries.get_latest_run_date(project_code),
-            key=lambda run: getattr(run, "run_time", 0),
+            key=lambda run: getattr(run, "run_time", datetime.min),
         )
 
         refresh_kwargs = {

testgen/ui/views/test_runs.py

@@ -133,7 +133,7 @@ def arg_value_input(self) -> tuple[bool, list[typing.Any], dict[str, typing.Any]
             display_column="test_suite",
             required=True,
         )
-        return bool(ts_name), [], {"project_code": self.project_code, "test_suite_key": ts_name}
+        return bool(ts_name), [], {"project_key": self.project_code, "test_suite_key": ts_name}
 
 
 def render_empty_state(project_code: str, user_can_run: bool) -> bool: