Merge pull request ClickHouse#78356 from ClickHouse/ci_fix_custom_data

CI: Fix workflow custom data storage and retrieval

Author: maxknv

ci/praktika/info.py

@@ -160,23 +160,12 @@ def store_custom_data(self, key, value):
         assert (
             self.env.JOB_NAME == "Config Workflow"
         ), "Custom data can be stored only in Config Workflow Job"
-        custom_data = {key: value}
-        if Path(Settings.CUSTOM_DATA_FILE).is_file():
-            with open(Settings.CUSTOM_DATA_FILE, "r", encoding="utf8") as f:
-                custom_data = json.load(f)
-                custom_data[key] = value
-        with open(Settings.CUSTOM_DATA_FILE, "w", encoding="utf8") as f:
-            json.dump(custom_data, f, indent=4)
+        workflow_config = RunConfig.from_fs(self.env.WORKFLOW_NAME)
+        workflow_config.custom_data[key] = value
+        workflow_config.dump()
 
     def get_custom_data(self, key=None):
-        # todo: remove intermediary file CUSTOM_DATA_FILE and store/get directly to/from RunConfig
-        if Path(Settings.CUSTOM_DATA_FILE).is_file():
-            # first check CUSTOM_DATA_FILE in case data is not yet in RunConfig
-            #   might happen if data stored in one pre-hook and fetched in another
-            with open(Settings.CUSTOM_DATA_FILE, "r", encoding="utf8") as f:
-                custom_data = json.load(f)
-        else:
-            custom_data = RunConfig.from_fs(self.env.WORKFLOW_NAME).custom_data
+        custom_data = RunConfig.from_fs(self.env.WORKFLOW_NAME).custom_data
         if key:
             return custom_data.get(key, None)
         return custom_data

ci/praktika/native_jobs.py

@@ -233,6 +233,19 @@ def _check_db(workflow):
     print(f"Start [{job_name}], workflow [{workflow.name}]")
     results = []
     files = []
+    env = _Environment.get()
+    _ = RunConfig(
+        name=workflow.name,
+        digest_jobs={},
+        digest_dockers={},
+        sha=env.SHA,
+        cache_success=[],
+        cache_success_base64=[],
+        cache_artifacts={},
+        cache_jobs={},
+        filtered_jobs={},
+        custom_data={},
+    ).dump()
 
     if workflow.pre_hooks:
         sw_ = Utils.Stopwatch()
@@ -267,31 +280,12 @@ def _check_db(workflow):
         result_ = _check_db(workflow)
         results.append(result_)
 
-    if Path(Settings.CUSTOM_DATA_FILE).is_file():
-        with open(Settings.CUSTOM_DATA_FILE, "r", encoding="utf8") as f:
-            custom_data = json.load(f)
-        print(f"Custom data: [{custom_data}]")
-    else:
-        custom_data = {}
-        print(f"Custom data has not been provided")
-
-    env = _Environment.get()
-    workflow_config = RunConfig(
-        name=workflow.name,
-        digest_jobs={},
-        digest_dockers={},
-        sha=env.SHA,
-        cache_success=[],
-        cache_success_base64=[],
-        cache_artifacts={},
-        cache_jobs={},
-        filtered_jobs={},
-        custom_data=custom_data,
-    ).dump()
-
     if workflow.enable_merge_commit:
         assert False, "NOT implemented"
 
+    # read object from fs after .pre_hooks as some users's custom data may be added there
+    workflow_config = RunConfig.from_fs(workflow.name)
+
     if results[-1].is_ok() and workflow.dockers:
         sw_ = Utils.Stopwatch()
         print("Calculate docker's digests")

ci/praktika/settings.py

@@ -68,7 +68,6 @@ class _Settings:
     WORKFLOW_STATUS_FILE: str = f"{TEMP_DIR}/workflow_status.json"
     WORKFLOW_INPUTS_FILE: str = f"{TEMP_DIR}/workflow_inputs.json"
     ARTIFACT_URLS_FILE: str = f"{TEMP_DIR}/artifact_urls.json"
-    CUSTOM_DATA_FILE: str = "/tmp/custom_data.json"
 
     ######################################
     #        CI Cache settings           #