Merge pull request #701 from Altinity/24.8_leak_check_2

24.8 Scan files for secrets in _upload_file_to_s3

Author: MyroTk

.github/workflows/release_branches.yml

@@ -538,8 +538,8 @@ jobs:
 ##################################### REGRESSION TESTS ######################################
 #############################################################################################
   RegressionTestsRelease:
-    needs: [BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
+    needs: [RunConfig, BuilderDebRelease]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'regression') }}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:
@@ -549,8 +549,8 @@ jobs:
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300
   RegressionTestsAarch64:
-    needs: [BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
+    needs: [RunConfig, BuilderDebAarch64]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'regression') && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'aarch64')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:

tests/ci/s3_helper.py

@@ -6,6 +6,7 @@
 from multiprocessing.dummy import Pool
 from pathlib import Path
 from typing import Any, List, Union
+import os
 
 import boto3  # type: ignore
 import botocore  # type: ignore
@@ -19,6 +20,42 @@
     S3_URL,
 )
 
+sensitive_var_pattern = re.compile(
+    r"\b[A-Z_]*(?<!WRONG_)(SECRET|PASSWORD|ACCESS_KEY|TOKEN)[A-Z_]*\b(?!%)(?!=clickhouse$)(?!=minio)(?!: \*{3}$)(?! '\[HIDDEN\]')"
+)
+sensitive_strings = {
+    var: value for var, value in os.environ.items() if sensitive_var_pattern.match(var)
+}
+
+
+def scan_file_for_sensitive_data(file_content, file_name):
+    """
+    Scan the content of a file for sensitive strings.
+    Raises ValueError if any sensitive values are found.
+    """
+
+    def clean_line(line):
+        for name, value in sensitive_strings.items():
+            line = line.replace(value, f"SECRET[{name}]")
+        return line
+
+    matches = []
+    for line_number, line in enumerate(file_content.splitlines(), start=1):
+        for match in sensitive_var_pattern.finditer(line):
+            matches.append((file_name, line_number, clean_line(line)))
+        for name, value in sensitive_strings.items():
+            if value in line:
+                matches.append((file_name, line_number, clean_line(line)))
+
+    if not matches:
+        return
+
+    logging.error(f"Sensitive values found in {file_name}")
+    for file_name, line_number, match in matches:
+        logging.error(f"{file_name}:{line_number}: {match}")
+
+    raise ValueError(f"Sensitive values found in {file_name}")
+
 
 def _flatten_list(lst):
     result = []
@@ -45,6 +82,14 @@ def __init__(self, client: Any = None, endpoint: str = S3_URL):
     def _upload_file_to_s3(
         self, bucket_name: str, file_path: Path, s3_path: str
     ) -> str:
+        logging.debug("Checking %s for sensitive values", file_path)
+        try:
+            file_content = file_path.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            logging.warning("Failed to read file %s, unknown encoding", file_path)
+        else:
+            scan_file_for_sensitive_data(file_content, file_path.name)
+
         logging.debug(
             "Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path
         )