Merge branch 'feat/diag_purge_file' into 'master'

feat(tools): enable passing the purge file as an argument to diag

Closes IDF-11825

See merge request espressif/esp-idf!36659

Author: dobairoland

tools/idf_py_actions/diag/purge/README.md

@@ -0,0 +1,16 @@
+# Purge format description for idf.py diag
+
+Once diagnostic information is collected, the purge file is utilized to remove
+any sensitive data from the gathered files. By default, the purge file located
+at `tools/idf_py_actions/diag/purge/purge.yml` is used unless it is specified
+with the --purge argument, in which case the default file is not used.
+
+## Overview of Purge Structure
+
+It is a straightforward YAML file that includes a list of regular expressions
+and the corresponding strings that should replace any matches.
+
+    - regex: regular expression to look for
+      repl: substitute string for match
+
+The `regex.sub` function from Python is used internally.

tools/idf_py_actions/diag/purge.yml renamed to tools/idf_py_actions/diag/purge/purge.yml

@@ -116,8 +116,6 @@ def log(level: int, msg: str, prefix: str) -> None:
     else:
         log_prefix = ''
 
-    msg = textwrap.indent(msg, prefix=log_prefix)
-
     if LOG_FILE:
         try:
             log_msg = textwrap.indent(msg, prefix=f'{prefix} ')
@@ -131,6 +129,8 @@ def log(level: int, msg: str, prefix: str) -> None:
     if level > LOG_LEVEL:
         return
 
+    msg = textwrap.indent(msg, prefix=log_prefix)
+
     if not LOG_COLORS or level not in (LOG_FATAL, LOG_ERROR, LOG_WARNING):
         print(msg, file=sys.stderr)
         sys.stderr.flush()
@@ -257,11 +257,8 @@ def diff_dirs(dir1: Path, dir2: Path) -> None:
                 dbg(line.strip())
 
 
-def redact_files(dir1: Path, dir2: Path) -> None:
+def redact_files(dir1: Path, dir2: Path, purge: list) -> None:
     """Show differences in files between two directories."""
-    purge_path = Path(__file__).parent / 'diag' / 'purge.yml'
-    with open(purge_path, 'r') as f:
-        purge = yaml.safe_load(f.read())
 
     regexes: List = []
     for entry in purge:
@@ -488,6 +485,47 @@ def validate_recipe(recipe: Dict) -> None:
                 raise RuntimeError(f'Unknown command "{cmd}" in step "{step_name}"')
 
 
+def validate_purge(purge: Any) -> None:
+    """Validate the loaded purge file. This is done manually to avoid any
+    dependencies and to provide more informative error messages.
+    """
+
+    if type(purge) is not list:
+        raise RuntimeError(f'Purge is not of type "list"')
+
+    regex_keys = ['regex', 'repl']
+
+    for entry in purge:
+        if type(entry) is not dict:
+            raise RuntimeError(f'Purge entry "{entry}" is not of type "dict"')
+
+        if 'regex' in entry:
+            for key in entry:
+                if key not in regex_keys:
+                    raise RuntimeError((f'Unknown purge key "{key}" in "{entry}", '
+                                        f'expecting "{regex_keys}"'))
+
+            regex = entry.get('regex')
+            repl = entry.get('repl')
+
+            # Required arguments
+            if type(regex) is not str:
+                raise RuntimeError(f'Argument "regex" for purge entry "{entry}" is not of type "str"')
+            try:
+                re.compile(regex)
+            except re.error as e:
+                raise RuntimeError((f'Argument "regex" for purge entry "{entry}" is not '
+                                    f'a valid regular expression: {e}'))
+
+            if not repl:
+                raise RuntimeError(f'Purge entry "{entry}" is missing "repl" argument')
+            if type(repl) is not str:
+                raise RuntimeError(f'Argument "repl" for purge entry "{entry}" is not of type "str"')
+
+        else:
+            raise RuntimeError(f'Unknown purge entry "{entry}"')
+
+
 def get_output_path(src: Optional[str],
                     dst: Optional[str],
                     step: Dict,
@@ -934,6 +972,7 @@ def create(action: str,
            check_recipes: bool,
            cmdl_recipes: Tuple,
            cmdl_tags: Tuple,
+           purge_file: str,
            append: bool,
            output: Optional[str]) -> None:
 
@@ -1032,11 +1071,25 @@ def create(action: str,
     except Exception:
         die(f'File "{recipe_file}" is not a valid diagnostic file')
 
+    # Load purge file
+    dbg(f'Purge file: {purge_file}')
+    try:
+        with open(purge_file, 'r') as f:
+            purge = yaml.safe_load(f.read())
+    except Exception:
+        die(f'Cannot load purge file "{purge_file}"')
+
+    # Validate purge file
+    try:
+        validate_purge(purge)
+    except Exception:
+        die(f'File "{purge_file}" is not a valid purge file')
+
     # Cook recipes
     try:
         for recipe_file, recipe in recipes.items():
             desc = recipe.get('description')
-            dbg(f'Processing recipe "{desc} "file "{recipe_file}"')
+            dbg(f'Processing recipe "{desc}" file "{recipe_file}"')
             print(f'{desc}')
             process_recipe(recipe)
     except Exception:
@@ -1050,9 +1103,9 @@ def create(action: str,
         LOG_FILE = None
 
     try:
-        redact_files(TMP_DIR_REPORT_PATH, TMP_DIR_REPORT_REDACTED_PATH)
+        redact_files(TMP_DIR_REPORT_PATH, TMP_DIR_REPORT_REDACTED_PATH, purge)
     except Exception:
-        err(f'The redaction was unsuccessful.')
+        err(f'The redaction was unsuccessful')
 
     try:
         shutil.move(TMP_DIR_REPORT_REDACTED_PATH, output_dir_path)
@@ -1142,6 +1195,15 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
                                  'and the report directory specified with the --zip option with a zip '
                                  'extension is used for the zip file archive.')
                     },
+                    {
+                        'names': ['-p', '--purge', 'purge_file'],
+                        'metavar': 'PATH',
+                        'type': str,
+                        'default': str(Path(__file__).parent / 'diag' / 'purge' / 'purge.yml'),
+                        'help': ('Purge file PATH containing a description of what information '
+                                 'should be redacted from the resulting report. '
+                                 'Default is "tools/idf_py_actions/diag/purge/purge.yml"')
+                    },
                 ],
             },
         },