feat(tools): enable passing the purge file as an argument to diag

At present, the diag tool uses its default purge file. However, users
may find it beneficial to specify and reuse their own purge file. A new
command line option, --purge, has been introduced to allow users to
provide their own purge file to diag. When this option is used, the
default purge file is ignored.

Signed-off-by: Frantisek Hrbata <[email protected]>

Author: fhrbata

tools/idf_py_actions/diag/purge/README.md

@@ -0,0 +1,16 @@
+# Purge format description for idf.py diag
+
+Once diagnostic information is collected, the purge file is utilized to remove
+any sensitive data from the gathered files. By default, the purge file located
+at `tools/idf_py_actions/diag/purge/purge.yml` is used unless it is specified
+with the --purge argument, in which case the default file is not used.
+
+## Overview of Purge Structure
+
+It is a straightforward YAML file that includes a list of regular expressions
+and the corresponding strings that should replace any matches.
+
+    - regex: regular expression to look for
+      repl: substitute string for match
+
+The `regex.sub` function from Python is used internally.

tools/idf_py_actions/diag/purge.yml renamed to tools/idf_py_actions/diag/purge/purge.yml

@@ -116,8 +116,6 @@ def log(level: int, msg: str, prefix: str) -> None:
     else:
         log_prefix = ''
 
-    msg = textwrap.indent(msg, prefix=log_prefix)
-
     if LOG_FILE:
         try:
             log_msg = textwrap.indent(msg, prefix=f'{prefix} ')
@@ -131,6 +129,8 @@ def log(level: int, msg: str, prefix: str) -> None:
     if level > LOG_LEVEL:
         return
 
+    msg = textwrap.indent(msg, prefix=log_prefix)
+
     if not LOG_COLORS or level not in (LOG_FATAL, LOG_ERROR, LOG_WARNING):
         print(msg, file=sys.stderr)
         sys.stderr.flush()
@@ -257,11 +257,8 @@ def diff_dirs(dir1: Path, dir2: Path) -> None:
                 dbg(line.strip())
 
 
-def redact_files(dir1: Path, dir2: Path) -> None:
+def redact_files(dir1: Path, dir2: Path, purge: list) -> None:
     """Show differences in files between two directories."""
-    purge_path = Path(__file__).parent / 'diag' / 'purge.yml'
-    with open(purge_path, 'r') as f:
-        purge = yaml.safe_load(f.read())
 
     regexes: List = []
     for entry in purge:
@@ -488,6 +485,47 @@ def validate_recipe(recipe: Dict) -> None:
                 raise RuntimeError(f'Unknown command "{cmd}" in step "{step_name}"')
 
 
+def validate_purge(purge: Any) -> None:
+    """Validate the loaded purge file. This is done manually to avoid any
+    dependencies and to provide more informative error messages.
+    """
+
+    if type(purge) is not list:
+        raise RuntimeError(f'Purge is not of type "list"')
+
+    regex_keys = ['regex', 'repl']
+
+    for entry in purge:
+        if type(entry) is not dict:
+            raise RuntimeError(f'Purge entry "{entry}" is not of type "dict"')
+
+        if 'regex' in entry:
+            for key in entry:
+                if key not in regex_keys:
+                    raise RuntimeError((f'Unknown purge key "{key}" in "{entry}", '
+                                        f'expecting "{regex_keys}"'))
+
+            regex = entry.get('regex')
+            repl = entry.get('repl')
+
+            # Required arguments
+            if type(regex) is not str:
+                raise RuntimeError(f'Argument "regex" for purge entry "{entry}" is not of type "str"')
+            try:
+                re.compile(regex)
+            except re.error as e:
+                raise RuntimeError((f'Argument "regex" for purge entry "{entry}" is not '
+                                    f'a valid regular expression: {e}'))
+
+            if not repl:
+                raise RuntimeError(f'Purge entry "{entry}" is missing "repl" argument')
+            if type(repl) is not str:
+                raise RuntimeError(f'Argument "repl" for purge entry "{entry}" is not of type "str"')
+
+        else:
+            raise RuntimeError(f'Unknown purge entry "{entry}"')
+
+
 def get_output_path(src: Optional[str],
                     dst: Optional[str],
                     step: Dict,
@@ -934,6 +972,7 @@ def create(action: str,
            check_recipes: bool,
            cmdl_recipes: Tuple,
            cmdl_tags: Tuple,
+           purge_file: str,
            append: bool,
            output: Optional[str]) -> None:
 
@@ -1032,11 +1071,25 @@ def create(action: str,
     except Exception:
         die(f'File "{recipe_file}" is not a valid diagnostic file')
 
+    # Load purge file
+    dbg(f'Purge file: {purge_file}')
+    try:
+        with open(purge_file, 'r') as f:
+            purge = yaml.safe_load(f.read())
+    except Exception:
+        die(f'Cannot load purge file "{purge_file}"')
+
+    # Validate purge file
+    try:
+        validate_purge(purge)
+    except Exception:
+        die(f'File "{purge_file}" is not a valid purge file')
+
     # Cook recipes
     try:
         for recipe_file, recipe in recipes.items():
             desc = recipe.get('description')
-            dbg(f'Processing recipe "{desc} "file "{recipe_file}"')
+            dbg(f'Processing recipe "{desc}" file "{recipe_file}"')
             print(f'{desc}')
             process_recipe(recipe)
     except Exception:
@@ -1050,9 +1103,9 @@ def create(action: str,
         LOG_FILE = None
 
     try:
-        redact_files(TMP_DIR_REPORT_PATH, TMP_DIR_REPORT_REDACTED_PATH)
+        redact_files(TMP_DIR_REPORT_PATH, TMP_DIR_REPORT_REDACTED_PATH, purge)
     except Exception:
-        err(f'The redaction was unsuccessful.')
+        err(f'The redaction was unsuccessful')
 
     try:
         shutil.move(TMP_DIR_REPORT_REDACTED_PATH, output_dir_path)
@@ -1142,6 +1195,15 @@ def action_extensions(base_actions: Dict, project_path: str) -> Any:
                                  'and the report directory specified with the --zip option with a zip '
                                  'extension is used for the zip file archive.')
                     },
+                    {
+                        'names': ['-p', '--purge', 'purge_file'],
+                        'metavar': 'PATH',
+                        'type': str,
+                        'default': str(Path(__file__).parent / 'diag' / 'purge' / 'purge.yml'),
+                        'help': ('Purge file PATH containing a description of what information '
+                                 'should be redacted from the resulting report. '
+                                 'Default is "tools/idf_py_actions/diag/purge/purge.yml"')
+                    },
                 ],
             },
         },