Automatically create output directories for all LOBSTER tools (#521)

Automatically create output directories for all LOBSTER tools to prevent
crashes when paths don't exist

Issue: SWF-20884

Author: SurajBDeore

CHANGELOG.md

@@ -5,6 +5,11 @@
 
 ### 1.0.3-dev
 
+* All tools now automatically create output directories if they don't exist.
+  Previously, tools would crash with an exception if the specified output
+  directory path did not exist. This enhancement improves usability and
+  prevents unexpected failures when working with nested directory structures.
+
 * `lobster-codebeamer`:
   - Improved error messages with detailed troubleshooting information:
     - Connection timeout errors now include the URL and suggest increasing timeout parameter

lobster/common/io.py

@@ -17,6 +17,7 @@
 # License along with this program. If not, see
 # <https://www.gnu.org/licenses/>.
 
+import os
 import json
 from typing import Dict, Optional, Sequence, TextIO, Type, Union, Iterable
 
@@ -145,3 +146,10 @@ def signal_duplicate_items(
                         f"{items[item.tag.key()].location.to_string()}",
                 fatal=(counter == len(duplicate_items)),
             )
+
+
+def ensure_output_directory(file_path: str) -> None:
+    """Create parent directories for the output file if they don't exist."""
+    output_dir = os.path.dirname(file_path)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)

lobster/common/multi_file_input_tool.py

@@ -21,7 +21,7 @@
 from typing import Iterable, List, Optional, Type, Union
 from lobster.common.errors import Message_Handler
 from lobster.common.items import Requirement, Implementation, Activity
-from lobster.common.io import lobster_write
+from lobster.common.io import lobster_write, ensure_output_directory
 from lobster.common.meta_data_tool_base import MetaDataToolBase
 from lobster.common.multi_file_input_config import Config
 from lobster.common.file_collector import FileCollector
@@ -132,6 +132,7 @@ def _write_output(
             out_file: str,
             items: List[Union[Activity, Implementation, Requirement]],
     ):
+        ensure_output_directory(out_file)
         with open(out_file, "w", encoding="UTF-8") as fd:
             lobster_write(fd, schema, self.name, items)
         print(f"{self.name}: wrote {len(items)} items to {out_file}")

lobster/common/report.py

@@ -16,7 +16,6 @@
 # You should have received a copy of the GNU Affero General Public
 # License along with this program. If not, see
 # <https://www.gnu.org/licenses/>.
-
 import json
 from collections import OrderedDict
 from dataclasses import dataclass
@@ -25,7 +24,7 @@
 from lobster.common.items import Tracing_Status, Requirement, Implementation, Activity
 from lobster.common.parser import load as load_config
 from lobster.common.errors import Message_Handler
-from lobster.common.io import lobster_read
+from lobster.common.io import lobster_read, ensure_output_directory
 from lobster.common.location import File_Reference
 
 
@@ -141,6 +140,7 @@ def write_report(self, filename):
             "matrix"    : [],
         }
 
+        ensure_output_directory(filename)
         with open(filename, "w", encoding="UTF-8") as fd:
             json.dump(report, fd, indent=2)
             fd.write("\n")

lobster/common/tool.py

@@ -29,7 +29,7 @@
 from lobster.common.errors import Message_Handler
 from lobster.common.location import File_Reference
 from lobster.common.items import Requirement, Implementation, Activity
-from lobster.common.io import lobster_write
+from lobster.common.io import lobster_write, ensure_output_directory
 from lobster.common.meta_data_tool_base import MetaDataToolBase
 
 
@@ -262,6 +262,7 @@ def write_output(
                    for item in items)
 
         if options.out:
+            ensure_output_directory(options.out)
             with open(options.out, "w", encoding="UTF-8") as fd:
                 lobster_write(fd, self.schema, self.name, items)
             print(f"{self.name}: wrote {len(items)} items to {options.out}")

lobster/tools/codebeamer/codebeamer.py

@@ -54,7 +54,7 @@
 from lobster.common.items import Tracing_Tag, Requirement, Implementation, Activity
 from lobster.common.location import Codebeamer_Reference
 from lobster.common.errors import Message_Handler, LOBSTER_Error
-from lobster.common.io import lobster_read, lobster_write
+from lobster.common.io import lobster_read, lobster_write, ensure_output_directory
 from lobster.common.meta_data_tool_base import MetaDataToolBase
 from lobster.tools.codebeamer.bearer_auth import BearerAuth
 from lobster.tools.codebeamer.config import AuthenticationConfig, Config
@@ -657,6 +657,7 @@ def _execute(self, options: argparse.Namespace) -> None:
 
 def _get_out_stream(config_out: Optional[str]) -> TextIO:
     if config_out:
+        ensure_output_directory(config_out)
         return open(config_out, "w", encoding="UTF-8")
     return sys.stdout
 
@@ -672,6 +673,7 @@ def lobster_codebeamer(config: Config, out_file: str) -> None:
     """
     # This is an API function.
     items = get_query(config, config.import_query)
+    ensure_output_directory(out_file)
     with open(out_file, "w", encoding="UTF-8") as fd:
         _cb_items_to_lobster(items, config, fd)
 

lobster/tools/core/html_report/html_report.py

@@ -30,6 +30,7 @@
 from lobster.common.version import LOBSTER_VERSION
 from lobster.htmldoc import htmldoc
 from lobster.common.report import Report
+from lobster.common.io import ensure_output_directory
 from lobster.common.location import (Void_Reference,
                                      File_Reference,
                                      Github_Reference,
@@ -286,6 +287,14 @@ def generate_custom_data(report) -> str:
     return "".join(content)
 
 
+def write_html_to_file(html_content: str, output_path: str) -> None:
+    """Write HTML content to file, creating parent directories if needed."""
+    ensure_output_directory(output_path)
+    with open(output_path, "w", encoding="UTF-8") as fd:
+        fd.write(html_content)
+        fd.write("\n")
+
+
 def write_html(report, dot, high_contrast, render_md) -> str:
     assert isinstance(report, Report)
 
@@ -613,9 +622,7 @@ def _run_impl(self, options: argparse.Namespace) -> int:
             high_contrast = options.high_contrast,
             render_md = options.render_md,
         )
-        with open(options.out, "w", encoding="UTF-8") as fd:
-            fd.write(html_content)
-            fd.write("\n")
+        write_html_to_file(html_content, options.out)
         print(f"LOBSTER HTML report written to {options.out}")
 
         return 0
@@ -646,9 +653,7 @@ def lobster_html_report(
         high_contrast=high_contrast,
         render_md=render_md,
     )
-    with open(output_html_path, "w", encoding="UTF-8") as fd:
-        fd.write(html_content)
-        fd.write("\n")
+    write_html_to_file(html_content, output_html_path)
 
 
 def main(args: Optional[Sequence[str]] = None) -> int:

lobster/tools/cpptest/cpptest.py

@@ -19,7 +19,7 @@
 
 import sys
 from argparse import Namespace
-import os.path
+import os
 from copy import copy
 from dataclasses import dataclass, field
 from typing import List, Optional, Sequence, Union
@@ -29,7 +29,7 @@
 from lobster.common.exceptions import LOBSTER_Exception
 from lobster.common.items import Tracing_Tag, Activity
 from lobster.common.location import File_Reference
-from lobster.common.io import lobster_write
+from lobster.common.io import lobster_write, ensure_output_directory
 from lobster.common.file_tag_generator import FileTagGenerator
 from lobster.tools.cpptest.constants import Constants
 from lobster.tools.cpptest.requirements_parser import \
@@ -303,16 +303,16 @@ def write_lobster_items_output_dict(lobster_items_output_dict: dict):
         lobster_items_dict.update(orphan_test_items)
         item_count = len(lobster_items_dict)
 
-        if output_file_name:
-            with open(output_file_name, "w", encoding="UTF-8") as output_file:
-                lobster_write(
-                    output_file,
-                    Activity,
-                    lobster_generator,
-                    lobster_items_dict.values()
-                )
-            print(f'Written {item_count} lobster items to '
-                  f'"{output_file_name}".')
+        ensure_output_directory(output_file_name)
+        with open(output_file_name, "w", encoding="UTF-8") as output_file:
+            lobster_write(
+                output_file,
+                Activity,
+                lobster_generator,
+                lobster_items_dict.values()
+            )
+        print(f'Written {item_count} lobster items to '
+                f'"{output_file_name}".')
 
 
 def run_lobster_cpptest(config: Config):

lobster/tools/gtest/gtest.py

@@ -19,13 +19,13 @@
 
 from argparse import Namespace
 import sys
-import os.path
+import os
 from typing import Optional, Sequence
 import xml.etree.ElementTree as ET
 
 from lobster.common.items import Tracing_Tag, Activity
 from lobster.common.location import Void_Reference, File_Reference
-from lobster.common.io import lobster_write
+from lobster.common.io import lobster_write, ensure_output_directory
 from lobster.common.meta_data_tool_base import MetaDataToolBase
 
 
@@ -141,6 +141,7 @@ def _run_impl(self, options: Namespace) -> int:
                     items.append(item)
 
         if options.out:
+            ensure_output_directory(options.out)
             with open(options.out, "w", encoding="UTF-8") as fd:
                 lobster_write(fd, Activity, "lobster_gtest", items)
             print(f"Written output for {len(items)} items to {options.out}")

lobster/tools/python/python.py

@@ -30,7 +30,7 @@
 
 from lobster.common.items import Tracing_Tag, Implementation, Activity
 from lobster.common.location import File_Reference
-from lobster.common.io import lobster_write
+from lobster.common.io import lobster_write, ensure_output_directory
 from lobster.common.meta_data_tool_base import MetaDataToolBase
 
 LOBSTER_TRACE_PREFIX = "# lobster-trace: "
@@ -534,6 +534,7 @@ def _run_impl(self, options: Namespace) -> int:
             schema = Implementation
 
         if options.out:
+            ensure_output_directory(options.out)
             with open(options.out, "w", encoding="UTF-8") as fd:
                 lobster_write(fd, schema, "lobster_python", items)
             print(f"Written output for {len(items)} items to {options.out}")