Merge pull request #285 from nsidc/issue-272

Issue-272: Upgrade log file management

Author: eigenbeam

CHANGELOG.md

@@ -1,7 +1,9 @@
 ## UNRELEASED
 
 * Issue-255: Remove CSV reader capability (generic & SNOWEX-specific readers)
-* Fix configuration issues with some integration tests
+* Issue-282: Fix configuration issues with some integration tests
+* Issue-272: Add a user-provided logfile directory location. Add the configuration
+  file's basename and the metgenc start datetime to the logfile's name.
 
 ## v1.11.0 (2025-09-02)
 

README.md

@@ -326,12 +326,13 @@ Example running **init**
 
     $ metgenc init -c ./init/<name of config file to create or modify>.ini
 
-##### INI RULES:  
+##### INI RULES:
 * The .ini file's `checksum_type = SHA256` should never be edited
 * The `kinesis_stream_name` and `staging_bucket_name` should never be edited
 * `auth_id` and `version` must accurately reflect the collection's authID and versionID
+* `log_dir` specifies the directory where log files will be written. Log files are named `metgenc-{config-name}-{timestamp}.log` where config-name is the base name of the .ini file and timestamp is in YYYYMMDD-HHMM format. The log directory must exist and be writable. If not specified, defaults to `/share/logs/metgenc`
 * provider is a free text attribute where, for now, the version of metgenc being run should be documented
-  * running `metgenc --version` will return the current version    
+  * running `metgenc --version` will return the current version
 
 #### Required and Optional Configuration Elements
 Some attribute values may be read from the .ini file if the values
@@ -381,7 +382,7 @@ Note column:
    with multiple associated browse files work fine with MetGenC! The default is `_brws`, change it to reflect
    the browse file names of the data delivered. This element is prompted for when running `metgenc init`.
 3. The file name pattern to be used for multi-file granules to define a file name pattern to appropriately
-   group files together as a granule using the elements common amongst their names.  
+   group files together as a granule using the elements common amongst their names.
    - This must result in a globally unique: product/name (in CNM), and Identifier (as the IdentifierType: ProducerGranuleId in UMM-G)
      generated for each granule. This init element value must be added manually as it's **not** included in the `metgenc init` prompts.
 5. The file name pattern identifying a single file for metgenc to reference as the primary
@@ -517,7 +518,7 @@ When a granule has an associated `.spatial` file containing geodetic point data
 | Spatial       | spatial_polygon_cartesian_tolerance | float | 0.0001  | Minimum distance between polygon points in degrees (0.00001-0.01) |
 
 ##### Example Spatial Polygon Generation Configuration
-Example showing content added to an .ini file, having edited the CMR default vertex tolerance 
+Example showing content added to an .ini file, having edited the CMR default vertex tolerance
 (distance between two vertices) to decrease the precision of the GPoly coordinate pairs listed
 in the UMMG json files MetGenC generates:
 ```ini
@@ -590,6 +591,7 @@ Using configuration:
   + write_cnm_file: True
   + overwrite_ummg: True
   + checksum_type: SHA256
+  + log_dir: /share/logs/metgenc
   + number: 1000000
   + dry_run: False
   + premet_dir: None

fixtures/test.ini

@@ -26,3 +26,4 @@ overwrite_ummg = True
 
 [Settings]
 checksum_type = SHA256
+log_dir = /tmp

src/nsidc/metgen/cli.py

@@ -56,7 +56,7 @@ def info(config_filename):
     configuration = config.configuration(
         config.config_parser_factory(config_filename), {}
     )
-    metgen.init_logging()
+    metgen.init_logging(configuration)
     configuration.show()
 
 
@@ -82,7 +82,7 @@ def validate(config_filename, content_type):
     configuration = config.configuration(
         config.config_parser_factory(config_filename), {}
     )
-    metgen.init_logging()
+    metgen.init_logging(configuration)
     metgen.validate(configuration, content_type)
 
 
@@ -146,7 +146,7 @@ def process(config_filename, dry_run, env, number, write_cnm, overwrite):
         configuration = config.configuration(
             config.config_parser_factory(config_filename), overrides, env
         )
-        metgen.init_logging()
+        metgen.init_logging(configuration)
         configuration.show()
         config.validate(configuration)
         config.validate_spatial_source(configuration)

src/nsidc/metgen/config.py

@@ -50,6 +50,8 @@ class Config:
     spatial_polygon_target_coverage: Optional[float] = None
     spatial_polygon_max_vertices: Optional[int] = None
     spatial_polygon_cartesian_tolerance: Optional[float] = None
+    log_dir: Optional[str] = None
+    name: Optional[str] = None
 
     def show(self):
         # TODO: add section headings in the right spot
@@ -87,6 +89,10 @@ def config_parser_factory(configuration_file):
     # If the config parser gets no value (empty string), interpret it as False
     cfg_parser.BOOLEAN_STATES |= [("", False)]
     cfg_parser.read(configuration_file)
+
+    # Store the config file basename for use in logging
+    cfg_parser._config_name = os.path.splitext(os.path.basename(configuration_file))[0]
+
     return cfg_parser
 
 
@@ -148,6 +154,7 @@ def configuration(
         "spatial_polygon_target_coverage": constants.DEFAULT_SPATIAL_POLYGON_TARGET_COVERAGE,
         "spatial_polygon_max_vertices": constants.DEFAULT_SPATIAL_POLYGON_MAX_VERTICES,
         "spatial_polygon_cartesian_tolerance": constants.DEFAULT_SPATIAL_POLYGON_CARTESIAN_TOLERANCE,
+        "log_dir": constants.DEFAULT_LOG_DIR,
     }
     try:
         return Config(
@@ -318,6 +325,15 @@ def configuration(
                 config_parser,
                 overrides,
             ),
+            _get_configuration_value(
+                environment,
+                "Settings",
+                "log_dir",
+                str,
+                config_parser,
+                overrides,
+            ),
+            getattr(config_parser, "_config_name", "metgenc"),
         )
     except Exception as e:
         raise Exception("Unable to read the configuration file", e)
@@ -390,6 +406,13 @@ def validate(configuration):
             else True,
             "The spatial polygon cartesian tolerance must be between 0.00001 and 0.01 degrees.",
         ],
+        [
+            "log_dir",
+            lambda log_dir: os.path.exists(log_dir) and os.access(log_dir, os.W_OK)
+            if log_dir
+            else True,
+            "The log directory does not exist or is not writable.",
+        ],
     ]
     errors = [
         msg for name, fn, msg in validations if not fn(getattr(configuration, name))

src/nsidc/metgen/constants.py

@@ -15,6 +15,7 @@
 DEFAULT_BROWSE_REGEX = "_brws"
 DEFAULT_COLLECTION_GEOMETRY_OVERRIDE = False
 DEFAULT_COLLECTION_TEMPORAL_OVERRIDE = False
+DEFAULT_LOG_DIR = "/share/logs/metgenc"
 
 # Spatial polygon defaults
 DEFAULT_SPATIAL_POLYGON_ENABLED = True

src/nsidc/metgen/metgen.py

@@ -52,7 +52,7 @@
 # -------------------------------------------------------------------
 
 
-def init_logging():
+def init_logging(configuration=None):
     """
     Initialize the logger for metgenc.
     """
@@ -64,7 +64,21 @@ def init_logging():
     console_handler.setFormatter(logging.Formatter(CONSOLE_FORMAT))
     logger.addHandler(console_handler)
 
-    logfile_handler = logging.FileHandler(constants.ROOT_LOGGER + ".log", "a")
+    # Generate log filename
+    log_dir = constants.DEFAULT_LOG_DIR
+    if configuration and configuration.log_dir:
+        log_dir = configuration.log_dir
+
+    # Generate filename: metgenc-{name}-{datetime}.log
+    config_basename = "metgenc"
+    if configuration and configuration.name:
+        config_basename = configuration.name
+
+    timestamp = dt.datetime.now().strftime("%Y%m%d-%H%M")
+    log_filename = f"metgenc-{config_basename}-{timestamp}.log"
+    log_path = os.path.join(log_dir, log_filename)
+
+    logfile_handler = logging.FileHandler(log_path, "a")
     logfile_handler.setLevel(logging.DEBUG)
     logfile_handler.setFormatter(logging.Formatter(LOGFILE_FORMAT))
     logger.addHandler(logfile_handler)
@@ -225,6 +239,11 @@ def init_config(configuration_file):
         "checksum_type",
         Prompt.ask("Checksum type", default=constants.DEFAULT_CHECKSUM_TYPE),
     )
+    cfg_parser.set(
+        constants.SETTINGS_SECTION_NAME,
+        "log_dir",
+        Prompt.ask("Log directory", default=constants.DEFAULT_LOG_DIR),
+    )
 
     print()
     print(f"Saving new configuration: {configuration_file}")

tests/integration/configs/IPFLT1B_DUCk.ini

@@ -22,3 +22,4 @@ overwrite_ummg = True
 
 [Settings]
 checksum_type = SHA256
+log_dir = ./output/logs

tests/integration/configs/IRTIT3DUCk.ini

@@ -19,3 +19,4 @@ overwrite_ummg = False
 
 [Settings]
 checksum_type = SHA256
+log_dir = ./output/logs

tests/integration/configs/IRWIS2DUCk.ini

@@ -21,3 +21,4 @@ overwrite_ummg = false
 
 [Settings]
 checksum_type = output
+log_dir = ./output/logs