correct key for spp create date

Author: AnneONS

src/utils/s3_mods.py

@@ -462,7 +462,8 @@ def rd_list_manifest_files(prefix: str) -> dict:
         boto3_client.exceptions.ClientError: If there is an error accessing S3.
 
     Returns:
-        dict: A dictionary with manifest file keys and their last modified dates.
+        dict: A dictionary with manifest file keys and their last modified dates, sorted
+              by last modified date in descending order.
     """
     try:
         manifest_files = {}
@@ -473,7 +474,13 @@ def rd_list_manifest_files(prefix: str) -> dict:
                 last_modified = obj["LastModified"]  # This is a datetime object
                 if key.endswith(".mani"):
                     manifest_files[key] = last_modified
-        return manifest_files
+
+        # Use lambda to sort dictionary items by their values (last_modified_date)
+        sorted_manifest_files = dict(
+            sorted(manifest_files.items(), key=lambda item: item[1], reverse=True)
+        )
+        return sorted_manifest_files
+
     except s3_client.exceptions.ClientError as e:
         s3_logger.error(f"Error listing manifest files with prefix {prefix}: {e}")
         raise e

src/utils/snapshot_metadata_utils.md

@@ -0,0 +1,46 @@
+Main Goal:
+- Find the correct SPP snapshot file from S3 manifest metadata, based on date, version,
+  and filename pattern.
+
+1. Get all manifest files from S3 with a given prefix (filepath).
+   - Use S3 client to list all files ending with ".mani" under the prefix.
+   - Store as {filename: last_modified_date}, sorted by date descending.
+
+2. Filter manifest files by substring
+   - Use substring search to obtain files relating to the correct survey and survey year
+
+3. If required, get the most recent modified date and check files modified this date.
+   - Read metadata for all files modified this date
+   - Check whether the spp created date is the same  for each file
+   - Check whether other metadata is valid for each file
+   - Create a list of candidate files with their metadata
+
+4. If a specified date was given, check all files modified this date.
+   - Read metadata for all files modified this date
+   - Check whether the spp created date is the same date for each file
+   - Check whether other metadata is valid for each file
+   - Create a list of candidate files with their metadata
+
+5. If the created date for a file is earlier than it's modified date (it can't be later)
+   - Save the created date in a "most recent created" variable
+   - Continue the loop through files to the next most recent modified file and onwards
+   - If a created date is found that is more recent than that in the "most recent created" varible, and the modified date is the same, that is the candidate.
+   - When the next most recent modified file is modified after the "most recent
+     created" date, the "most recent created" date is the candidate.
+   - All files created on a "most recent created" date are candidates.
+
+6. From candidate files:
+   - If multiple, select the most recent by last_modified_date.
+   - If multiple with same filename, select the one with the highest version.
+
+7. Return the chosen snapshot filename, version, and created date.
+   - Conditionally return file info for the most recent date
+   - Conditionally return file info for specified date
+
+Helper Functions:
+- filter_manifest_files(files_dict, target_date, wanted_str)
+- get_spp_file_info_from_manifest(mani_filename, mani_last_modified_date)
+- check_metadata(metadata, target_date)
+- check_files(mani_files_dict, target_date)
+- get_most_recent_file(file_list)
+- get_lastest_version_file(file_list)

src/utils/snapshot_metadata_utils.py

@@ -18,7 +18,7 @@
 class SPPMetadata:
     # include default values to check against
     mani_filename: str
-    mani_last_modified_date: datetime
+    mani_last_modified_date: str
     spp_filename: str
     spp_created_date: str
     version: int
@@ -28,7 +28,7 @@ class SPPMetadata:
 
 def filter_manifest_files(
     files_dict: dict[str, datetime], target_date: str = "", wanted_str: str = ""
-) -> dict[str, datetime]:
+) -> dict[str, str]:
     """Filter manifest files by target date and/ or filename substring.
 
     Args:
@@ -41,29 +41,29 @@ def filter_manifest_files(
     """
     if wanted_str:
         filtered_files = {
-            filename: last_mod_date
+            filename: last_mod_date.strftime("%Y-%m-%d")
             for filename, last_mod_date in files_dict.items()
             if wanted_str in filename
         }
     if target_date:
         filtered_files = {
-            filename: last_mod_date
+            filename: last_mod_date.strftime("%Y-%m-%d")
             for filename, last_mod_date in files_dict.items()
             if last_mod_date.strftime("%Y-%m-%d") == target_date
         }
     return filtered_files
 
 
 def get_spp_file_info_from_manifest(
-    mani_filename: str, mani_last_modified_date: datetime
+    mani_filename: str, mani_last_modified_date: str
 ) -> SPPMetadata:
     """Get SPP file information from a manifest file as SPPMetadata.
 
     Required fields are stored in a SPPMetadata data class for clarity.
 
     Args:
         mani_filename (str): The manifest filename.
-        mani_last_modified_date (datetime): The last modified date of the manifest file.
+        mani_last_modified_date (str): The last modified date of the manifest file.
 
     Returns:
         SPPMetadata: The SPPMetadata object with information from the manifest.
@@ -74,7 +74,7 @@ def get_spp_file_info_from_manifest(
         mani_filename=mani_filename,
         mani_last_modified_date=mani_last_modified_date,
         spp_filename=manif_file_dict["files"][0]["name"],
-        spp_created_date=str(manif_file_dict["tdzComplete"])[:10],
+        spp_created_date=str(manif_file_dict["files"][0]["scanFileUploadTime"])[:10],
         version=manif_file_dict.get("version", 1),
         description=manif_file_dict.get("description", ""),
         iterationL1=manif_file_dict.get("iterationL1", ""),
@@ -98,10 +98,12 @@ def check_metadata(metadata: SPPMetadata, target_date: str) -> dict:
     exp_version = metadata.version
     exp_mani_filename = metadata.mani_filename
     exp_mani_last_modified_date = metadata.mani_last_modified_date
-    # remove the path and .mani extension from the filename checking
+    # remove the path and .mani extension from the manifest filename for the spp name
     exp_spp_filename = os.path.basename(exp_mani_filename).replace(".mani", "")
+    # if no target date is given, check whether the created date is the same as
+    # the manifest last modified date
     if target_date == "":
-        target_date = metadata.spp_created_date
+        target_date = metadata.mani_last_modified_date
 
     # Create an expected SPPMetadata object for checks.
     expected_metadata = SPPMetadata(
@@ -114,13 +116,13 @@ def check_metadata(metadata: SPPMetadata, target_date: str) -> dict:
     )
 
     error_dict = {
-        k: v
+        k: {"expected": v, "found": getattr(metadata, k, None)}
         for k, v in expected_metadata.__dict__.items()
         if getattr(metadata, k, None) != v
     }
     if error_dict:
         msg = (
-            f"Metadata check failed for date {target_date}. "
+            f"Metadata check failed for target date {target_date}. "
             f"Mismatched entries: {error_dict}"
         )
         MetadataLogger.error(msg)
@@ -131,6 +133,14 @@ def check_metadata(metadata: SPPMetadata, target_date: str) -> dict:
 def check_files(mani_files_dict: dict, target_date: str = "") -> list[SPPMetadata]:
     """Check filtered manifest file dict for metadata validity and (optionally) date.
 
+    If a target date is provided, the files dict only contain files from that date.
+
+    If no target date is provided, then the function will check the newest files first
+    until a valid file is found (the dictionary has been sorted by last modified date).
+
+    However, if the `spp_created_date` is earlier than the modified date, the function
+    will continue checking further files to find the latest creation date.
+
     Args:
         mani_files_dict (dict): {manifest_filename: last_modified_date}
         target_date (str): Target date in 'YYYY-MM-DD' format. If empty, no date check.
@@ -142,6 +152,7 @@ def check_files(mani_files_dict: dict, target_date: str = "") -> list[SPPMetadat
     for mani_filename, mani_mod_date in mani_files_dict.items():
         spp_metadata = get_spp_file_info_from_manifest(mani_filename, mani_mod_date)
         error_dict = check_metadata(spp_metadata, target_date)
+        # TODO: if the only error is created date < modified date, continue checking-
         if not error_dict:
             new_candidate = SPPMetadata(
                 mani_filename=mani_filename,
@@ -151,6 +162,7 @@ def check_files(mani_files_dict: dict, target_date: str = "") -> list[SPPMetadat
                 version=spp_metadata.version,
             )
             candidate_file_list.append(new_candidate)
+
     return candidate_file_list
 
 
@@ -200,8 +212,11 @@ def get_snapshot_name(
 ) -> tuple[str, int, str]:
     """Return the name, version and created date of an spp snapshot.
 
-    Optionally check first against a target date. The latest version is returned
-    if multiple files match the criteria.
+    s3_client code exists to list manifest filenames with their last modified dates.
+    However, it is necessary to read the manifest files to get the snapshot created date
+    and version number.
+
+    This code will start by filtering for the required
 
     Args:
         survey_year (str): The survey year the snapshot belongs to.
@@ -212,7 +227,7 @@ def get_snapshot_name(
     """
     # get a dictionary of all manifest files with the given prefix
     files_dict = rd_list_manifest_files(prefix)
-    # filter the manifest files for the given date and wanted string
+    # filter the manifest files for the given date (if given) and wanted string
     wanted_str = f"snapshot-{survey_year}12-002-"
     filtered_mani_file_list = filter_manifest_files(files_dict, spp_date, wanted_str)
     # check filtered files for metadata matching the target date and metadata validity