TIND, PyPDF: Make metadata consistent and coherent

Due to the way Langchain uses LanceDB, the Arrow schema cannot change
between documents.  By default, the PyPDF document loader from Langchain
ingests all metadata from the PDF (from both properties and XMP) and
saves them as keys in the ``metadata`` dictionary.  Unfortunately, the
PDFs we are processing do not have uniform usage of metadata keys between
the different PDF files.  Some have Company set to ``UC Berkeley``, some
don't; some have author information, some don't.

What's more, the way we parse TIND records into further metadata meant
that for a given key X, some records would have ``null`` values, some had
a single ``str``, and some had a ``list`` of ``str``.

This commit ensures that only the PDF metadata Langchain uses to process
documents is included, and that all TIND properties are lists of strings.
Empty values are now a list containing a single empty string.

The alternative would be to define our own LanceDB connection, define our
own schema ahead of time, and hope it never needs to change.  It would
also make Langchain integration more difficult.

Additionally, fix up unit tests to handle metadata being lists now.

Closes: AP-462

Author: awilfox

tests/tind/test_format_validate_pymarc.py

@@ -60,7 +60,7 @@ def test_multiple_subfields_returns_array_of_values(self) -> None:
     def test_metadata_simple(self) -> None:
         """Test parsing of a simple record into metadata."""
         result = format_validate_pymarc.pymarc_to_metadata(self.pymarc_records[0])
-        self.assertEqual(result['tind_id'], '19217')
+        self.assertEqual(result['tind_id'], ['19217'])
         self.assertListEqual(result['subject'], ['Ranches', 'Persea americana', 'Agriculture'])
 
     def test_metadata_multi_subject(self) -> None:
@@ -72,7 +72,7 @@ def test_metadata_multi_subject(self) -> None:
         })
 
         result = format_validate_pymarc.pymarc_to_metadata(self.pymarc_records[0])
-        self.assertEqual(result['tind_id'], '19217')
+        self.assertEqual(result['tind_id'], ['19217'])
         self.assertListEqual(result['subject'], ['Ranches', 'Persea americana', 'Agriculture',
                                                  'Testing'])
 
@@ -87,7 +87,7 @@ def test_metadata_multi_lists(self) -> None:
         })
 
         result = format_validate_pymarc.pymarc_to_metadata(self.pymarc_records[0])
-        self.assertEqual(result['tind_id'], '19217')
+        self.assertEqual(result['tind_id'], ['19217'])
         self.assertListEqual(result['subject'], ['Ranches', 'Persea americana', 'Agriculture',
                                                  'Element 1', 'Element 2'])
 
@@ -102,7 +102,7 @@ def test_metadata_multi_none(self) -> None:
         })
 
         result = format_validate_pymarc.pymarc_to_metadata(self.pymarc_records[0])
-        self.assertEqual(result['tind_id'], '19217')
+        self.assertEqual(result['tind_id'], ['19217'])
         self.assertListEqual(result['subject'], ['Ranches', 'Persea americana', 'Agriculture'])
 
         format_validate_pymarc.parse_pymarc = parse_pymarc

willa/lcvendor/pypdf.py

@@ -122,10 +122,11 @@ def _purge_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
             # Normalize key with others PDF parser
             new_metadata[map_key[k]] = v
             new_metadata[k] = v
-        elif isinstance(v, str):
-            new_metadata[k] = v.strip()
-        elif isinstance(v, int):
-            new_metadata[k] = v
+        elif k in _STD_METADATA_KEYS:
+            if isinstance(v, str):
+                new_metadata[k] = v.strip()
+            elif isinstance(v, int):
+                new_metadata[k] = v
     return new_metadata
 
 

willa/tind/format_tind_context.py

@@ -30,7 +30,7 @@ def get_tind_context(docs: list) -> str:
     tind_data = ''
     tind_ids = {}
     for doc in docs:
-        tind_id = doc.metadata['tind_metadata']['tind_id']
+        tind_id = doc.metadata['tind_metadata']['tind_id'][0]
         if tind_id in tind_ids:
             continue
 
@@ -58,6 +58,6 @@ def process_fields(tind_rec: dict) -> str:
         elif tind_rec[field] is not None:
             formatted_str += f"{DISPLAY_MAPPINGS[field]} {tind_rec[field]}\n\n"
 
-    formatted_str += f"Catalogue Link: {get_tind_url(tind_rec['tind_id'])}"
+    formatted_str += f"Catalogue Link: {get_tind_url(tind_rec['tind_id'][0])}"
 
     return formatted_str

willa/tind/format_validate_pymarc.py

@@ -138,29 +138,27 @@ def pymarc_to_metadata(record: Record) -> dict:
     """
     marc_values = parse_pymarc(record)
 
-    metadata: dict[str, str | list | None] = {}
+    metadata: dict[str, list] = {}
     for key, value in marc_values.items():
+        if value is None:
+            continue  # Skip adding blanks to content.
+
         meta_key = KEY_MAPPINGS[key]
         if meta_key in metadata:
-            if value is None:
-                continue  # Skip adding blanks to existing content.
-
-            if isinstance(metadata[meta_key], str):
-                metadata[meta_key] = [metadata[meta_key]]  # Turn our str into a one-element list.
-            elif metadata[meta_key] is None:
-                metadata[meta_key] = []
-
             if isinstance(value, list):
                 # Add our list to the list.
                 metadata[meta_key].extend(value)  # type: ignore[union-attr]
             else:
                 # Add our value to the list.
                 metadata[meta_key].append(value)  # type: ignore[union-attr]
         else:
-            metadata[meta_key] = value
+            if isinstance(value, list):
+                metadata[meta_key] = value
+            else:
+                metadata[meta_key] = [value]
 
     for meta_key in set(KEY_MAPPINGS.values()):
         if meta_key not in metadata:
-            metadata[meta_key] = None
+            metadata[meta_key] = ['']
 
     return metadata