Merge pull request internetarchive#11179 from cdrini/11177/feature/new-solr-list-fields

Add last_modified, seed_count fields to solr for lists

Author: mekarpeles

conf/solr/conf/managed-schema.xml

@@ -267,6 +267,10 @@
    <!-- additional fields used by documents of type subject -->
    <field name="subject_type" type="string" indexed="true" stored="true"/>
 
+   <!-- fields used by documents of type list -->
+   <field name="last_modified" type="pdate" indexed="true" stored="true"/>
+   <field name="seed_count" type="pint" indexed="true" stored="true"/>
+
     <!-- dynamic fields -->
     <dynamicField name="id_*"  type="string"  indexed="true"  stored="true" multiValued="true"/>
 

openlibrary/plugins/worksearch/code.py

@@ -734,7 +734,7 @@ class list_search(delegate.page):
     path = '/search/lists'
 
     def GET(self):  # referenced subject_search
-        req = ListSearchRequest.from_web_input(web.input(api='new'))
+        req = ListSearchRequest.from_web_input(web.input(api='next'))
         # Can't set fields when rendering html
         req.fields = 'key'
         resp = self.get_results(req, 'LIST_SEARCH')

openlibrary/plugins/worksearch/schemes/lists.py

@@ -26,6 +26,8 @@ class ListSearchScheme(SearchScheme):
             'place_key',
             'time',
             'time_key',
+            'last_modified',
+            'seed_count',
         }
     )
 
@@ -36,7 +38,13 @@ class ListSearchScheme(SearchScheme):
     field_name_map = MappingProxyType({})
     sorts = MappingProxyType(
         {
-            'name asc': 'name asc',  # sort alphabetically
+            'name asc': 'name asc',
+            'last_modified': 'last_modified desc',
+            'last_modified asc': 'last_modified asc',
+            'last_modified desc': 'last_modified desc',
+            'seed_count': 'seed_count desc',
+            'seed_count asc': 'seed_count asc',
+            'seed_count desc': 'seed_count desc',
             # Random (kept from SubjectSearchScheme)
             'random': 'random_1 asc',
             'random asc': 'random_1 asc',

openlibrary/solr/solr_types.py

@@ -1,4 +1,6 @@
 # This file is auto-generated by types_generator.py
+# To regenerate:
+#   python openlibrary/solr/types_generator.py > openlibrary/solr/solr_types.py
 # fmt: off
 # ruff: noqa: UP045
 from typing import Literal, Optional, TypedDict
@@ -121,6 +123,8 @@ class SolrDocument(TypedDict):
     top_work: Optional[str]
     top_subjects: Optional[list[str]]
     subject_type: Optional[str]
+    last_modified: Optional[str]
+    seed_count: Optional[int]
     public_scan_b: Optional[bool]
     printdisabled_s: Optional[str]
     lending_edition_s: Optional[str]

openlibrary/solr/types_generator.py

@@ -31,6 +31,7 @@ def generate():
         typ = field.get('type')
         multivalued = field.get('multiValued') == 'true'
         type_map = {
+            'pdate': 'str',
             'pint': 'int',
             'string': 'str',
             'text_en_splitting': 'str',
@@ -81,6 +82,8 @@ def generate():
 
     body = '\n'.join(python_fields)
     python = f"""# This file is auto-generated by types_generator.py
+# To regenerate:
+#   python openlibrary/solr/types_generator.py > openlibrary/solr/solr_types.py
 # fmt: off
 # ruff: noqa: UP045
 from typing import Literal, Optional, TypedDict

openlibrary/solr/updater/list.py

@@ -1,5 +1,6 @@
 import re
 from collections import defaultdict
+from functools import cached_property
 from typing import cast
 
 import httpx
@@ -106,7 +107,7 @@ def type(self) -> str:
     def name(self) -> str | None:
         return self._list.get('name')
 
-    @property
+    @cached_property
     def seed(self) -> list[str]:
         return [
             (
@@ -116,3 +117,12 @@ def seed(self) -> list[str]:
             )
             for seed in self._list.get('seeds', [])
         ]
+
+    @property
+    def last_modified(self) -> str:
+        # Solr expects UTC ISO 8601 with 'Z' suffix, e.g. 2024-12-09T12:57:14.074200Z
+        return self._list['last_modified']['value'] + 'Z'
+
+    @property
+    def seed_count(self) -> int:
+        return len(self.seed)