Merge pull request #177 from mlibrary/api-stub-in-search-results

LIBSEARCH-1164-stub-in-search-results

Author: niquerio

.gitignore

@@ -53,3 +53,6 @@ overlap/*.tsv
 
 #this is so github actions doesn't try to add all the gems to the repo when updating a translation map
 vendor/
+
+api/*.md
+api/*.txt

api/catalog_api/main.py

@@ -6,6 +6,7 @@
 from catalog_api import schemas
 from catalog_api.solr_client import NotFoundError
 from catalog_api.record import record_for
+from catalog_api.results import Results
 
 app = FastAPI(
     title="Catalog Search API", description="REST API for Catalog Search Solr"
@@ -42,3 +43,12 @@ def get_record(id: str) -> schemas.Record:
         return result
     except NotFoundError:
         raise HTTPException(status_code=404, detail="Item not found")
+
+
+@app.get("/search", response_model_exclude_none=True)
+def get_search_results(offset: int = 0) -> schemas.Results:
+    """
+    Does a search in catalog solr
+    """
+    results = Results({"offset": offset})
+    return results

api/catalog_api/results.py

@@ -0,0 +1,87 @@
+import json
+from pathlib import Path
+from dataclasses import dataclass
+from catalog_api.record import Record
+
+
+class Results:
+    fixture_path = Path(__file__).parents[0] / "../tests/fixtures/results/"
+    with open(fixture_path / "page1.json") as f:
+        page1 = json.load(f)
+
+    with open(fixture_path / "page2.json") as f:
+        page2 = json.load(f)
+
+    with open(fixture_path / "page3.json") as f:
+        page3 = json.load(f)
+
+    def __init__(self, data: dict):
+        offset = data["offset"]
+        if offset < 10:
+            self.data = self.page1
+        elif offset < 20:
+            self.data = self.page2
+        else:
+            self.data = self.page3
+
+    @property
+    def records(self):
+        return [Record(data) for data in self.data["response"]["docs"]]
+
+    @property
+    def filters(self):
+        facet_fields = self.data["facet_counts"]["facet_fields"]
+        return [
+            Filter(field=x, values=facet_fields[x])
+            for x in facet_fields.keys()
+            if x in Filter.filter_field_map
+        ]
+
+    @property
+    def total(self):
+        return self.data["response"]["numFound"]
+
+    @property
+    def limit(self):
+        return self.data["responseHeader"]["params"]["rows"]
+
+    @property
+    def offset(self):
+        return self.data["response"]["start"]
+
+
+# problems with filter data.
+# 1. the fields aren't the correct names
+# 2. location filter has the wrong name (time for the other api???)
+# 3. some of them (just search only?) are empty
+# 4. availability has some special rules
+class Filter:
+    filter_field_map = {
+        "availability": "availability",
+        "format": "format",
+        "topicStr": "subject",
+        "publishDateRange": "date_of_publication",
+        "language": "language",
+        "collection": "collection",
+        "hlb3Str": "academic_discipline",
+        "authorStr": "author",
+        "place_of_publication": "place_of_publication",
+        "geographicSt": "region",
+        "building": "location",
+    }  # institution and search_only are skipped for this
+
+    def __init__(self, field: str, values: list):
+        self.field = self.filter_field_map[field]
+        self.values = self.get_values(values)
+
+    def get_values(self, values):
+        result = []
+        for x in range(0, len(values), 2):
+            result.append(FilterValue(text=values[x], count=values[x + 1]))
+        return result
+
+
+@dataclass(frozen=True)
+class FilterValue:
+    text: str
+    count: int

api/catalog_api/schemas.py

@@ -264,6 +264,24 @@ class Record(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
 
+class FilterValue(BaseModel):
+    text: str
+    count: int
+
+
+class Filter(BaseModel):
+    field: str
+    values: list[FilterValue]
+
+
+class Results(BaseModel):
+    records: list[Record]
+    filters: list[Filter]
+    limit: int
+    offset: int
+    total: int
+
+
 class Response(BaseModel):
     detail: str
 

api/notes.md

@@ -0,0 +1,41 @@
+# Notes for Search Results
+
+## Example solr results query
+
+```
+Below is close to what current search is asking for. Facet count is just limited to 50 instead of 500
+http://bulleit-1.umdl.umich.edu:8026/solr/biblio/select?f.authorStr.facet.mincount=1&df=allfields&f.authorStr.facet.sort=count&f.hlb3Str.facet.offset=0&f.publishDateRange.facet.limit=50&qq1=%22jazz+singer+book%22&tie=0.1&f.institution.facet.sort=count&f.building.facet.mincount=1&f.hlb3Str.facet.sort=count&f.topicStr.facet.sort=count&f.authorStr.facet.limit=50&f.language.facet.sort=count&f.format.facet.limit=50&f.topicStr.facet.limit=50&f.building.facet.sort=count&qq=%22_query_\:\{\!edismax+mm%3D$default_mm+mm.autoRelax%3D$mm.autoRelax+tie%3D$tie+qf%3D$all_fields_qf+pf%3D$all_fields_pf+pf2%3D$all_fields_pf2+ps2%3D$all_fields_ps2+boost%3D$all_fields_boost+v%3D$q1\}%22&per_page=10&f.search_only.facet.offset=0&f.search_only.facet.mincount=1&qt=standard&f.building.facet.limit=50&f.institution.facet.limit=50&sort=score+desc&f.availability.facet.limit=50&f.search_only.facet.sort=count&default_mm=2%3C-1+5%3C67%25&f.geographicStr.facet.limit=50&mm.autoRelax=true&f.institution.facet.offset=0&f.hlb3Str.facet.mincount=1&f.geographicStr.facet.mincount=1&f.format.facet.offset=0&f.place_of_publication.facet.offset=0&f.topicStr.facet.mincount=1&page=1&t1=%22jazz+singer%22+book&f.place_of_publication.facet.mincount=1&f.language.facet.mincount=1&f.availability.facet.mincount=1&f.language.facet.offset=0&all_fields_pf2=title_author^500+title_equiv^80+title_l^50&fl=*,score&f.location.facet.mincount=1&f.publishDateRange.facet.offset=0&f.language.facet.limit=50&fq=institution:(UM\+Ann\+Arbor\+Libraries)&fq=%2B(new_availability:physical+OR+new_availability:hathi_trust_full_text_or_electronic_holding)&f.location.facet.sort=count&f.publishDateRange.facet.sort=count&all_fields_pf=title_equiv^40+title_top^20+title_rest^10+author^80+author_top^30+author_rest^20&f.format.facet.mincount=1&f.hlb3Str.facet.limit=50&f.geographicStr.facet.sort=count&f.institution.facet.mincount=1&facet.threads=10&clean_string=(%22jazz+singer%22+book)&wt=json&f.search_only.facet.limit=50&f.topicStr.facet.offset=0&f.location.facet.offset=0&q1=(%22jazz+singer%22+book)&facet.field=search_only&facet.field=availability&facet.field=format&facet.field=topicStr&facet.field=publishDateRange&facet.field=language&facet.field=location&facet.field=hlb3Str&facet.field=authorStr&facet.field=place_of_publication&facet.field=geographicStr&facet.field=institution&facet.field=building&f.place_of_publication.facet.sort=count&f.geographicStr.facet.offset=0&start=0&rows=10&all_fields_boost=product(+if(termfreq(%27format%27,+%27Journal%27),+1.4,+1),+max(+map(+query({!field+f%3Dtitle_common_exact+v%3D$q1},+1),+0,+1,+1,+180+),+map(+query({!field+f%3Dtitle_equiv_exact+v%3D$q1},+1),+0,+1,+1,+50+),+map(+query({!field+f%3Dtitle_a_exact+v%3D$q1},+1),+0,+1,+1,+10+)),+map(+query({!dismax+f%3Dtitle_author+v%3D$q1+mm%3D%22100%25%22},+1),+0,+1,+1,+50+),+)&f.availability.facet.sort=count&q=_query_:{!edismax+mm%3D$default_mm+mm.autoRelax%3D$mm.autoRelax+tie%3D$tie+qf%3D$all_fields_qf+pf%3D$all_fields_pf+pf2%3D$all_fields_pf2+ps2%3D$all_fields_ps2+boost%3D$all_fields_boost+v%3D$q1}&f.location.facet.limit=50&f.place_of_publication.facet.limit=50&f.authorStr.facet.offset=0&f.building.facet.offset=0&all_fields_ps2=2&all_fields_qf=allfieldsProper^2+allfields^1+title_common^50+title_equiv^10+mainauthor^80+author^50+isbn+issn+oclc+lccn+barcode+htid+callnosearch+bookplate&f.format.facet.sort=count&f.publishDateRange.facet.mincount=1&facet=true&f.availability.facet.offset=0
+
+
+This is what we want. location is changed to collection. Facet count is 50
+http://bulleit-1.umdl.umich.edu:8026/solr/biblio/select?f.authorStr.facet.mincount=1&df=allfields&f.authorStr.facet.sort=count&f.hlb3Str.facet.offset=0&f.publishDateRange.facet.limit=50&qq1=%22jazz+singer+book%22&tie=0.1&f.institution.facet.sort=count&f.building.facet.mincount=1&f.hlb3Str.facet.sort=count&f.topicStr.facet.sort=count&f.authorStr.facet.limit=50&f.language.facet.sort=count&f.format.facet.limit=50&f.topicStr.facet.limit=50&f.building.facet.sort=count&qq=%22_query_\:\{\!edismax+mm%3D$default_mm+mm.autoRelax%3D$mm.autoRelax+tie%3D$tie+qf%3D$all_fields_qf+pf%3D$all_fields_pf+pf2%3D$all_fields_pf2+ps2%3D$all_fields_ps2+boost%3D$all_fields_boost+v%3D$q1\}%22&per_page=10&f.search_only.facet.offset=0&f.search_only.facet.mincount=1&qt=standard&f.building.facet.limit=50&f.institution.facet.limit=50&sort=score+desc&f.availability.facet.limit=50&f.search_only.facet.sort=count&default_mm=2%3C-1+5%3C67%25&f.geographicStr.facet.limit=50&mm.autoRelax=true&f.institution.facet.offset=0&f.hlb3Str.facet.mincount=1&f.geographicStr.facet.mincount=1&f.format.facet.offset=0&f.place_of_publication.facet.offset=0&f.topicStr.facet.mincount=1&page=1&t1=%22jazz+singer%22+book&f.place_of_publication.facet.mincount=1&f.language.facet.mincount=1&f.availability.facet.mincount=1&f.language.facet.offset=0&all_fields_pf2=title_author^500+title_equiv^80+title_l^50&fl=*,score&f.collection.facet.mincount=1&f.publishDateRange.facet.offset=0&f.language.facet.limit=50&fq=institution:(UM\+Ann\+Arbor\+Libraries)&fq=%2B(new_availability:physical+OR+new_availability:hathi_trust_full_text_or_electronic_holding)&f.collection.facet.sort=count&f.publishDateRange.facet.sort=count&all_fields_pf=title_equiv^40+title_top^20+title_rest^10+author^80+author_top^30+author_rest^20&f.format.facet.mincount=1&f.hlb3Str.facet.limit=50&f.geographicStr.facet.sort=count&f.institution.facet.mincount=1&facet.threads=10&clean_string=(%22jazz+singer%22+book)&wt=json&f.search_only.facet.limit=50&f.topicStr.facet.offset=0&f.collection.facet.offset=0&q1=(%22jazz+singer%22+book)&facet.field=search_only&facet.field=availability&facet.field=format&facet.field=topicStr&facet.field=publishDateRange&facet.field=language&facet.field=collection&facet.field=hlb3Str&facet.field=authorStr&facet.field=place_of_publication&facet.field=geographicStr&facet.field=institution&facet.field=building&f.place_of_publication.facet.sort=count&f.geographicStr.facet.offset=0&start=0&rows=10&all_fields_boost=product(+if(termfreq(%27format%27,+%27Journal%27),+1.4,+1),+max(+map(+query({!field+f%3Dtitle_common_exact+v%3D$q1},+1),+0,+1,+1,+180+),+map(+query({!field+f%3Dtitle_equiv_exact+v%3D$q1},+1),+0,+1,+1,+50+),+map(+query({!field+f%3Dtitle_a_exact+v%3D$q1},+1),+0,+1,+1,+10+)),+map(+query({!dismax+f%3Dtitle_author+v%3D$q1+mm%3D%22100%25%22},+1),+0,+1,+1,+50+),+)&f.availability.facet.sort=count&q=_query_:{!edismax+mm%3D$default_mm+mm.autoRelax%3D$mm.autoRelax+tie%3D$tie+qf%3D$all_fields_qf+pf%3D$all_fields_pf+pf2%3D$all_fields_pf2+ps2%3D$all_fields_ps2+boost%3D$all_fields_boost+v%3D$q1}&f.collection.facet.limit=50&f.place_of_publication.facet.limit=50&f.authorStr.facet.offset=0&f.building.facet.offset=0&all_fields_ps2=2&all_fields_qf=allfieldsProper^2+allfields^1+title_common^50+title_equiv^10+mainauthor^80+author^50+isbn+issn+oclc+lccn+barcode+htid+callnosearch+bookplate&f.format.facet.sort=count&f.publishDateRange.facet.mincount=1&facet=true&f.availability.facet.offset=0
+```
+
+## Paths for query_parser_api
+
+```
+/catalog/search (this is the url for searching. takes in a bunch of parameters)
+/onlinejournals/search (this is the url for searching online journals)
+```
+
+```
+# Potential path; It's a different solr, but I don't think that matters. It uses the query parser so it ought to go here.
+/website.search
+
+# Other catalog paths?
+/catalog/record # could make the catalog_api not know how to query solr.
+/catalog/debug # show some output for debugging a query
+/catalog/solr # send solr queries straight through. Might be useful for super advanced search. 
+```
+
+This handles changing the complicated availability filter into something solr/query parser can read.
+
+## Results response
+
+{
+  records: []
+  filters: []
+  total: int
+  limit: int
+  offset: int
+}