diff --git a/.github/workflows/test-plone-5.2.yml b/.github/workflows/test-plone-5.2.yml index 18f3097..4fc410e 100644 --- a/.github/workflows/test-plone-5.2.yml +++ b/.github/workflows/test-plone-5.2.yml @@ -39,7 +39,7 @@ jobs: ## regardless what I try. # python cache - - uses: actions/cache@v1 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} diff --git a/Makefile b/Makefile index 67dcae4..4e1a65c 100644 --- a/Makefile +++ b/Makefile @@ -163,6 +163,11 @@ solr-start-fg: test-compose-project-name ## Start solr in foreground @echo "Start solr in foreground" @COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME} docker compose -f ${SOLR_ONLY_COMPOSE} up +.PHONY: solr-start-and-rebuild +solr-start-and-rebuild: ## Start solr, force rebuild + @echo "Start solr, force rebuild, erases data" + @COMPOSE_PROJECT_NAME=${STACK_NAME} docker compose -f ${SOLR_ONLY_COMPOSE} up -d --build + .PHONY: solr-stop solr-stop: test-compose-project-name ## Stop solr @echo "Stop solr" diff --git a/README.md b/README.md index ff273aa..707cc09 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,16 @@ Run only tests that match `TestEndpointEncoding`, but stop on the first error an ./bin/tox -e test -- -k TestEndpointEncoding -x --pdb ``` +## Remark about testing and configuration + +The package configuration (and the test setup) is quite complex, and especially for testing the highlighting, multiple steps must be present in the configuration. If some of this is missing, the tests will fail. + +The highlighting is by default based on the `body_text` index which this package via an indexer. So first, the zcml must be loaded. But also worth noting that the `plone.volto` package must be loaded, if not then the `IBlocks` behavior is not active, and although the index is added, the indexer will not be called. As a consequence, the body_text indexer will not be invoked, leading to potential issues in highlighting. + +This is why `volto-solr` is referred from both `metadata.xml` and `dependencies.zcml` in this package. If any of this would be missing, the highlighting would fail. + +In an application that uses the `kitconcept.solr` package, loading its zcml and its default profile would be sufficient, because `kitconcept.solr` will pull in the behavior from `plone.volto` and the indexer from `collective.solr`, as needed. + ## Credits The development of this add-on has been kindly sponsored by [German Aerospace Center (DLR)](https://www.dlr.de) and [Forschungszentrum Jülich](https://www.fz-juelich.de). diff --git a/docker-compose.yml b/docker-compose.yml index b03a90d..ec40c2c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3" services: solr: build: diff --git a/solr/etc/conf/schema.xml b/solr/etc/conf/schema.xml index 342e6d7..14a2110 100644 --- a/solr/etc/conf/schema.xml +++ b/solr/etc/conf/schema.xml @@ -379,4 +379,4 @@ - + \ No newline at end of file diff --git a/solr/etc/conf/solrconfig.xml b/solr/etc/conf/solrconfig.xml index 4dd8773..e9b4b25 100644 --- a/solr/etc/conf/solrconfig.xml +++ b/solr/etc/conf/solrconfig.xml @@ -211,7 +211,26 @@ 10 SearchableText xml + + + json + true + true + + on + suggestDictionary + false + 4 + 2 + 0 + false + false + + + + suggest + 0. true + + + default + suggest + solr.DirectSolrSpellChecker + internal + 0.5 + + diff --git a/src/kitconcept/solr/dependencies.zcml b/src/kitconcept/solr/dependencies.zcml index 10fbfa5..a892b41 100644 --- a/src/kitconcept/solr/dependencies.zcml +++ b/src/kitconcept/solr/dependencies.zcml @@ -3,6 +3,7 @@ + diff --git a/src/kitconcept/solr/indexers/text.py b/src/kitconcept/solr/indexers/text.py index 14a65a3..9fbb0d8 100644 --- a/src/kitconcept/solr/indexers/text.py +++ b/src/kitconcept/solr/indexers/text.py @@ -137,5 +137,4 @@ def body_text_blocks(obj): for block_id in blocks_layout.get("items", []): block = blocks.get(block_id, {}) blocks_text.append(extract_text(block, obj, request)) - return " ".join([text.strip() for text in blocks_text if text.strip()]) diff --git a/src/kitconcept/solr/profiles/default/catalog.xml b/src/kitconcept/solr/profiles/default/catalog.xml new file mode 100644 index 0000000..a02795a --- /dev/null +++ b/src/kitconcept/solr/profiles/default/catalog.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + diff --git a/src/kitconcept/solr/profiles/default/metadata.xml b/src/kitconcept/solr/profiles/default/metadata.xml index 5625aab..28e92de 100644 --- a/src/kitconcept/solr/profiles/default/metadata.xml +++ b/src/kitconcept/solr/profiles/default/metadata.xml @@ -1,7 +1,8 @@ - 1000 + 1001 + profile-plone.volto:default profile-collective.solr:default diff --git a/src/kitconcept/solr/profiles/default/registry/kitconcept.solr.interfaces.IKitconceptSolrSettings.xml b/src/kitconcept/solr/profiles/default/registry/kitconcept.solr.interfaces.IKitconceptSolrSettings.xml index 4f03639..c26f09c 100644 --- a/src/kitconcept/solr/profiles/default/registry/kitconcept.solr.interfaces.IKitconceptSolrSettings.xml +++ b/src/kitconcept/solr/profiles/default/registry/kitconcept.solr.interfaces.IKitconceptSolrSettings.xml @@ -26,6 +26,11 @@ "image_scales", "image_field" ], + "highlightingFields": [ + {"field": "content", "prop": "highlighting"}, + {"field": "title", "prop": "highlighting_title"}, + {"field": "description", "prop": "highlighting_description"} + ], "searchTabs": [ { "label": "All", diff --git a/src/kitconcept/solr/services/solr.py b/src/kitconcept/solr/services/solr.py index 08be630..a393dcb 100644 --- a/src/kitconcept/solr/services/solr.py +++ b/src/kitconcept/solr/services/solr.py @@ -1,20 +1,15 @@ +from .solr_highlighting_utils import SolrHighlightingUtils +from .solr_params import SolrParams from .solr_utils import escape -from .solr_utils import FacetConditions from .solr_utils import get_facet_fields_result from .solr_utils import replace_colon from .solr_utils import replace_reserved from .solr_utils import SolrConfig -from .solr_utils_extra import SolrExtraConditions from AccessControl.SecurityManagement import getSecurityManager from collective.solr.interfaces import ISolrConnectionManager -from functools import reduce from itertools import zip_longest -from plone import api -from plone.app.multilingual.interfaces import ITranslatable -from plone.app.multilingual.interfaces import ITranslationManager from plone.restapi.services import Service from Products.CMFPlone.utils import base_hasattr -from zExceptions import BadRequest from zope.component import queryUtility from zope.interface import alsoProvides @@ -60,67 +55,7 @@ def reply(self): self.request, plone.protect.interfaces.IDisableCSRFProtection ) - query = self.request.form.get("q", None) - start = self.request.form.get("start", None) - rows = self.request.form.get("rows", None) - sort = self.request.form.get("sort", None) - group_select = self.request.form.get("group_select", None) - path_prefix = self.request.form.get("path_prefix", "") - portal_type = self.request.form.get("portal_type", None) - lang = self.request.form.get("lang", None) - keep_full_solr_response = ( - self.request.form.get("keep_full_solr_response", "").lower() - == "true" - ) - - # search in multilingual path_prefix by default - unless lang is specified - is_multilinqual_txt = self.request.form.get( - "is_multilingual", "true" if lang is None else "false" - ) - is_multilingual = is_multilinqual_txt.lower() == "true" - portal = api.portal.get() - portal_path_segments = portal.getPhysicalPath() - portal_path = "/".join(portal_path_segments) - - facet_conditions = FacetConditions.from_encoded( - self.request.form.get("facet_conditions", None) - ) - - extra_conditions = SolrExtraConditions.from_encoded( - self.request.form.get("extra_conditions", None) - ) - - if not path_prefix: - # The path_prefix can optionally be used, but the root url is - # used as default, in case path_prefix is undefined. - context_path_segments = self.context.getPhysicalPath() - # Acuqire relative path - path_prefix_segments = context_path_segments[ - len(portal_path_segments) : - ] - path_prefix = "/".join(path_prefix_segments) - - if lang and is_multilingual: - raise BadRequest( - "Property 'lang` and `is_multilingual` are mutually exclusive" - ) - - if group_select is not None: - try: - group_select = int(group_select) - except ValueError: - raise BadRequest( - "Property 'group_select` must be an integer, if specified" - ) - elif len(solr_config.filters) > 0: - # By default select group 0 (unless there are no filters defined) - group_select = 0 - - facet_fields = ( - solr_config.select_facet_fields(group_select) - if group_select is not None - else [] - ) + params = SolrParams(self.context, self.request, solr_config) # Get the solr connection manager = queryUtility(ISolrConnectionManager) @@ -134,7 +69,11 @@ def reply(self): # # In addition. support empty search to search all terms, in case this # is configured. - term = "(" + escape(replace_reserved(query)) + ")" if query else "*" + term = ( + "(" + escape(replace_reserved(params.query)) + ")" + if params.query + else "*" + ) # Search # q: query parameter @@ -166,30 +105,32 @@ def reply(self): d = { "q": f"+(Title:{term}^5 OR Description:{term}^2 OR id:{term}^0.75 OR text_prefix:{term}^0.75 OR text_suffix:{term}^0.75 OR default:{term} OR body_text:{term} OR SearchableText:{term} OR Subject:{term} OR searchwords:({term})^1000) -showinsearch:False", # noqa "wt": "json", - "hl": "true", - "hl.fl": "content", # content only used for highlighting, the field is not indexed # noqa - "fq": [security_filter()], + "fq": [security_filter()] + params.extra_fq(self.context), "fl": solr_config.field_list, "facet": "true", "facet.contains.ignoreCase": "true", "facet.field": list( map( - lambda info: facet_conditions.ex_field_facet(info["name"]) + lambda info: params.facet_conditions.ex_field_facet( + info["name"] + ) + info["name"], - facet_fields, + params.facet_fields, ) ), } - if start is not None: - d["start"] = start - if rows is not None: - d["rows"] = rows - if sort is not None: - d["sort"] = sort - if group_select is not None: - d["fq"] = d["fq"] + [solr_config.select_condition(group_select)] - ex_all_facets = facet_conditions.ex_all_facets( + if params.start is not None: + d["start"] = params.start + if params.rows is not None: + d["rows"] = params.rows + if params.sort is not None: + d["sort"] = params.sort + if params.group_select is not None: + d["fq"] = d["fq"] + [ + solr_config.select_condition(params.group_select) + ] + ex_all_facets = params.facet_conditions.ex_all_facets( extending=["typefilter"] ) d["facet.query"] = list( @@ -198,86 +139,42 @@ def reply(self): solr_config.filters, ) ) - if path_prefix: - is_excluding = re_is_excluding.search(path_prefix) - if is_multilingual: - # All the translation folders have to be queried, if the - # multilingual option is selected. (By default.) - relative_path = re_relative_path.sub("", path_prefix) - folder = self.context.unrestrictedTraverse(relative_path) - if ITranslatable.providedBy(folder): - translations = ( - ITranslationManager(folder).get_translations().values() - ) - else: - # Untranslated - translations = [folder] - path_list = map( - lambda o: "/".join(o.getPhysicalPath()), translations - ) - else: - # Not multilingual. Search stricly in the given path. - if is_excluding: - path_prefix = path_prefix[:-1] - path_list = [portal_path + path_prefix] - if is_excluding: - # Expressions with a trailing / will exclude the - # parent folder and include everything else. - path_expr = reduce( - lambda sum, path: sum - + [f"path_string:{escape(path + '/')}*"], - path_list, - [], - ) - else: - # Expression that include a folder must be split up to - # two conditions, to avoid `/something` to match - # '/something-else'. - path_expr = reduce( - lambda sum, path: sum - + [ - f"path_string:{escape(path)}", - f"path_string:{escape(path + '/')}*", - ], - path_list, - [], - ) - if len(path_expr) > 0: - d["fq"] = d["fq"] + ["(" + " OR ".join(path_expr) + ")"] - if portal_type: - # Convert to Type condition. - d["fq"] = d["fq"] + [ - "Type:(" - + " OR ".join( - map(lambda txt: '"' + escape(txt) + '"', portal_type) - ) - + ")" - ] - if lang: - d["fq"] = d["fq"] + ["Language:(" + escape(lang) + ")"] - d["fq"] = d["fq"] + facet_conditions.field_conditions_solr - - # extra_conditions = [ - # [ - # "start", - # "date-range", - # {"ge": "2021-07-19T13:00:00Z", "le": "2025-07-19T13:00:00Z"}, - # ], - # ] - query_list = extra_conditions.query_list() - d["fq"] = d["fq"] + query_list + d.update(params.facet_conditions.contains_query) + d.update( + params.facet_conditions.more_query( + params.facet_fields, multiplier=2 + ) + ) - d.update(facet_conditions.contains_query) - d.update(facet_conditions.more_query(facet_fields, multiplier=2)) + highlighting_utils = SolrHighlightingUtils(solr_config) + d["hl"] = "true" if highlighting_utils.enabled else "false" + d["hl.fl"] = highlighting_utils.fields raw_result = connection.search(**d).read() - result = json.loads(raw_result) + + # Replace result with "Did you mean" response, if there are no results + # and it's requested by the collate parameter + if ( + result["response"]["numFound"] == 0 + and "spellcheck" in result + and "collations" in result["spellcheck"] + and len(result["spellcheck"]["collations"]) > 1 + and params.collate + ): + collation = result["spellcheck"]["collations"][1] + d["q"] = collation["collationQuery"] + raw_result = connection.search(**d).read() + result = json.loads(raw_result) + result["collation_misspellings"] = collation[ + "misspellingsAndCorrections" + ] + # Add portal path to the result. This can be used by # the front-end to calculate the @id from the path_string # for each item. - result["portal_path"] = portal_path + result["portal_path"] = params.portal_path # Add the group counts in order. Needed because the # facet_queries dictionary will not preserve key order, when # marshalled to the client. @@ -290,14 +187,24 @@ def reply(self): ) result["facet_fields"] = get_facet_fields_result( result["facet_counts"]["facet_fields"], - facet_fields, - facet_conditions.more_dict(facet_fields, multiplier=1), + params.facet_fields, + params.facet_conditions.more_dict( + params.facet_fields, multiplier=1 + ), + ) + # Add highlighting information to the result + highlighting_utils.enhance_items( + result.get("response", {}).get("docs", []), + result.get("highlighting", {}), ) # Solr response is pruned of the unnecessary parts, unless explicitly requested. - if not keep_full_solr_response: - del result["facet_counts"] + if not params.keep_full_solr_response: + if "facet_counts" in result: + del result["facet_counts"] + if "highlighting" in result: + del result["highlighting"] # Embellish result with supplemental information for the front-end - if group_select is not None: - layouts = solr_config.select_layouts(group_select) + if params.group_select is not None: + layouts = solr_config.select_layouts(params.group_select) result["layouts"] = layouts return result diff --git a/src/kitconcept/solr/services/solr_highlighting_utils.py b/src/kitconcept/solr/services/solr_highlighting_utils.py new file mode 100644 index 0000000..9618eaa --- /dev/null +++ b/src/kitconcept/solr/services/solr_highlighting_utils.py @@ -0,0 +1,23 @@ +class SolrHighlightingUtils: + fields: list + enabled: bool + propByField: dict + + def __init__(self, solr_config): + highlightingFields = solr_config.config.get("highlightingFields", []) + self.fields = list( + map(lambda field: field["field"], highlightingFields) + ) + self.enabled = len(self.fields) > 0 + self.propByField = dict( + map( + lambda field: (field["field"], field["prop"]), + highlightingFields, + ) + ) + + def enhance_items(self, items: list, highlighting: dict): + if self.enabled: + for item in items: + for field, value in highlighting.get(item["UID"], {}).items(): + item[self.propByField[field]] = value diff --git a/src/kitconcept/solr/services/solr_params.py b/src/kitconcept/solr/services/solr_params.py new file mode 100644 index 0000000..7791ed5 --- /dev/null +++ b/src/kitconcept/solr/services/solr_params.py @@ -0,0 +1,154 @@ +from .solr_utils import escape +from .solr_utils import FacetConditions +from .solr_utils_extra import SolrExtraConditions +from functools import reduce +from plone import api +from plone.app.multilingual.interfaces import ITranslatable +from plone.app.multilingual.interfaces import ITranslationManager +from zExceptions import BadRequest + +import re + + +re_relative_path = re.compile("^/+") +re_is_excluding = re.compile("/$") + + +class SolrParams: + def __init__(self, context, request, solr_config): + form = request.form + self.query = form.get("q", None) + self.start = form.get("start", None) + self.rows = form.get("rows", None) + self.sort = form.get("sort", None) + group_select = form.get("group_select", None) + self.path_prefix = form.get("path_prefix", "") + self.portal_type = form.get("portal_type", None) + self.lang = form.get("lang", None) + self.keep_full_solr_response = ( + form.get("keep_full_solr_response", "").lower() == "true" + ) + self.collate = form.get("collate", "").lower() == "true" + + # search in multilingual path_prefix by default - unless lang is specified + is_multilinqual_txt = form.get( + "is_multilingual", "true" if self.lang is None else "false" + ) + self.is_multilingual = is_multilinqual_txt.lower() == "true" + portal = api.portal.get() + portal_path_segments = portal.getPhysicalPath() + self.portal_path = "/".join(portal_path_segments) + + self.facet_conditions = FacetConditions.from_encoded( + form.get("facet_conditions", None) + ) + + self.extra_conditions = SolrExtraConditions.from_encoded( + form.get("extra_conditions", None) + ) + + if not self.path_prefix: + # The path_prefix can optionally be used, but the root url is + # used as default, in case path_prefix is undefined. + context_path_segments = context.getPhysicalPath() + # Acuqire relative path + path_prefix_segments = context_path_segments[ + len(portal_path_segments) : + ] + self.path_prefix = "/".join(path_prefix_segments) + + if self.lang and self.is_multilingual: + raise BadRequest( + "Property 'lang` and `is_multilingual` are mutually exclusive" + ) + + if group_select is not None: + try: + self.group_select = int(group_select) + except ValueError: + raise BadRequest( + "Property 'group_select` must be an integer, if specified" + ) + elif len(solr_config.filters) > 0: + # By default select group 0 (unless there are no filters defined) + self.group_select = 0 + + self.facet_fields = ( + solr_config.select_facet_fields(self.group_select) + if group_select is not None + else [] + ) + + def extra_fq(self, context): + fq = [] + if self.path_prefix: + is_excluding = re_is_excluding.search(self.path_prefix) + if self.is_multilingual: + # All the translation folders have to be queried, if the + # multilingual option is selected. (By default.) + relative_path = re_relative_path.sub("", self.path_prefix) + folder = context.unrestrictedTraverse(relative_path) + if ITranslatable.providedBy(folder): + translations = ( + ITranslationManager(folder).get_translations().values() + ) + else: + # Untranslated + translations = [folder] + path_list = map( + lambda o: "/".join(o.getPhysicalPath()), translations + ) + else: + # Not multilingual. Search stricly in the given path. + if is_excluding: + self.path_prefix = self.path_prefix[:-1] + path_list = [self.portal_path + self.path_prefix] + if is_excluding: + # Expressions with a trailing / will exclude the + # parent folder and include everything else. + path_expr = reduce( + lambda sum, path: sum + + [f"path_string:{escape(path + '/')}*"], + path_list, + [], + ) + else: + # Expression that include a folder must be split up to + # two conditions, to avoid `/something` to match + # '/something-else'. + path_expr = reduce( + lambda sum, path: sum + + [ + f"path_string:{escape(path)}", + f"path_string:{escape(path + '/')}*", + ], + path_list, + [], + ) + if len(path_expr) > 0: + fq += ["(" + " OR ".join(path_expr) + ")"] + if self.portal_type: + # Convert to Type condition. + fq += [ + "Type:(" + + " OR ".join( + map(lambda txt: '"' + escape(txt) + '"', self.portal_type) + ) + + ")" + ] + if self.lang: + fq += ["Language:(" + escape(self.lang) + ")"] + fq += self.facet_conditions.field_conditions_solr + + # extra_conditions = [ + # [ + # "start", + # "date-range", + # {"ge": "2021-07-19T13:00:00Z", "le": "2025-07-19T13:00:00Z"}, + # ], + # ] + + query_list = self.extra_conditions.query_list() + fq += query_list + + return fq diff --git a/src/kitconcept/solr/upgrades/configure.zcml b/src/kitconcept/solr/upgrades/configure.zcml index eaa5c6f..3351cda 100644 --- a/src/kitconcept/solr/upgrades/configure.zcml +++ b/src/kitconcept/solr/upgrades/configure.zcml @@ -4,4 +4,15 @@ i18n_domain="kitconcept.solr" > + + + + diff --git a/tests/services/content_filters/conftest.py b/tests/services/content_filters/conftest.py index d63d75e..557a895 100644 --- a/tests/services/content_filters/conftest.py +++ b/tests/services/content_filters/conftest.py @@ -41,7 +41,7 @@ def contents() -> List: "_container": "", "type": "News Item", "id": "blue", - "title": "Blue news item", + "title": "A news item", "language": "en", }, ] diff --git a/tests/services/content_filters/test_endpoint_contentfields.py b/tests/services/content_filters/test_endpoint_contentfields.py index bebc801..dc72dfd 100644 --- a/tests/services/content_filters/test_endpoint_contentfields.py +++ b/tests/services/content_filters/test_endpoint_contentfields.py @@ -10,7 +10,7 @@ def _init(self, portal_with_content, manager_request): class TestEndpointContentFieldsId(TestEndpointContentFields): - url = "/@solr?q=red" + url = "/@solr?q=redandblue" @pytest.mark.parametrize( "path,expected", @@ -31,6 +31,76 @@ def test_count(self): assert self.data["response"]["numFound"] == 1 +class TestEndpointContentFieldsIdPrefixNotSupported(TestEndpointContentFields): + url = "/@solr?q=red" + + @pytest.mark.parametrize( + "path,expected", + [ + ("/plone/redandblue", False), + ("/plone/news1", False), + ("/plone/news2", False), + ("/plone/news3", False), + ("/plone/blue", False), + ], + ) + def test_paths(self, all_path_string, path: str, expected: bool): + path_strings = all_path_string(self.data) + assert (path in path_strings) is expected + + def test_count(self): + assert "response" in self.data + assert self.data["response"]["numFound"] == 0 + + +class TestEndpointContentFieldsIdContainmentNotSupported( + TestEndpointContentFields +): + url = "/@solr?q=and" + + @pytest.mark.parametrize( + "path,expected", + [ + ("/plone/redandblue", False), + ("/plone/news1", False), + ("/plone/news2", False), + ("/plone/news3", False), + ("/plone/blue", False), + ], + ) + def test_paths(self, all_path_string, path: str, expected: bool): + path_strings = all_path_string(self.data) + assert (path in path_strings) is expected + + def test_count(self): + assert "response" in self.data + assert self.data["response"]["numFound"] == 0 + + +class TestEndpointContentFieldsIdPostfixNotSupported( + TestEndpointContentFields +): + url = "/@solr?q=blue" + + @pytest.mark.parametrize( + "path,expected", + [ + ("/plone/redandblue", False), + ("/plone/news1", False), + ("/plone/news2", False), + ("/plone/news3", False), + ("/plone/blue", True), + ], + ) + def test_paths(self, all_path_string, path: str, expected: bool): + path_strings = all_path_string(self.data) + assert (path in path_strings) is expected + + def test_count(self): + assert "response" in self.data + assert self.data["response"]["numFound"] == 1 + + class TestEndpointContentFieldsTitle(TestEndpointContentFields): url = "/@solr?q=news" @@ -58,7 +128,7 @@ def test_count(self): (0, "News Item 1", ""), (1, "News Item 2", ""), (2, "News Item 3", "My location"), - (3, "Blue news item", ""), + (3, "A news item", ""), ], ) def test_attributes(self, idx: int, title: str, location: str): diff --git a/tests/services/content_filters/test_endpoint_portaltype.py b/tests/services/content_filters/test_endpoint_portaltype.py index 047a537..560e0e7 100644 --- a/tests/services/content_filters/test_endpoint_portaltype.py +++ b/tests/services/content_filters/test_endpoint_portaltype.py @@ -51,7 +51,7 @@ class TestEndpointPortalTypeMultiple(TestEndpointPortalType): @pytest.mark.parametrize( "path,expected", [ - ("/plone/redandblue", True), + ("/plone/redandblue", False), ("/plone/news1", False), ("/plone/news2", False), ("/plone/news3", False), diff --git a/tests/services/encoding/conftest.py b/tests/services/encoding/conftest.py index 27d42ab..23bf6f8 100644 --- a/tests/services/encoding/conftest.py +++ b/tests/services/encoding/conftest.py @@ -17,14 +17,14 @@ def contents() -> List: }, { "_container": "", - "type": "Folder", + "type": "Document", "id": "myfolder", "title": "My Folder to store everything about Noam Chomsky", "language": "en", }, { "_container": "", - "type": "Folder", + "type": "Document", "id": "myotherfolder", "title": "My other Folder", "language": "en", diff --git a/tests/services/extra_conditions/conftest.py b/tests/services/extra_conditions/conftest.py index 7470712..999ddea 100644 --- a/tests/services/extra_conditions/conftest.py +++ b/tests/services/extra_conditions/conftest.py @@ -17,7 +17,7 @@ def contents() -> List: }, { "_container": "", - "type": "Folder", + "type": "Document", "id": "myfolder", "title": "My Folder to store everything about Noam Chomsky", "language": "en", diff --git a/tests/services/extra_conditions/test_extra_conditions.py b/tests/services/extra_conditions/test_extra_conditions.py index 253f103..b1b3f40 100644 --- a/tests/services/extra_conditions/test_extra_conditions.py +++ b/tests/services/extra_conditions/test_extra_conditions.py @@ -96,7 +96,7 @@ def test_facet_fields(self): assert self.data.get("facet_fields") == [ [ {"name": "Title", "label": "The title"}, - [["chomsky", 6], ["bar", 3], ["foo", 3]], + [["chomsky", 7], ["bar", 3], ["foo", 3]], ], [ {"name": "Description", "label": "The description"}, @@ -108,7 +108,7 @@ def test_facet_fields(self): "path,expected", [ ("/plone/myimage", False), - ("/plone/myfolder", False), + ("/plone/myfolder", True), ("/plone/myfolder/mynews", False), ("/plone/foo_alpha", True), ("/plone/foo_beta", True), diff --git a/tests/services/facet_conditions/conftest.py b/tests/services/facet_conditions/conftest.py index 60163c2..310482c 100644 --- a/tests/services/facet_conditions/conftest.py +++ b/tests/services/facet_conditions/conftest.py @@ -17,7 +17,7 @@ def contents() -> List: }, { "_container": "", - "type": "Folder", + "type": "Document", "id": "myfolder", "title": "My Folder to store everything about Noam Chomsky", "language": "en", diff --git a/tests/services/facet_conditions/test_facet_conditions.py b/tests/services/facet_conditions/test_facet_conditions.py index aac2c85..9666abd 100644 --- a/tests/services/facet_conditions/test_facet_conditions.py +++ b/tests/services/facet_conditions/test_facet_conditions.py @@ -96,7 +96,7 @@ def test_facet_fields(self): assert self.data.get("facet_fields") == [ [ {"name": "Title", "label": "The title"}, - [["chomsky", 6], ["bar", 3], ["foo", 3]], + [["chomsky", 7], ["bar", 3], ["foo", 3]], ], [ {"name": "Description", "label": "The description"}, @@ -108,7 +108,7 @@ def test_facet_fields(self): "path,expected", [ ("/plone/myimage", False), - ("/plone/myfolder", False), + ("/plone/myfolder", True), ("/plone/myfolder/mynews", False), ("/plone/foo_alpha", True), ("/plone/foo_beta", True), @@ -131,7 +131,7 @@ def test_facet_fields(self): assert self.data.get("facet_fields") == [ [ {"name": "Title", "label": "The title"}, - [["chomsky", 6], ["bar", 3], ["foo", 3]], + [["chomsky", 7], ["bar", 3], ["foo", 3]], ], [ {"name": "Description", "label": "The description"}, @@ -242,7 +242,7 @@ def test_facet_fields(self): assert self.data.get("facet_fields") == [ [ {"name": "Title", "label": "The title"}, - [["chomsky", 6], ["bar", 3], ["foo", 3]], + [["chomsky", 7], ["bar", 3], ["foo", 3]], ], [ {"name": "Description", "label": "The description"}, diff --git a/tests/services/highlighting/conftest.py b/tests/services/highlighting/conftest.py new file mode 100644 index 0000000..c0d8343 --- /dev/null +++ b/tests/services/highlighting/conftest.py @@ -0,0 +1,76 @@ +from typing import List + +import pytest + + +@pytest.fixture +def contents() -> List: + """Content to be created.""" + return [ + { + "_container": "", + "type": "Image", + "id": "noamchomsky", + "title": "Prof. Dr. Noam Chomsky", + "description": "The real Chomsky is here.", + "subjects": ["mymembersubject", "mymembersubjecttwo"], + "_image": b"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAA1JREFUGFdjCDO+/R8ABKsCZD++CcMAAAAASUVORK5CYII=", # noQA + }, + { + "_container": "", + "type": "Document", + "id": "mydocument", + "title": "My Document about Noam Chomsky", + # Highlighting by default is based on `body_text` which is produced + # from the block content. + "blocks_layout": { + "items": [ + "262c25c6-b5e1-4193-9931-a8a060e2ad93", + ] + }, + "blocks": { + "262c25c6-b5e1-4193-9931-a8a060e2ad93": { + "@type": "description", + "text": "This is a description about Chomsky", + } + }, + }, + { + "_container": "", + "type": "News Item", + "id": "mynews", + "title": "My News Item with Noam Chomsky", + # Highlighting by default is based on `body_text` which is produced + # from the block content. + "blocks_layout": { + "items": [ + "262c25c6-b5e1-4193-9931-a8a060e2ad93", + ] + }, + "blocks": { + "262c25c6-b5e1-4193-9931-a8a060e2ad93": { + "@type": "description", + "text": "Some more news about Chomsky", + } + }, + }, + { + "_container": "", + "type": "Document", + "id": "myotherfolder", + "title": "My other Folder", + # Highlighting by default is based on `body_text` which is produced + # from the block content. + "blocks_layout": { + "items": [ + "262c25c6-b5e1-4193-9931-a8a060e2ad93", + ] + }, + "blocks": { + "262c25c6-b5e1-4193-9931-a8a060e2ad93": { + "@type": "description", + "text": "Container for material about Chomsky", + } + }, + }, + ] diff --git a/tests/services/highlighting/test_endpoint_highlighting.py b/tests/services/highlighting/test_endpoint_highlighting.py new file mode 100644 index 0000000..a99db8c --- /dev/null +++ b/tests/services/highlighting/test_endpoint_highlighting.py @@ -0,0 +1,51 @@ +import pytest + + +class TestEndpointDefault: + @pytest.fixture(autouse=True) + def _init(self, portal_with_content, manager_request): + self.portal = portal_with_content + response = manager_request.get(self.url) + self.data = response.json() + + def func(data: dict) -> list[str]: + return [item["path_string"] for item in data["response"]["docs"]] + + url = "/@solr?q=chomsky" + + +class TestHighlighting(TestEndpointDefault): + @pytest.mark.parametrize( + "path,highlight_field,highlight", + [ + ( + "/plone/mydocument", + "highlighting", + ["This is a description about Chomsky"], + ), + ( + "/plone/noamchomsky", + "highlighting", + None, + ), + ( + "/plone/mynews", + "highlighting", + ["Some more news about Chomsky"], + ), + ( + "/plone/myotherfolder", + "highlighting", + ["Container for material about Chomsky"], + ), + ], + ) + def test_highlight_field( + self, path: str, highlight_field: str, highlight: str + ): + filtered = [ + item.get(highlight_field, None) + for item in self.data["response"]["docs"] + if item["path_string"] == path + ] + assert (filtered[0] if len(filtered) > 0 else None) == highlight diff --git a/tests/services/highlighting/test_endpoint_highlighting_alternate_field.py b/tests/services/highlighting/test_endpoint_highlighting_alternate_field.py new file mode 100644 index 0000000..6ba3876 --- /dev/null +++ b/tests/services/highlighting/test_endpoint_highlighting_alternate_field.py @@ -0,0 +1,93 @@ +from test_endpoint_highlighting import TestEndpointDefault + +import pytest + + +solr_config = { + "fieldList": [ + "UID", + "Title", + "Description", + "Type", + "effective", + "start", + "created", + "end", + "path_string", + "phone", + "email", + "location", + ], + "highlightingFields": [{"field": "content", "prop": "alternate"}], + "searchTabs": [ + { + "label": "All", + "filter": "Type(*)", + }, + { + "label": "Pages", + "filter": "Type:(Page)", + "facetFields": [ + { + "name": "Title", + "label": "The title", + }, + {"name": "Description", "label": "The description"}, + ], + }, + { + "label": "News Items", + "filter": 'Type:("News Item")', + }, + { + "label": "Pages and News Items", + "filter": 'Type:(Page OR "News Item")', + }, + ], +} + + +@pytest.fixture() +def registry_config() -> dict: + """Override registry configuration.""" + return { + "collective.solr.active": 1, + "kitconcept.solr.config": solr_config, + } + + +class TestHighlightingAlternateField(TestEndpointDefault): + @pytest.mark.parametrize( + "path,highlight_field,highlight", + [ + ( + "/plone/mydocument", + "alternate", + ["This is a description about Chomsky"], + ), + ( + "/plone/noamchomsky", + "alternate", + None, + ), + ( + "/plone/mynews", + "alternate", + ["Some more news about Chomsky"], + ), + ( + "/plone/myotherfolder", + "alternate", + ["Container for material about Chomsky"], + ), + ], + ) + def test_highlight_field( + self, path: str, highlight_field: str, highlight: str + ): + filtered = [ + item.get(highlight_field, None) + for item in self.data["response"]["docs"] + if item["path_string"] == path + ] + assert (filtered[0] if len(filtered) > 0 else None) == highlight diff --git a/tests/services/language/conftest.py b/tests/services/language/conftest.py index 0c5ecea..5ffdb4d 100644 --- a/tests/services/language/conftest.py +++ b/tests/services/language/conftest.py @@ -17,14 +17,14 @@ def contents() -> List: }, { "_container": "", - "type": "Folder", + "type": "Document", "id": "myfolder", "title": "My Folder to store everything about Noam Chomsky", "language": "en", }, { "_container": "", - "type": "Folder", + "type": "Document", "id": "myotherfolder", "title": "My other Folder", "language": "en", @@ -45,7 +45,7 @@ def contents() -> List: }, { "_container": "", - "type": "Folder", + "type": "Document", "id": "myfolder_de", "title": "My Folder to store everything about Noam Chomsky in Deutsch", "language": "de", diff --git a/tests/services/local_search/test_endpoint_local.py b/tests/services/local_search/test_endpoint_local.py index ba0acec..89fe89b 100644 --- a/tests/services/local_search/test_endpoint_local.py +++ b/tests/services/local_search/test_endpoint_local.py @@ -16,13 +16,13 @@ def contents() -> List: }, { "_container": "", - "type": "Folder", + "type": "Document", "id": "myfolder", "title": "My Folder to store everything about Noam Chomsky", }, { "_container": "", - "type": "Folder", + "type": "Document", "id": "myotherfolder", "title": "My other Folder", }, diff --git a/tests/services/response/test_response.py b/tests/services/response/test_response.py index bd112aa..7ee888c 100644 --- a/tests/services/response/test_response.py +++ b/tests/services/response/test_response.py @@ -16,6 +16,7 @@ "email", "location", ], + "highlightingFields": [{"field": "content", "prop": "highlighting"}], "searchTabs": [ { "label": "All", @@ -60,6 +61,7 @@ class TestResponseKeepFullSolrResponseDefault(TestResponseCustom): def test_facet_counts(self): assert "response" in self.data assert "facet_counts" not in self.data + assert "highlighting" not in self.data class TestResponseKeepFullSolrResponseIsFalse(TestResponseCustom): @@ -68,6 +70,7 @@ class TestResponseKeepFullSolrResponseIsFalse(TestResponseCustom): def test_facet_counts(self): assert "response" in self.data assert "facet_counts" not in self.data + assert "highlighting" not in self.data class TestResponseKeepFullSolrResponseDefaultIsTrue(TestResponseCustom): @@ -76,3 +79,4 @@ class TestResponseKeepFullSolrResponseDefaultIsTrue(TestResponseCustom): def test_facet_counts(self): assert "response" in self.data assert "facet_counts" in self.data + assert "highlighting" in self.data diff --git a/tests/services/suggest/test_suggest.py b/tests/services/suggest/test_suggest.py index d1e3cf3..cd90fa1 100644 --- a/tests/services/suggest/test_suggest.py +++ b/tests/services/suggest/test_suggest.py @@ -10,21 +10,9 @@ def _init(self, portal_with_content, manager_request): self.data = response.json() -@pytest.fixture -def get_suggest_result_props(): - def func( - a: dict, - ) -> bool: - ac = dict(a) - del ac["@id"] - return ac - - return func - - @pytest.fixture def get_suggest_result_path(): - def func(a: dict) -> bool: + def func(a: dict) -> str: return urllib.parse.urlparse(a["@id"]).path return func @@ -40,51 +28,23 @@ def func(data, index: int) -> dict: class TestSuggestDefaultBaseSearch(TestSuggestDefault): url = "/@solr-suggest?query=chomsky" - expected_result = [ - { - "@id": "http://localhost:59793/plone/mydocument", - "@type": "Document", - "description": "", - "review_state": "private", - "title": "My Document about Noam Chomsky", - "type_title": "Page", - }, - { - "@id": "http://localhost:59793/plone/mynews", - "@type": "News Item", - "description": "", - "review_state": "private", - "title": "My News Item with Noam Chomsky", - "type_title": "News Item", - }, + expected = [ + "/plone/mydocument", + "/plone/mynews", ] @pytest.mark.parametrize( - "index,expected_dict", - enumerate(expected_result), + "index,expected_path", + enumerate(expected), ) def test_suggest_result_path( self, get_suggest_item, get_suggest_result_path, index: int, - expected_dict: dict, - ): - assert get_suggest_result_path( - get_suggest_item(self.data, index) - ) == get_suggest_result_path(expected_dict) - - @pytest.mark.parametrize( - "index,expected_dict", - enumerate(expected_result), - ) - def test_suggest_result_props( - self, - get_suggest_item, - get_suggest_result_props, - index: int, - expected_dict: dict, + expected_path: str, ): - assert get_suggest_result_props( - get_suggest_item(self.data, index) - ) == get_suggest_result_props(expected_dict) + assert ( + get_suggest_result_path(get_suggest_item(self.data, index)) + == expected_path + ) diff --git a/tests/setup/test_setup_install.py b/tests/setup/test_setup_install.py index 4a6b2eb..b4bfc30 100644 --- a/tests/setup/test_setup_install.py +++ b/tests/setup/test_setup_install.py @@ -10,7 +10,7 @@ def test_addon_installed(self, installer): def test_latest_version(self, profile_last_version): """Test latest version of default profile.""" - assert profile_last_version(f"{PACKAGE_NAME}:default") == "1000" + assert profile_last_version(f"{PACKAGE_NAME}:default") == "1001" def test_browserlayer(self, browser_layers): """Test that IKitconceptSolrLayer is registered.""" diff --git a/tests/utils/test_utils_highlighting.py b/tests/utils/test_utils_highlighting.py new file mode 100644 index 0000000..407f5a5 --- /dev/null +++ b/tests/utils/test_utils_highlighting.py @@ -0,0 +1,115 @@ +# fmt: off +from kitconcept.solr.services.solr_highlighting_utils import SolrHighlightingUtils + +import pytest + + +class MockSolrConfig: + def __init__(self, config): + self.config = config + + +class TestSolrHighlightingUtils: + @pytest.fixture + def solr_config(self): + return { + "highlightingFields": [ + {"field": "title", "prop": "highlight_title"}, + {"field": "description", "prop": "highlight_description"}, + ] + } + + @pytest.fixture + def items(self): + return [ + { + "UID": "UID1", + "field_a": "value_a1", + "field_b": "value_b1", + }, + { + "UID": "UID2", + "field_a": "value_a2", + "field_b": "value_b2", + }, + ] + + def test_init(self, solr_config): + utils = SolrHighlightingUtils(MockSolrConfig(solr_config)) + assert utils.fields == ["title", "description"] + assert utils.enabled + assert utils.propByField == { + "title": "highlight_title", + "description": "highlight_description", + } + + def test_enhance_items(self, solr_config, items): + utils = SolrHighlightingUtils(MockSolrConfig(solr_config)) + highlighting = { + "UID1": { + "title": ["highlighted_title1"], + "description": ["highlighted_description1"], + }, + "UID2": { + "title": ["highlighted_title2"], + "description": ["highlighted_description2"], + }, + } + utils.enhance_items(items, highlighting) + assert items == [ + { + "UID": "UID1", + "field_a": "value_a1", + "field_b": "value_b1", + "highlight_title": ["highlighted_title1"], + "highlight_description": ["highlighted_description1"], + }, + { + "UID": "UID2", + "field_a": "value_a2", + "field_b": "value_b2", + "highlight_title": ["highlighted_title2"], + "highlight_description": ["highlighted_description2"], + }, + ] + + +class TestSolrHighlightingUtilsEmpty: + @pytest.fixture + def solr_config(self): + return {} + + def test_empty(self, solr_config): + utils = SolrHighlightingUtils(MockSolrConfig(solr_config)) + assert utils.fields == [] + assert not utils.enabled + assert utils.propByField == {} + + def test_enhance_items_empty(self, solr_config): + utils = SolrHighlightingUtils(MockSolrConfig(solr_config)) + items = [ + { + "UID": "UID1", + "field_a": "value_a1", + "field_b": "value_b1", + }, + { + "UID": "UID2", + "field_a": "value_a2", + "field_b": "value_b2", + }, + ] + highlighting = {} + utils.enhance_items(items, highlighting) + assert items == [ + { + "UID": "UID1", + "field_a": "value_a1", + "field_b": "value_b1", + }, + { + "UID": "UID2", + "field_a": "value_a2", + "field_b": "value_b2", + }, + ]