Database for queries and filters (#35)

geritwagner · web-flow · commit 235d6cf959fa · 2025-05-12T13:42:31.000+02:00
* extract translator

* add cli:translate

* rename: syntax &gt; platform

* copy objects etc

* revise serializers

* introduce serializer class

* add serializer_base

* serializer class

* minor issues

* refactor serializer

* revise tests

* update

* extract translator

* format

* refactor translators/constants modules

* revisions

* exclude docs skeletons from linters

* rename function

* revise generic_search_field_to_syntax_field

* update docs

* wos: handling unbalanced quotes

* initial code

* update import_lib files

* add importlib dependency for python3.8

* fix format

* revise

* update docs

* fix typing infos
diff --git a/docs/source/load.rst b/docs/source/load.rst
@@ -3,6 +3,11 @@
 Load
 ====================
 
+Queries can be loaded from strings/files, defined as objects, or retrieved from the internal database.
+
+String/File
+-------------------------
+
 Search-query can parse queries from strings and JSON query files.
 To load a JSON query file, run the parser:
 
@@ -28,11 +33,57 @@ JSON files in the standard format (Haddaway et al. 2022). Example:
       "search_string": "TS=(quantum AND dot AND spin)"
    }
 
+
+Query objects
+-------------------------
+
 Query objects can also be created programmatically.
 
-Filters (TODO)
+.. code-block:: python
+
+   from search_query import OrQuery, AndQuery
+
+   # Typical building-blocks approach
+   digital_synonyms = OrQuery(["digital", "virtual", "online"], search_field="Abstract")
+   work_synonyms = OrQuery(["work", "labor", "service"], search_field="Abstract")
+   query = AndQuery([digital_synonyms, work_synonyms], search_field="Author Keywords")
+
+Database
 ---------------------
 
+.. code-block:: python
+
+   from search-query import database
+
+   query = database.load_query("journals_FT50")
+
+
+
+.. code-block:: python
+
+   from search_query.database import FT50, clinical_trials
+
+   print(FT50)
+   > OR[issn=1234, issn=5678, JN="MIS Quartery", ...]
+
+   print(clinical_trials)
+   > OR[title=rct, title="clinical trial", title="randomized controlled trial", title="experiment", ...]
+
+   # Combination with custom query blocks
+   custom_block = ORQuery(....)
+   full_query = ANDQuery(custom_block, clinical_trials, FT50)
+
+In addition, the ``database_queries`` offer direct programmatic access to full queries and filters:
+
+.. code-block:: python
+
+   from search_query.database_queries import FT50
+
+   print(FT50)
+
+
+Links:
+
 - `search blocks <https://blocks.bmi-online.nl/>`_ are available under a creative-commons license
 - `overview_1 <https://sites.google.com/york.ac.uk/sureinfo/home/search-filters>`_
 - `overview_2 <https://sites.google.com/a/york.ac.uk/issg-search-filters-resource/home/https-sites-google-com-a-york-ac-uk-issg-search-filters-resource-collections-of-search-filters>`_
diff --git a/pyproject.toml b/pyproject.toml
@@ -26,6 +26,9 @@ classifiers = [
 ]
 include = ["LICENSE", "README.md"]
 requires-python = ">=3.8"
+dependencies = [
+  'importlib-resources>=5.1; python_version < "3.9"'
+]
 
 [project.optional-dependencies]
 docs = [
diff --git a/search_query/database.py b/search_query/database.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Database and filters."""
+import typing
+
+try:
+    from importlib.resources import files  # Python 3.9+
+except ImportError:
+    from importlib_resources import files  # pip install importlib_resources
+
+from search_query.parser import parse
+from search_query.search_file import load_search_file
+
+if typing.TYPE_CHECKING:
+    from search_query.query import Query
+
+# mypy: disable-error-code=attr-defined
+
+
+def load_query(name: str) -> "Query":
+    """Load a query object from JSON by name."""
+    try:
+        name = name.replace(".json", "")
+        json_path = files("search_query") / f"json_db/{name}.json"  # mypy: ignore
+        # print(json_path)
+        search = load_search_file(json_path)
+        query = parse(search.search_string, platform=search.platform)
+        # print(query.to_structured_string())
+        return query
+    except FileNotFoundError as exc:
+        raise KeyError(f"No query file named {name}.json found") from exc
+
+
+def list_queries() -> typing.List[str]:
+    """List all available predefined query identifiers (without .json)."""
+
+    # TODO : also give details (dictionary?)
+
+    json_dir = files("search_query") / "json_db"
+    return [
+        file.name.replace(".json", "")
+        for file in json_dir.iterdir()
+        if file.suffix == ".json"
+    ]
+
+
+# # TODO : offer an alternative load_search_file() function
+# (which gives users access to more information)
diff --git a/search_query/database_queries.py b/search_query/database_queries.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+"""Database and filters."""
+from search_query.database import load_query
+
+FT50 = load_query("journals_FT50")
+AIS_8 = load_query("ais_senior_scholars_basket")
+AIS_11 = load_query("ais_senior_scholars_list_of_premier_journals")
+
+__all__ = ["FT50", "AIS_8", "AIS_11"]
diff --git a/search_query/json_db/ais_senior_scholars_basket.json b/search_query/json_db/ais_senior_scholars_basket.json
@@ -0,0 +1,13 @@
+{
+   "record_info": {},
+   "authors": [{"name": "Gerit Wagner"}],
+   "date": {"data_entry": "2025-05-12"},
+   "platform": "wos",
+   "type": "filter",
+   "database": [],
+   "search_string": "SO=(\"European Journal of Information Systems\" OR \"Information Systems Journal\" OR \"Information Systems Research\" OR \"Journal of the Association for Information Systems\" OR \"Journal of Information Technology\" OR \"Journal of Management Information Systems\" OR \"Journal of Strategic Information Systems\" OR \"MIS Quarterly\") OR IS=(0960-085X OR 1476-9344 OR 1350-1917 OR 1365-2575 OR 1047-7047 OR 1526-5536 OR 1536-9323 OR 0268-3962 OR 1466-4437 OR 0742-1222 OR 1557-928X OR 0963-8687 OR 1873-1198 OR 0276-7783 OR 2162-9730)",
+   "title": "Search filter: AIS Senior Scholars Basket",
+   "description": "Search filter for the AIS Senior Scholars Basket of Journals (eight)",
+   "keywords": "AIS, journals, information systems",
+   "license": "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License."
+}
diff --git a/search_query/json_db/ais_senior_scholars_list_of_premier_journals.json b/search_query/json_db/ais_senior_scholars_list_of_premier_journals.json
@@ -0,0 +1,13 @@
+{
+   "record_info": {},
+   "authors": [{"name": "Gerit Wagner"}],
+   "date": {"data_entry": "2025-05-12"},
+   "platform": "wos",
+   "type": "filter",
+   "database": [],
+   "search_string": "SO=(\"European Journal of Information Systems\" OR \"Information Systems Journal\" OR \"Information Systems Research\" OR \"Journal of the Association for Information Systems\" OR \"Journal of Information Technology\" OR \"Journal of Management Information Systems\" OR \"Journal of Strategic Information Systems\" OR \"MIS Quarterly\" OR \"Decision Support Systems\" OR \"Information & Management\" OR \"Information and Organization\") OR IS=(0960-085X OR 1476-9344 OR 1350-1917 OR 1365-2575 OR 1047-7047 OR 1526-5536 OR 1536-9323 OR 0268-3962 OR 1466-4437 OR 0742-1222 OR 1557-928X OR 0963-8687 OR 1873-1198 OR 0276-7783 OR 2162-9730 OR 0167-9236 OR 1873-5797 OR 0378-7206 OR 1872-7530 OR 1471-7727 OR 1873-7919)",
+   "title": "Search filter: AIS Senior Scholars List of Premier Journals",
+   "description": "Search filter for the AIS Senior Scholars List of Premier Journals (eleven), see https://aisnet.org/page/SeniorScholarListofPremierJournals",
+   "keywords": "AIS, journals, information systems",
+   "license": "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License."
+}
diff --git a/search_query/json_db/blocks_bmi_343.json b/search_query/json_db/blocks_bmi_343.json
@@ -0,0 +1,15 @@
+{
+   "record_info": {
+      "url": "https://blocks.bmi-online.nl/catalog/343"
+   },
+   "authors": [{"name": "Ket JCF"}],
+   "date": {"data_entry": "2019-01-15"},
+   "platform": "PubMed",
+   "type": "filter",
+   "database": [],
+   "search_string": "(treatment*[tiab] AND usual[tiab]) OR (standard[tiab] AND care[tiab]) OR (standard[tiab] AND treatment[tiab]) OR (routine[tiab] AND care[tiab]) OR (usual[tiab] AND medication*[tiab]) OR (usual[tiab] AND care[tiab]) OR tau[tiab] OR waitlist*[tiab] OR wait list*[tiab] OR waiting list*[tiab] OR (waiting[tiab] AND (condition[tiab] OR control[tiab])) OR wlc[tiab] OR (delay*[tiab] AND (start[tiab] OR treatment*[tiab])) OR \"no intervention\"[tiab] OR non treatment*[tiab] OR nontreatment*[tiab] OR (minim*[tiab] AND treatment*[tiab]) OR untreated group*[tiab] OR untreated control*[tiab] OR \"without any treatment\"[tiab] OR (untreated[tiab] AND (patients[tiab] OR participants[tiab] OR subjects[tiab] OR group*[tiab] OR control*[tiab])) OR non intervention*[tiab] OR (\"without any\"[tiab] AND intervention*[tiab]) OR (receiv*[tiab] AND nothing[tiab]) OR \"did not receive\"[tiab] OR standard control[tiab] OR non therap*[tiab] OR nontherap*[tiab] OR nonpsychotherap*[tiab] OR (minim*[tiab] AND therap*[tiab]) OR pseudotherap*[tiab] OR pseudo therap*[tiab] OR (therap*[tiab] AND as usual[tiab]) OR usual therap*[tiab] OR reference group[tiab] OR observation group[tiab] OR (convention*[tiab] AND treatment[tiab]) OR conventional therap*[tiab] OR standard treatment*[tiab] OR (standard[tiab] AND therap*[tiab])",
+   "title": "Search block: Treatment as usual",
+   "description": "original drafts by Sarah Dawson, Cochrane CCDAN group, 21 Jun 2016. TAU-filters may retrieve other comparative studies than RCTs, the comparison here being 'treatment as usual', in stead of placebo or gold standard therapy. See also 'no treatment'. For CINAHL the 'Treatment as usual'-filter is integrated in the RCT-filter.",
+   "keywords": "EBSCO/CINAHL; EBSCO/PsycInfo (PI); EMbase.com (EM); PubMed (PM); Study types; Therapy",
+   "license": "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License."
+}
diff --git a/search_query/json_db/journals_FT50.json b/search_query/json_db/journals_FT50.json
@@ -0,0 +1,13 @@
+{
+   "record_info": {},
+   "authors": [{"name": "Gerit Wagner"}],
+   "date": {"data_entry": "2025-05-12"},
+   "platform": "wos",
+   "type": "filter",
+   "database": [],
+   "search_string": "SO=(\"Academy of Management Journal\" OR \"Academy of Management Review\" OR \"Accounting, Organizations and Society\" OR \"Administrative Science Quarterly\" OR \"American Economic Review\" OR \"Contemporary Accounting Research\" OR \"Econometrica\" OR \"Entrepreneurship Theory and Practice\" OR \"Harvard Business Review\" OR \"Human Relations\" OR \"Human Resource Management\" OR \"Information Systems Research\" OR \"Journal of Accounting and Economics\" OR \"Journal of Accounting Research\" OR \"Journal of Applied Psychology\" OR \"Journal of Business Ethics\" OR \"Journal of Business Venturing\" OR \"Journal of Consumer Psychology\" OR \"Journal of Consumer Research\" OR \"Journal of Finance\" OR \"Journal of Financial and Quantitative Analysis\" OR \"Journal of Financial Economics\" OR \"Journal of International Business Studies\" OR \"Journal of Management\" OR \"Journal of Management Information Systems\" OR \"Journal of Management Studies\" OR \"Journal of Marketing\" OR \"Journal of Marketing Research\" OR \"Journal of Operations Management\" OR \"Journal of Political Economy\" OR \"Journal of the Academy of Marketing Science\" OR \"Management Science\" OR \"Manufacturing and Service Operations Management\" OR \"Marketing Science\" OR \"MIS Quarterly\" OR \"Operations Research\" OR \"Organization Science\" OR \"Organization Studies\" OR \"Organizational Behavior and Human Decision Processes\" OR \"Production and Operations Management\" OR \"Quarterly Journal of Economics\" OR \"Research Policy\" OR \"Review of Accounting Studies\" OR \"Review of Economic Studies\" OR \"Review of Finance\" OR \"Review of Financial Studies\" OR \"Sloan Management Review\" OR \"Strategic Entrepreneurship Journal\" OR \"Strategic Management Journal\" OR \"The Accounting Review\") OR IS=(0001-4273 OR 0363-7425 OR 0361-3682 OR 0001-8392 OR 0002-8282 OR 0823-9150 OR 0012-9682 OR 1042-2587 OR 0017-8012 OR 0018-7267 OR 0090-4848 OR 1047-7047 OR 0165-4101 OR 0021-8456 OR 0021-9010 OR 0167-4544 OR 0883-9026 OR 1057-7408 OR 0093-5301 OR 0022-1082 OR 0022-1090 OR 0304-405X OR 0047-2506 OR 0149-2063 OR 0742-1222 OR 0022-2429 OR 0022-2437 OR 0272-6963 OR 0022-3808 OR 0092-0703 OR 0025-1909 OR 1523-4614 OR 0732-2399 OR 0276-7783 OR 0030-364X OR 1047-7039 OR 0170-8406 OR 0749-5978 OR 1059-1478 OR 0033-5533 OR 0048-7333 OR 1380-6653 OR 0034-6527 OR 1572-3097 OR 0893-9454 OR 0036-8075 OR 1932-4391 OR 0143-2095 OR 0001-4826)",
+   "title": "Search filter: FT50 journals",
+   "description": "Search filter for the FT50 (Financial Times 50) journals. The FT50 list is a list of 50 academic journals in the field of business and management that are considered to be the most prestigious and influential in the world. The list is published annually by the Financial Times and is used as a benchmark for academic research and publishing in the field. More information: https://www.ft.com/content/3405a512-5cbb-11e1-8f1f-00144feabdc0",
+   "keywords": "FT50, journals, business, management",
+   "license": "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License."
+}
diff --git a/search_query/linter_pubmed.py b/search_query/linter_pubmed.py
@@ -460,6 +460,7 @@ def _extract_subqueries(
         if not query.children:
             subqueries[subquery_id].append(query)
 
+    # pylint: disable=duplicate-code
     def check_unsupported_pubmed_search_fields(self) -> None:
         """Check for the correct format of fields."""
 
diff --git a/test/test_database.py b/test/test_database.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""Tests for search query database."""
+from search_query.database import load_query
+from search_query.query import Query
+
+# ruff: noqa: E501
+# flake8: noqa: E501
+
+
+def test_load_query_journals_ft50() -> None:
+    """Test loading the JOURNALS_FT50 query file."""
+    query = load_query("journals_FT50")
+
+    assert isinstance(query, Query)
+    print(query.to_generic_string())
+    assert (
+        query.to_generic_string()
+        == 'OR[OR[SO=]["Academy of Management Journal", "Academy of Management Review", "Accounting, Organizations and Society", "Administrative Science Quarterly", "American Economic Review", "Contemporary Accounting Research", "Econometrica", "Entrepreneurship Theory and Practice", "Harvard Business Review", "Human Relations", "Human Resource Management", "Information Systems Research", "Journal of Accounting and Economics", "Journal of Accounting Research", "Journal of Applied Psychology", "Journal of Business Ethics", "Journal of Business Venturing", "Journal of Consumer Psychology", "Journal of Consumer Research", "Journal of Finance", "Journal of Financial and Quantitative Analysis", "Journal of Financial Economics", "Journal of International Business Studies", "Journal of Management", "Journal of Management Information Systems", "Journal of Management Studies", "Journal of Marketing", "Journal of Marketing Research", "Journal of Operations Management", "Journal of Political Economy", "Journal of the Academy of Marketing Science", "Management Science", "Manufacturing and Service Operations Management", "Marketing Science", "MIS Quarterly", "Operations Research", "Organization Science", "Organization Studies", "Organizational Behavior and Human Decision Processes", "Production and Operations Management", "Quarterly Journal of Economics", "Research Policy", "Review of Accounting Studies", "Review of Economic Studies", "Review of Finance", "Review of Financial Studies", "Sloan Management Review", "Strategic Entrepreneurship Journal", "Strategic Management Journal", "The Accounting Review"], OR[IS=][0001-4273, 0363-7425, 0361-3682, 0001-8392, 0002-8282, 0823-9150, 0012-9682, 1042-2587, 0017-8012, 0018-7267, 0090-4848, 1047-7047, 0165-4101, 0021-8456, 0021-9010, 0167-4544, 0883-9026, 1057-7408, 0093-5301, 0022-1082, 0022-1090, 0304-405X, 0047-2506, 0149-2063, 0742-1222, 0022-2429, 0022-2437, 0272-6963, 0022-3808, 0092-0703, 0025-1909, 1523-4614, 0732-2399, 0276-7783, 0030-364X, 1047-7039, 0170-8406, 0749-5978, 1059-1478, 0033-5533, 0048-7333, 1380-6653, 0034-6527, 1572-3097, 0893-9454, 0036-8075, 1932-4391, 0143-2095, 0001-4826]]'
+    )
diff --git a/test/test_wos.py b/test/test_wos.py
@@ -14,11 +14,8 @@
 from search_query.parser_wos import WOSListParser
 from search_query.parser_wos import WOSParser
 
-# ruff: noqa: E501
-# flake8: noqa: E501
 
 # ruff: noqa: E501
-# pylint: disable=too-many-lines
 # flake8: noqa: E501
 
 

Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,9 @@ classifiers = [`
`26`	`26`	`]`
`27`	`27`	`include = ["LICENSE", "README.md"]`
`28`	`28`	`requires-python = ">=3.8"`
	`29`	`+dependencies = [`
	`30`	`+ 'importlib-resources>=5.1; python_version < "3.9"'`
	`31`	`+]`
`29`	`32`
`30`	`33`	`[project.optional-dependencies]`
`31`	`34`	`docs = [`