Migrate fetchers to Search.g4 ANTLR parser. (#13691)

* pr: migrate fetchers to Search.g4 ANTLR parser.

-Changed inherited fetcher classes to use ANTLR generated classes instead of lucene libraries.
- Changed ACMPortalFetcher.java logic for transforming the parsed syntax to URL

* pr: migrate fetchers to Search.g4 ANTLR parser.

- Changed ACMPortalFetcherTest unit test code to use Search.g4 generated classes instead of Lucene
- Removed trivial comment from ACMPortalFetcher

* pr: migrate fetchers to Search.g4 ANTLR parser.

- Changed AbstractQueryTransformer methods to obey Search.g4 parser rules
- Modified ACMPortalFetcher to use the changed transformer logic

* Added a new interface for search nodes in order to parse operators correctly

* Added 2 new node types and changed SearchQueryNode to implement the new interface

* Created a new visitor class for parsing the search syntax and modified the search based fetcher classes to use it

* Updated AbstractQueryTransformer to be more in line with the older code while still being compatible with Search.g4 parser

* Updated ACMPortalFetcher to use the new parser and node logic

* Updated ArxivPortalFetcher to use the new parser and node logic

* Updated BvbFetcher and BvbFetcherTest to use the new parser and node logic

* Updated DBLPFetcher and DBLPQueryTransformerTest to use the new parser and node logic

* Updated DOABFetcher and DOABFetcherTest to use the new parser and node logic

* Updated DOAJFetcher to use the new parser and node logic

* Updated GoogleScholar and SearchBasedFetcherCapabilityTest to use the new parser and node logic

* Updated GvkFetcher GvkFetcherTest and GVKQueryTransformerTest to use the new Search.g4 logic. Fixed how AbstractQueryTransformer handles fields.

* Updated IEEE and IEEEQueryTransformerTest to use Search.g4 based parser

* Updated InfixTransformerTest to use Search.g4 based parser.

* Updated INSPIREFetcher to use Search.g4 based parser.

* Updated ISIDOREFetcher and ISIDOREFetcherTest to use Search.g4 based parser

* Updated JstorFetcher to use Search.g4 based fetcher

* Updated LOBIDFetcher to use Search.g4 based fetcher

* Updated MathSciNet to use Search.g4 based fetcher

* Changed general exceptions to more specific ones according to feedback

* Removed null pointer exception

* Removed null pointer exception

* Updated ScholarArchiveFetcher to use the ANTLR parser

* Updated SpringerFetcher to use the ANTLR parser

* Updated rest of the classes needed for compiling

* Updated ANTLR parser grammar and updated the transformer testers to be compatible with ANTLR-based parser logic.

* Updated ZbMathQueryTransformerTest to throw the appropriate exception

* IEEEQueryTransformerTest wont check for the case where the string is just "and" as this is not valid syntax and throws an exception when parsing

* SearchBasedFetcher performSearch function now throws fetcher exception if query is not valid

* Changed description for function getURLForQuery

* Changed how "not comparison expressions" are handled

* Added the change to CHANGELOG.md

* Update CHANGELOG.md

* Reword

---------

Co-authored-by: Carl Christian Snethlage <[email protected]>

Author: turhantolgaunal

CHANGELOG.md

@@ -78,6 +78,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - We introduced walkthrough functionality [#12664](https://github.com/JabRef/jabref/issues/12664)
 - The Welcome tab now has a responsive layout. [#12664](https://github.com/JabRef/jabref/issues/12664)
 - We introduced a donation prompt in the Welcome tab. [#12664](https://github.com/JabRef/jabref/issues/12664)
+- We changed to syntax for the websearch to the one of the main search bar. [#13607](https://github.com/JabRef/jabref/issues/13607)
 
 ### Fixed
 

jablib/src/main/antlr/org/jabref/search/Search.g4

@@ -39,7 +39,7 @@ CONTAINS: 'CONTAINS';
 MATCHES: 'MATCHES';
 NOT: 'NOT';
 
-FIELD: [A-Z]+;
+FIELD: [A-Z]([A-Z] | '-' | '_')*;           // field name should allow for - or _
 STRING_LITERAL: '"' ('\\"' | ~["])* '"';    // " should be escaped with a backslash
 TERM: ('\\' [=!~()] | ~[ \t\n\r=!~()])+;    // =!~() should be escaped with a backslash
 

jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java

@@ -3,22 +3,22 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.jabref.logic.search.query.SearchQueryVisitor;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.paging.Page;
+import org.jabref.model.search.query.BaseQueryNode;
+import org.jabref.model.search.query.SearchQuery;
 
-import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException;
-import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
-import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser;
-import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser;
+import org.antlr.v4.runtime.misc.ParseCancellationException;
 
 public interface PagedSearchBasedFetcher extends SearchBasedFetcher {
 
     /**
-     * @param luceneQuery the root node of the lucene query
+     * @param queryNode first search node
      * @param pageNumber       requested site number indexed from 0
      * @return Page with search results
      */
-    Page<BibEntry> performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException;
+    Page<BibEntry> performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException;
 
     /**
      * @param searchQuery query string that can be parsed into a lucene query
@@ -29,12 +29,12 @@ default Page<BibEntry> performSearchPaged(String searchQuery, int pageNumber) th
         if (searchQuery.isBlank()) {
             return new Page<>(searchQuery, pageNumber, List.of());
         }
-        SyntaxParser parser = new StandardSyntaxParser();
-        final String NO_EXPLICIT_FIELD = "default";
+        SearchQuery searchQueryObject = new SearchQuery(searchQuery);
+        SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags());
         try {
-            return this.performSearchPaged(parser.parse(searchQuery, NO_EXPLICIT_FIELD), pageNumber);
-        } catch (QueryNodeParseException e) {
-            throw new FetcherException("An error occurred during parsing of the query.");
+            return this.performSearchPaged(visitor.visitStart(searchQueryObject.getContext()), pageNumber);
+        } catch (ParseCancellationException e) {
+            throw new FetcherException("A syntax error occurred during parsing of the query");
         }
     }
 
@@ -48,11 +48,11 @@ default int getPageSize() {
     /**
      * This method is used to send complex queries using fielded search.
      *
-     * @param luceneQuery the root node of the lucene query
+     * @param queryNode the first search node
      * @return a list of {@link BibEntry}, which are matched by the query (may be empty)
      */
     @Override
-    default List<BibEntry> performSearch(QueryNode luceneQuery) throws FetcherException {
-        return new ArrayList<>(performSearchPaged(luceneQuery, 0).getContent());
+    default List<BibEntry> performSearch(BaseQueryNode queryNode) throws FetcherException {
+        return new ArrayList<>(performSearchPaged(queryNode, 0).getContent());
     }
 }

jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java

@@ -9,21 +9,20 @@
 
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.paging.Page;
-
-import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
+import org.jabref.model.search.query.BaseQueryNode;
 
 public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher, ParserFetcher {
 
     @Override
-    default Page<BibEntry> performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException {
+    default Page<BibEntry> performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException {
         // ADR-0014
         URL urlForQuery;
         try {
-            urlForQuery = getURLForQuery(luceneQuery, pageNumber);
+            urlForQuery = getURLForQuery(queryNode, pageNumber);
         } catch (URISyntaxException | MalformedURLException e) {
             throw new FetcherException("Search URI crafted from complex search query is malformed", e);
         }
-        return new Page<>(luceneQuery.toString(), pageNumber, getBibEntries(urlForQuery));
+        return new Page<>(queryNode.toString(), pageNumber, getBibEntries(urlForQuery));
     }
 
     private List<BibEntry> getBibEntries(URL urlForQuery) throws FetcherException {
@@ -41,18 +40,18 @@ private List<BibEntry> getBibEntries(URL urlForQuery) throws FetcherException {
     /**
      * Constructs a URL based on the query, size and page number.
      *
-     * @param luceneQuery the search query
+     * @param queryNode the first search node
      * @param pageNumber  the number of the page indexed from 0
      */
-    URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException;
+    URL getURLForQuery(BaseQueryNode queryNode, int pageNumber) throws URISyntaxException, MalformedURLException;
 
     @Override
-    default URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException {
-        return getURLForQuery(luceneQuery, 0);
+    default URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException {
+        return getURLForQuery(queryNode, 0);
     }
 
     @Override
-    default List<BibEntry> performSearch(QueryNode luceneQuery) throws FetcherException {
-        return SearchBasedParserFetcher.super.performSearch(luceneQuery);
+    default List<BibEntry> performSearch(BaseQueryNode queryNode) throws FetcherException {
+        return SearchBasedParserFetcher.super.performSearch(queryNode);
     }
 }

jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java

@@ -2,14 +2,12 @@
 
 import java.util.List;
 
+import org.jabref.logic.search.query.SearchQueryVisitor;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.search.query.BaseQueryNode;
+import org.jabref.model.search.query.SearchQuery;
 
-import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException;
-import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
-import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser;
-import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser;
-
-import static org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer.NO_EXPLICIT_FIELD;
+import org.antlr.v4.runtime.misc.ParseCancellationException;
 
 /**
  * Searches web resources for bibliographic information based on a free-text query.
@@ -23,28 +21,32 @@ public interface SearchBasedFetcher extends WebFetcher {
     /**
      * This method is used to send complex queries using fielded search.
      *
-     * @param luceneQuery the root node of the lucene query
+     * @param queryList the list that contains the parsed nodes
      * @return a list of {@link BibEntry}, which are matched by the query (may be empty)
      */
-    List<BibEntry> performSearch(QueryNode luceneQuery) throws FetcherException;
+    List<BibEntry> performSearch(BaseQueryNode queryList) throws FetcherException;
 
     /**
      * Looks for hits which are matched by the given free-text query.
      *
-     * @param searchQuery query string that can be parsed into a lucene query
+     * @param searchQuery query string that can be parsed into a search query
      * @return a list of {@link BibEntry}, which are matched by the query (may be empty)
      */
     default List<BibEntry> performSearch(String searchQuery) throws FetcherException {
         if (searchQuery.isBlank()) {
             return List.of();
         }
 
-        SyntaxParser parser = new StandardSyntaxParser();
-        QueryNode queryNode;
+        SearchQuery searchQueryObject = new SearchQuery(searchQuery);
+        if (!searchQueryObject.isValid()) {
+           throw new FetcherException("The query is not valid");
+        }
+        BaseQueryNode queryNode;
+        SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags());
         try {
-            queryNode = parser.parse(searchQuery, NO_EXPLICIT_FIELD);
-        } catch (QueryNodeParseException e) {
-            throw new FetcherException("An error occurred when parsing the query");
+            queryNode = visitor.visitStart(searchQueryObject.getContext());
+        } catch (ParseCancellationException e) {
+            throw new FetcherException("A syntax error occurred during parsing of the query");
         }
 
         return this.performSearch(queryNode);

jablib/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java

@@ -8,8 +8,7 @@
 import java.util.List;
 
 import org.jabref.model.entry.BibEntry;
-
-import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
+import org.jabref.model.search.query.BaseQueryNode;
 
 /**
  * Provides a convenient interface for search-based fetcher, which follows the usual three-step procedure:
@@ -40,14 +39,14 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher, ParserFetc
      * This method is necessary as the performSearch method does not support certain URL parameters that are used for
      * fielded search, such as a title, author, or year parameter.
      *
-     * @param luceneQuery the root node of the lucene query
+     * @param queryNode the first search node
      */
     @Override
-    default List<BibEntry> performSearch(QueryNode luceneQuery) throws FetcherException {
+    default List<BibEntry> performSearch(BaseQueryNode queryNode) throws FetcherException {
         // ADR-0014
         URL urlForQuery;
         try {
-            urlForQuery = getURLForQuery(luceneQuery);
+            urlForQuery = getURLForQuery(queryNode);
         } catch (URISyntaxException | MalformedURLException | FetcherException e) {
             throw new FetcherException("Search URI crafted from complex search query is malformed", e);
         }
@@ -76,7 +75,7 @@ private List<BibEntry> getBibEntries(URL urlForQuery) throws FetcherException {
     /**
      * Constructs a URL based on the lucene query.
      *
-     * @param luceneQuery the root node of the lucene query
+     * @param queryList the list that contains the parsed nodes
      */
-    URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException;
+    URL getURLForQuery(BaseQueryNode queryList) throws URISyntaxException, MalformedURLException, FetcherException;
 }

jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java

@@ -12,9 +12,9 @@
 import org.jabref.logic.importer.SearchBasedParserFetcher;
 import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer;
 import org.jabref.logic.importer.fileformat.ACMPortalParser;
+import org.jabref.model.search.query.BaseQueryNode;
 
 import org.apache.hc.core5.net.URIBuilder;
-import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
 
 public class ACMPortalFetcher implements SearchBasedParserFetcher {
 
@@ -37,20 +37,20 @@ public Optional<HelpFile> getHelpPage() {
         return Optional.of(HelpFile.FETCHER_ACM);
     }
 
-    private static String createQueryString(QueryNode query) {
-        return new DefaultQueryTransformer().transformLuceneQuery(query).orElse("");
+    private static String createQueryString(BaseQueryNode queryNode) {
+        return new DefaultQueryTransformer().transformSearchQuery(queryNode).orElse("");
     }
 
     /**
      * Constructing the url for the searchpage.
      *
-     * @param query query node
+     * @param queryNode the first query node
      * @return query URL
      */
     @Override
-    public URL getURLForQuery(QueryNode query) throws URISyntaxException, MalformedURLException {
+    public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException {
         URIBuilder uriBuilder = new URIBuilder(SEARCH_URL);
-        uriBuilder.addParameter("AllField", createQueryString(query));
+        uriBuilder.addParameter("AllField", createQueryString(queryNode));
         return uriBuilder.build().toURL();
     }
 

jablib/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java

@@ -45,11 +45,11 @@
 import org.jabref.model.entry.identifier.DOI;
 import org.jabref.model.entry.types.StandardEntryType;
 import org.jabref.model.paging.Page;
+import org.jabref.model.search.query.BaseQueryNode;
 import org.jabref.model.strings.StringUtil;
 import org.jabref.model.util.OptionalUtil;
 
 import org.apache.hc.core5.net.URIBuilder;
-import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
@@ -332,13 +332,13 @@ private void inplaceAsyncInfuseArXivWithDoi(CompletableFuture<Optional<BibEntry>
      * Constructs a complex query string using the field prefixes specified at https://arxiv.org/help/api/user-manual
      * and modify resulting BibEntries with additional info from the ArXiv-issued DOI
      *
-     * @param luceneQuery the root node of the lucene query
+     * @param queryNode the first search query node
      * @return A list of entries matching the complex query
      */
     @Override
-    public Page<BibEntry> performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException {
+    public Page<BibEntry> performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException {
 
-        Page<BibEntry> result = arXiv.performSearchPaged(luceneQuery, pageNumber);
+        Page<BibEntry> result = arXiv.performSearchPaged(queryNode, pageNumber);
         if (this.doiFetcher == null) {
             return result;
         }
@@ -604,13 +604,13 @@ public Optional<HelpFile> getHelpPage() {
         /**
          * Constructs a complex query string using the field prefixes specified at https://arxiv.org/help/api/user-manual
          *
-         * @param luceneQuery the root node of the lucene query
+         * @param queryNode the first search node
          * @return A list of entries matching the complex query
          */
         @Override
-        public Page<BibEntry> performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException {
+        public Page<BibEntry> performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException {
             ArXivQueryTransformer transformer = new ArXivQueryTransformer();
-            String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse("");
+            String transformedQuery = transformer.transformSearchQuery(queryNode).orElse("");
             List<BibEntry> searchResult = searchForEntries(transformedQuery, pageNumber)
                     .stream()
                     .map(arXivEntry -> arXivEntry.toBibEntry(importFormatPreferences.bibEntryPreferences().getKeywordSeparator()))

jablib/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java

@@ -33,13 +33,13 @@
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.entry.field.UnknownField;
 import org.jabref.model.paging.Page;
+import org.jabref.model.search.query.BaseQueryNode;
 import org.jabref.model.strings.StringUtil;
 
 import kong.unirest.core.json.JSONArray;
 import kong.unirest.core.json.JSONException;
 import kong.unirest.core.json.JSONObject;
 import org.apache.hc.core5.net.URIBuilder;
-import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
 
 /**
  * Fetches data from the SAO/NASA Astrophysics Data System (<a href="https://ui.adsabs.harvard.edu/">https://ui.adsabs.harvard.edu/</a>)
@@ -81,13 +81,14 @@ public String getName() {
     }
 
     /**
-     * @param luceneQuery query string, matching the apache solr format
+     * @param queryList the list that contains the parsed nodes
      * @return URL which points to a search request for given query
      */
     @Override
-    public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException {
+    public URL getURLForQuery(BaseQueryNode queryList, int pageNumber) throws URISyntaxException, MalformedURLException {
         URIBuilder builder = new URIBuilder(API_SEARCH_URL);
-        builder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse(""));
+        builder.addParameter("q", new DefaultQueryTransformer().transformSearchQuery(queryList).orElse(""));
+        builder.addParameter("q", "");
         builder.addParameter("fl", "bibcode");
         builder.addParameter("rows", String.valueOf(getPageSize()));
         builder.addParameter("start", String.valueOf(getPageSize() * pageNumber));
@@ -278,10 +279,10 @@ private List<BibEntry> performSearchByIds(Collection<String> identifiers) throws
     }
 
     @Override
-    public List<BibEntry> performSearch(QueryNode luceneQuery) throws FetcherException {
+    public List<BibEntry> performSearch(BaseQueryNode queryList) throws FetcherException {
         URL urlForQuery;
         try {
-            urlForQuery = getURLForQuery(luceneQuery);
+            urlForQuery = getURLForQuery(queryList);
         } catch (URISyntaxException | MalformedURLException e) {
             throw new FetcherException("Search URI is malformed", e);
         }
@@ -290,17 +291,17 @@ public List<BibEntry> performSearch(QueryNode luceneQuery) throws FetcherExcepti
     }
 
     @Override
-    public Page<BibEntry> performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException {
+    public Page<BibEntry> performSearchPaged(BaseQueryNode queryList, int pageNumber) throws FetcherException {
         URL urlForQuery;
         try {
-            urlForQuery = getURLForQuery(luceneQuery, pageNumber);
+            urlForQuery = getURLForQuery(queryList, pageNumber);
         } catch (URISyntaxException | MalformedURLException e) {
             throw new FetcherException("Search URI is malformed", e);
         }
         // This is currently just interpreting the complex query as a default string query
         List<String> bibCodes = fetchBibcodes(urlForQuery);
         Collection<BibEntry> results = performSearchByIds(bibCodes);
-        return new Page<>(luceneQuery.toString(), pageNumber, results);
+        return new Page<>(queryList.toString(), pageNumber, results);
     }
 
     @Override