From b02fbdec828c7c608275b8a3e9aff3ef6743c2da Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Wed, 13 Aug 2025 21:25:24 +0300 Subject: [PATCH 01/37] pr: migrate fetchers to Search.g4 ANTLR parser. -Changed inherited fetcher classes to use ANTLR generated classes instead of lucene libraries. - Changed ACMPortalFetcher.java logic for transforming the parsed syntax to URL --- .../importer/PagedSearchBasedFetcher.java | 26 +++++++++--------- .../PagedSearchBasedParserFetcher.java | 21 +++++++-------- .../logic/importer/SearchBasedFetcher.java | 27 +++++++++---------- .../importer/SearchBasedParserFetcher.java | 13 +++++---- .../importer/fetcher/ACMPortalFetcher.java | 19 ++++++++----- 5 files changed, 52 insertions(+), 54 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index 207b7e5e766..72edeec52b2 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -3,22 +3,20 @@ import java.util.ArrayList; import java.util.List; +import org.jabref.logic.search.query.SearchQueryExtractorVisitor; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; - -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; +import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchQueryNode; public interface PagedSearchBasedFetcher extends SearchBasedFetcher { /** - * @param luceneQuery the root node of the lucene query + * @param queryList the list that contains the parsed nodes * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException; + Page performSearchPaged(List queryList, int pageNumber) throws FetcherException; /** * @param searchQuery query string that can be parsed into a lucene query @@ -29,11 +27,11 @@ default Page performSearchPaged(String searchQuery, int pageNumber) th if (searchQuery.isBlank()) { return new Page<>(searchQuery, pageNumber, List.of()); } - SyntaxParser parser = new StandardSyntaxParser(); - final String NO_EXPLICIT_FIELD = "default"; + SearchQuery searchQueryObject = new SearchQuery(searchQuery); + SearchQueryExtractorVisitor visitor = new SearchQueryExtractorVisitor(searchQueryObject.getSearchFlags()); try { - return this.performSearchPaged(parser.parse(searchQuery, NO_EXPLICIT_FIELD), pageNumber); - } catch (QueryNodeParseException e) { + return this.performSearchPaged(visitor.visitStart(searchQueryObject.getContext()), pageNumber); + } catch (Exception e) { throw new FetcherException("An error occurred during parsing of the query."); } } @@ -48,11 +46,11 @@ default int getPageSize() { /** * This method is used to send complex queries using fielded search. * - * @param luceneQuery the root node of the lucene query + * @param queryList the list that contains the parsed nodes * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ @Override - default List performSearch(QueryNode luceneQuery) throws FetcherException { - return new ArrayList<>(performSearchPaged(luceneQuery, 0).getContent()); + default List performSearch(List queryList) throws FetcherException { + return new ArrayList<>(performSearchPaged(queryList, 0).getContent()); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index db8bae8ddf0..b03a8f8949b 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -9,21 +9,20 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; - -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.jabref.model.search.query.SearchQueryNode; public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher, ParserFetcher { @Override - default Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + default Page performSearchPaged(List queryList, int pageNumber) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(luceneQuery, pageNumber); + urlForQuery = getURLForQuery(queryList, pageNumber); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } - return new Page<>(luceneQuery.toString(), pageNumber, getBibEntries(urlForQuery)); + return new Page<>(queryList.toString(), pageNumber, getBibEntries(urlForQuery)); } private List getBibEntries(URL urlForQuery) throws FetcherException { @@ -41,18 +40,18 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { /** * Constructs a URL based on the query, size and page number. * - * @param luceneQuery the search query + * @param queryList the list that contains the parsed nodes * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException; + URL getURLForQuery(List queryList, int pageNumber) throws URISyntaxException, MalformedURLException; @Override - default URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { - return getURLForQuery(luceneQuery, 0); + default URL getURLForQuery(List queryList) throws URISyntaxException, MalformedURLException { + return getURLForQuery(queryList, 0); } @Override - default List performSearch(QueryNode luceneQuery) throws FetcherException { - return SearchBasedParserFetcher.super.performSearch(luceneQuery); + default List performSearch(List queryList) throws FetcherException { + return SearchBasedParserFetcher.super.performSearch(queryList); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index c114b35f7f3..5afe99c3779 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -2,14 +2,10 @@ import java.util.List; +import org.jabref.logic.search.query.SearchQueryExtractorVisitor; import org.jabref.model.entry.BibEntry; - -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; - -import static org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer.NO_EXPLICIT_FIELD; +import org.jabref.model.search.query.SearchQuery; +import org.jabref.model.search.query.SearchQueryNode; /** * Searches web resources for bibliographic information based on a free-text query. @@ -23,15 +19,15 @@ public interface SearchBasedFetcher extends WebFetcher { /** * This method is used to send complex queries using fielded search. * - * @param luceneQuery the root node of the lucene query + * @param queryList the list that contains the parsed nodes * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - List performSearch(QueryNode luceneQuery) throws FetcherException; + List performSearch(List queryList) throws FetcherException; /** * Looks for hits which are matched by the given free-text query. * - * @param searchQuery query string that can be parsed into a lucene query + * @param searchQuery query string that can be parsed into a search.g4 query * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ default List performSearch(String searchQuery) throws FetcherException { @@ -39,14 +35,15 @@ default List performSearch(String searchQuery) throws FetcherException return List.of(); } - SyntaxParser parser = new StandardSyntaxParser(); - QueryNode queryNode; + SearchQuery searchQueryObject = new SearchQuery(searchQuery); + SearchQueryExtractorVisitor visitor = new SearchQueryExtractorVisitor(searchQueryObject.getSearchFlags()); + List queryList; try { - queryNode = parser.parse(searchQuery, NO_EXPLICIT_FIELD); - } catch (QueryNodeParseException e) { + queryList = visitor.visitStart(searchQueryObject.getContext()); + } catch (Exception e) { throw new FetcherException("An error occurred when parsing the query"); } - return this.performSearch(queryNode); + return this.performSearch(queryList); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 4df75fb981f..c9f5acea211 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -8,8 +8,7 @@ import java.util.List; import org.jabref.model.entry.BibEntry; - -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.jabref.model.search.query.SearchQueryNode; /** * Provides a convenient interface for search-based fetcher, which follows the usual three-step procedure: @@ -40,14 +39,14 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher, ParserFetc * This method is necessary as the performSearch method does not support certain URL parameters that are used for * fielded search, such as a title, author, or year parameter. * - * @param luceneQuery the root node of the lucene query + * @param queryList the list that contains the parsed nodes */ @Override - default List performSearch(QueryNode luceneQuery) throws FetcherException { + default List performSearch(List queryList) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(luceneQuery); + urlForQuery = getURLForQuery(queryList); } catch (URISyntaxException | MalformedURLException | FetcherException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } @@ -76,7 +75,7 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { /** * Constructs a URL based on the lucene query. * - * @param luceneQuery the root node of the lucene query + * @param queryList the list that contains the parsed nodes */ - URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(List queryList) throws URISyntaxException, MalformedURLException, FetcherException; } diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index 93572f9b041..c20aee0557e 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -5,16 +5,16 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.List; import java.util.Optional; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.ACMPortalParser; +import org.jabref.model.search.query.SearchQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class ACMPortalFetcher implements SearchBasedParserFetcher { @@ -37,20 +37,25 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_ACM); } - private static String createQueryString(QueryNode query) { - return new DefaultQueryTransformer().transformLuceneQuery(query).orElse(""); + private static String createQueryString(List queryList) { + StringBuilder stringBuilder = new StringBuilder(); + for (SearchQueryNode term : queryList) { + stringBuilder.append(term.term()); + stringBuilder.append(" "); // Append a space as a delimiter + } + return stringBuilder.toString().trim(); } /** * Constructing the url for the searchpage. * - * @param query query node + * @param queryList list that contains the parsed nodes * @return query URL */ @Override - public URL getURLForQuery(QueryNode query) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(List queryList) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - uriBuilder.addParameter("AllField", createQueryString(query)); + uriBuilder.addParameter("AllField", createQueryString(queryList)); return uriBuilder.build().toURL(); } From 9892bec82d5f614295fe2becffa42532835949c7 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Thu, 14 Aug 2025 20:32:43 +0300 Subject: [PATCH 02/37] pr: migrate fetchers to Search.g4 ANTLR parser. - Changed ACMPortalFetcherTest unit test code to use Search.g4 generated classes instead of Lucene - Removed trivial comment from ACMPortalFetcher --- .../logic/importer/fetcher/ACMPortalFetcher.java | 2 +- .../importer/fetcher/ACMPortalFetcherTest.java | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index c20aee0557e..dedf0a93ede 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -41,7 +41,7 @@ private static String createQueryString(List queryList) { StringBuilder stringBuilder = new StringBuilder(); for (SearchQueryNode term : queryList) { stringBuilder.append(term.term()); - stringBuilder.append(" "); // Append a space as a delimiter + stringBuilder.append(" "); } return stringBuilder.toString().trim(); } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ACMPortalFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ACMPortalFetcherTest.java index 7876ed5fd52..5038bee9d64 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ACMPortalFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ACMPortalFetcherTest.java @@ -8,19 +8,17 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.fileformat.ACMPortalParser; +import org.jabref.logic.search.query.SearchQueryExtractorVisitor; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.SearchQuery; import org.jabref.support.DisabledOnCIServer; import org.jabref.testutils.category.FetcherTest; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer.NO_EXPLICIT_FIELD; import static org.junit.jupiter.api.Assertions.assertEquals; @FetcherTest @@ -62,10 +60,11 @@ void searchByQueryFindsEntry() throws FetcherException { } @Test - void getURLForQuery() throws MalformedURLException, URISyntaxException, QueryNodeParseException { + void getURLForQuery() throws MalformedURLException, URISyntaxException { String testQuery = "test query url"; - SyntaxParser parser = new StandardSyntaxParser(); - URL url = fetcher.getURLForQuery(parser.parse(testQuery, NO_EXPLICIT_FIELD)); + SearchQuery searchQueryObject = new SearchQuery(testQuery); + SearchQueryExtractorVisitor visitor = new SearchQueryExtractorVisitor(searchQueryObject.getSearchFlags()); + URL url = fetcher.getURLForQuery(visitor.visitStart(searchQueryObject.getContext())); String expected = "https://dl.acm.org/action/doSearch?AllField=test%20query%20url"; assertEquals(expected, url.toString()); } From d8025217c6ba522ea487a43b136a74fca59aacbf Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Thu, 14 Aug 2025 23:42:08 +0300 Subject: [PATCH 03/37] pr: migrate fetchers to Search.g4 ANTLR parser. - Changed AbstractQueryTransformer methods to obey Search.g4 parser rules - Modified ACMPortalFetcher to use the changed transformer logic --- .../importer/fetcher/ACMPortalFetcher.java | 8 +- .../AbstractQueryTransformer.java | 138 ++++++------------ 2 files changed, 45 insertions(+), 101 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index dedf0a93ede..54c1fa04c53 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -11,6 +11,7 @@ import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.ACMPortalParser; import org.jabref.model.search.query.SearchQueryNode; @@ -38,12 +39,7 @@ public Optional getHelpPage() { } private static String createQueryString(List queryList) { - StringBuilder stringBuilder = new StringBuilder(); - for (SearchQueryNode term : queryList) { - stringBuilder.append(term.term()); - stringBuilder.append(" "); - } - return stringBuilder.toString().trim(); + return new DefaultQueryTransformer().transformSearchQuery(queryList).orElse(""); } /** diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java index 0ff6be8399a..afb474c1186 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java @@ -1,17 +1,12 @@ package org.jabref.logic.importer.fetcher.transformers; +import java.util.List; import java.util.Optional; import java.util.StringJoiner; -import java.util.stream.Collectors; +import org.jabref.model.search.query.SearchQueryNode; import org.jabref.model.strings.StringUtil; -import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.GroupQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.ModifierQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,29 +22,6 @@ public abstract class AbstractQueryTransformer { protected int startYear = Integer.MAX_VALUE; protected int endYear = Integer.MIN_VALUE; - /** - * Transforms a and b and c to (a AND b AND c), where - * a, b, and c can be complex expressions. - */ - protected Optional transform(BooleanQueryNode query) { - String delimiter; - if (query instanceof OrQueryNode) { - delimiter = getLogicalOrOperator(); - } else { - // We define the logical AND operator as the default implementation - delimiter = getLogicalAndOperator(); - } - - String result = query.getChildren().stream() - .map(this::transform) - .flatMap(Optional::stream) - .collect(Collectors.joining(delimiter, "(", ")")); - if ("()".equals(result)) { - return Optional.empty(); - } - return Optional.of(result); - } - /** * Returns the logical AND operator used by the library * Note: whitespaces have to be included around the operator @@ -73,56 +45,10 @@ protected Optional transform(BooleanQueryNode query) { */ protected abstract String getLogicalNotOperator(); - private Optional transform(FieldQueryNode query) { - String term = query.getTextAsString(); - switch (query.getFieldAsString()) { - case "author" -> { - return Optional.of(handleAuthor(term)); - } - case "title" -> { - return Optional.of(handleTitle(term)); - } - case "journal" -> { - return Optional.of(handleJournal(term)); - } - case "year" -> { - String s = handleYear(term); - return s.isEmpty() ? Optional.empty() : Optional.of(s); - } - case "year-range" -> { - String s = handleYearRange(term); - return s.isEmpty() ? Optional.empty() : Optional.of(s); - } - case "doi" -> { - String s = handleDoi(term); - return s.isEmpty() ? Optional.empty() : Optional.of(s); - } - case NO_EXPLICIT_FIELD -> { - return handleUnFieldedTerm(term); - } - default -> { - // Just add unknown fields as default - return handleOtherField(query.getFieldAsString(), term); - } - } - } - protected String handleDoi(String term) { return "doi:" + term; } - /** - * Handles the not modifier, all other cases are silently ignored - */ - private Optional transform(ModifierQueryNode query) { - ModifierQueryNode.Modifier modifier = query.getModifier(); - if (modifier == ModifierQueryNode.Modifier.MOD_NOT) { - return transform(query.getChild()).map(s -> getLogicalNotOperator() + s); - } else { - return transform(query.getChild()); - } - } - /** * Return a string representation of the author fielded term */ @@ -202,38 +128,60 @@ protected Optional handleOtherField(String fieldAsString, String term) { return Optional.of(createKeyValuePair(fieldAsString, term)); } - protected Optional transform(QueryNode query) { - switch (query) { - case BooleanQueryNode booleanQueryNode -> { - return transform(booleanQueryNode); - } - case FieldQueryNode fieldQueryNode -> { - return transform(fieldQueryNode); - } - case GroupQueryNode groupQueryNode -> { - return transform(groupQueryNode.getChild()); - } - case ModifierQueryNode modifierQueryNode -> { - return transform(modifierQueryNode); + protected Optional transform(List queryList) { + StringBuilder stringBuilder = new StringBuilder(); + for (SearchQueryNode node : queryList) { + Optional result; + String fieldString = "default"; + if (node.field().isPresent()) { + fieldString = node.field().get().getName(); } - case null, default -> { - LOGGER.error("Unsupported case when transforming the query:\n {}", query); - return Optional.empty(); + switch (fieldString) { + case "author" -> { + result = Optional.of(handleAuthor(node.term())); + } + case "title" -> { + result = Optional.of(handleTitle(node.term())); + } + case "journal" -> { + result = Optional.of(handleJournal(node.term())); + } + case "year" -> { + String s = handleYear(node.term()); + result = s.isEmpty() ? Optional.empty() : Optional.of(s); + } + case "year-range" -> { + String s = handleYearRange(node.term()); + result = s.isEmpty() ? Optional.empty() : Optional.of(s); + } + case "doi" -> { + String s = handleDoi(node.term()); + result = s.isEmpty() ? Optional.empty() : Optional.of(s); + } + case NO_EXPLICIT_FIELD -> { + result = handleUnFieldedTerm(node.term()); + } + default -> { + // Just add unknown fields as default + result = handleOtherField(fieldString, node.term()); + } } + result.ifPresent(s -> stringBuilder.append(s).append(" ")); } + return stringBuilder.toString().trim().describeConstable(); } /** * Parses the given query string into a complex query using lucene. * Note: For unique fields, the alphabetically and numerically first instance in the query string is used in the complex query. * - * @param luceneQuery The lucene query tp transform + * @param queryList The list that contains the parsed nodes * @return A query string containing all fields that are contained in the original lucene query and * that are expressible in the library specific query language, other information either is discarded or * stored as part of the state of the transformer if it can be used e.g. as a URL parameter for the query. */ - public Optional transformLuceneQuery(QueryNode luceneQuery) { - Optional transformedQuery = transform(luceneQuery); + public Optional transformSearchQuery(List queryList) { + Optional transformedQuery = transform(queryList); transformedQuery = transformedQuery.map(this::removeOuterBraces); return transformedQuery; } From a645b384ae85e6e1abb2836ec0ba2a69a32fd37c Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sun, 17 Aug 2025 23:40:44 +0300 Subject: [PATCH 04/37] Added a new interface for search nodes in order to parse operators correctly --- .../java/org/jabref/model/search/query/BaseQueryNode.java | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 jablib/src/main/java/org/jabref/model/search/query/BaseQueryNode.java diff --git a/jablib/src/main/java/org/jabref/model/search/query/BaseQueryNode.java b/jablib/src/main/java/org/jabref/model/search/query/BaseQueryNode.java new file mode 100644 index 00000000000..6bfd5edfd47 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/search/query/BaseQueryNode.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.query; + +public interface BaseQueryNode { +} From 44853c3d973df44c9d73df52ccb37cb80633b504 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sun, 17 Aug 2025 23:41:42 +0300 Subject: [PATCH 05/37] Added 2 new node types and changed SearchQueryNode to implement the new interface --- .../java/org/jabref/model/search/query/NotNode.java | 4 ++++ .../org/jabref/model/search/query/OperatorNode.java | 10 ++++++++++ .../org/jabref/model/search/query/SearchQueryNode.java | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 jablib/src/main/java/org/jabref/model/search/query/NotNode.java create mode 100644 jablib/src/main/java/org/jabref/model/search/query/OperatorNode.java diff --git a/jablib/src/main/java/org/jabref/model/search/query/NotNode.java b/jablib/src/main/java/org/jabref/model/search/query/NotNode.java new file mode 100644 index 00000000000..f820f06df13 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/search/query/NotNode.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.query; + +public record NotNode(BaseQueryNode negatedNode) implements BaseQueryNode { +} diff --git a/jablib/src/main/java/org/jabref/model/search/query/OperatorNode.java b/jablib/src/main/java/org/jabref/model/search/query/OperatorNode.java new file mode 100644 index 00000000000..bc1c8429088 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/search/query/OperatorNode.java @@ -0,0 +1,10 @@ +package org.jabref.model.search.query; + +import java.util.List; + +public record OperatorNode(Operator op, List children) implements BaseQueryNode { + public enum Operator { + AND, + OR + } +} diff --git a/jablib/src/main/java/org/jabref/model/search/query/SearchQueryNode.java b/jablib/src/main/java/org/jabref/model/search/query/SearchQueryNode.java index 5d283110ea3..1ea6864b8ab 100644 --- a/jablib/src/main/java/org/jabref/model/search/query/SearchQueryNode.java +++ b/jablib/src/main/java/org/jabref/model/search/query/SearchQueryNode.java @@ -4,5 +4,5 @@ import org.jabref.model.entry.field.Field; -public record SearchQueryNode(Optional field, String term) { +public record SearchQueryNode(Optional field, String term) implements BaseQueryNode { } From a643d229b6e265c553e41f15d28d2a21ef3953c4 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sun, 17 Aug 2025 23:44:06 +0300 Subject: [PATCH 06/37] Created a new visitor class for parsing the search syntax and modified the search based fetcher classes to use it --- .../importer/PagedSearchBasedFetcher.java | 16 +-- .../PagedSearchBasedParserFetcher.java | 20 +-- .../logic/importer/SearchBasedFetcher.java | 14 +- .../importer/SearchBasedParserFetcher.java | 10 +- .../search/query/SearchQueryVisitor.java | 121 ++++++++++++++++++ .../SearchBasedFetcherCapabilityTest.java | 6 +- 6 files changed, 154 insertions(+), 33 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/search/query/SearchQueryVisitor.java diff --git a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index 72edeec52b2..2b3cfd7b13d 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -3,20 +3,20 @@ import java.util.ArrayList; import java.util.List; -import org.jabref.logic.search.query.SearchQueryExtractorVisitor; +import org.jabref.logic.search.query.SearchQueryVisitor; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.jabref.model.search.query.SearchQueryNode; public interface PagedSearchBasedFetcher extends SearchBasedFetcher { /** - * @param queryList the list that contains the parsed nodes + * @param queryNode first search node * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - Page performSearchPaged(List queryList, int pageNumber) throws FetcherException; + Page performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException; /** * @param searchQuery query string that can be parsed into a lucene query @@ -28,7 +28,7 @@ default Page performSearchPaged(String searchQuery, int pageNumber) th return new Page<>(searchQuery, pageNumber, List.of()); } SearchQuery searchQueryObject = new SearchQuery(searchQuery); - SearchQueryExtractorVisitor visitor = new SearchQueryExtractorVisitor(searchQueryObject.getSearchFlags()); + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); try { return this.performSearchPaged(visitor.visitStart(searchQueryObject.getContext()), pageNumber); } catch (Exception e) { @@ -46,11 +46,11 @@ default int getPageSize() { /** * This method is used to send complex queries using fielded search. * - * @param queryList the list that contains the parsed nodes + * @param queryNode the first search node * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ @Override - default List performSearch(List queryList) throws FetcherException { - return new ArrayList<>(performSearchPaged(queryList, 0).getContent()); + default List performSearch(BaseQueryNode queryNode) throws FetcherException { + return new ArrayList<>(performSearchPaged(queryNode, 0).getContent()); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index b03a8f8949b..61c83f747ad 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -9,20 +9,20 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; -import org.jabref.model.search.query.SearchQueryNode; +import org.jabref.model.search.query.BaseQueryNode; public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher, ParserFetcher { @Override - default Page performSearchPaged(List queryList, int pageNumber) throws FetcherException { + default Page performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(queryList, pageNumber); + urlForQuery = getURLForQuery(queryNode, pageNumber); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } - return new Page<>(queryList.toString(), pageNumber, getBibEntries(urlForQuery)); + return new Page<>(queryNode.toString(), pageNumber, getBibEntries(urlForQuery)); } private List getBibEntries(URL urlForQuery) throws FetcherException { @@ -40,18 +40,18 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { /** * Constructs a URL based on the query, size and page number. * - * @param queryList the list that contains the parsed nodes + * @param queryNode the first search node * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(List queryList, int pageNumber) throws URISyntaxException, MalformedURLException; + URL getURLForQuery(BaseQueryNode queryNode, int pageNumber) throws URISyntaxException, MalformedURLException; @Override - default URL getURLForQuery(List queryList) throws URISyntaxException, MalformedURLException { - return getURLForQuery(queryList, 0); + default URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { + return getURLForQuery(queryNode, 0); } @Override - default List performSearch(List queryList) throws FetcherException { - return SearchBasedParserFetcher.super.performSearch(queryList); + default List performSearch(BaseQueryNode queryNode) throws FetcherException { + return SearchBasedParserFetcher.super.performSearch(queryNode); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index 5afe99c3779..2741d106271 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -2,10 +2,10 @@ import java.util.List; -import org.jabref.logic.search.query.SearchQueryExtractorVisitor; +import org.jabref.logic.search.query.SearchQueryVisitor; import org.jabref.model.entry.BibEntry; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.jabref.model.search.query.SearchQueryNode; /** * Searches web resources for bibliographic information based on a free-text query. @@ -22,7 +22,7 @@ public interface SearchBasedFetcher extends WebFetcher { * @param queryList the list that contains the parsed nodes * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - List performSearch(List queryList) throws FetcherException; + List performSearch(BaseQueryNode queryList) throws FetcherException; /** * Looks for hits which are matched by the given free-text query. @@ -36,14 +36,14 @@ default List performSearch(String searchQuery) throws FetcherException } SearchQuery searchQueryObject = new SearchQuery(searchQuery); - SearchQueryExtractorVisitor visitor = new SearchQueryExtractorVisitor(searchQueryObject.getSearchFlags()); - List queryList; + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); + BaseQueryNode queryNode; try { - queryList = visitor.visitStart(searchQueryObject.getContext()); + queryNode = visitor.visitStart(searchQueryObject.getContext()); } catch (Exception e) { throw new FetcherException("An error occurred when parsing the query"); } - return this.performSearch(queryList); + return this.performSearch(queryNode); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index c9f5acea211..8e8c8f3ed54 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -8,7 +8,7 @@ import java.util.List; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.query.SearchQueryNode; +import org.jabref.model.search.query.BaseQueryNode; /** * Provides a convenient interface for search-based fetcher, which follows the usual three-step procedure: @@ -39,14 +39,14 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher, ParserFetc * This method is necessary as the performSearch method does not support certain URL parameters that are used for * fielded search, such as a title, author, or year parameter. * - * @param queryList the list that contains the parsed nodes + * @param queryNode the first search node */ @Override - default List performSearch(List queryList) throws FetcherException { + default List performSearch(BaseQueryNode queryNode) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(queryList); + urlForQuery = getURLForQuery(queryNode); } catch (URISyntaxException | MalformedURLException | FetcherException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } @@ -77,5 +77,5 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { * * @param queryList the list that contains the parsed nodes */ - URL getURLForQuery(List queryList) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(BaseQueryNode queryList) throws URISyntaxException, MalformedURLException, FetcherException; } diff --git a/jablib/src/main/java/org/jabref/logic/search/query/SearchQueryVisitor.java b/jablib/src/main/java/org/jabref/logic/search/query/SearchQueryVisitor.java new file mode 100644 index 00000000000..529f87e9f36 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/search/query/SearchQueryVisitor.java @@ -0,0 +1,121 @@ +package org.jabref.logic.search.query; + +import java.util.EnumSet; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.jabref.model.entry.field.FieldFactory; +import org.jabref.model.entry.field.InternalField; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.NotNode; +import org.jabref.model.search.query.OperatorNode; +import org.jabref.model.search.query.SearchQueryNode; +import org.jabref.search.SearchBaseVisitor; +import org.jabref.search.SearchParser; + +public class SearchQueryVisitor extends SearchBaseVisitor { + + private final boolean searchBarRegex; + + public SearchQueryVisitor(EnumSet searchFlags) { + searchBarRegex = searchFlags.contains(SearchFlags.REGULAR_EXPRESSION); + } + + @Override + public BaseQueryNode visitStart(SearchParser.StartContext ctx) { + return visit(ctx.andExpression()); + } + + @Override + public BaseQueryNode visitImplicitAndExpression(SearchParser.ImplicitAndExpressionContext ctx) { + List children = ctx.expression().stream() + .map(this::visit) + .collect(Collectors.toList()); + if (children.size() == 1) { + return children.getFirst(); + } + return new OperatorNode(OperatorNode.Operator.AND, children); + } + + @Override + public BaseQueryNode visitNegatedExpression(SearchParser.NegatedExpressionContext ctx) { + BaseQueryNode negated = visit(ctx.expression()); + return new NotNode(negated); + } + + @Override + public BaseQueryNode visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + BaseQueryNode left = visit(ctx.left); + BaseQueryNode right = visit(ctx.right); + + // Check the actual operator token + if (ctx.bin_op.getType() == SearchParser.AND) { + return new OperatorNode(OperatorNode.Operator.AND, List.of(left, right)); + } else { // Assuming the only other binary op is OR + return new OperatorNode(OperatorNode.Operator.OR, List.of(left, right)); + } + } + + @Override + public BaseQueryNode visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return visit(ctx.andExpression()); + } + + @Override + public BaseQueryNode visitComparisonExpression(SearchParser.ComparisonExpressionContext ctx) { + return visit(ctx.comparison()); + } + + @Override + public BaseQueryNode visitComparison(SearchParser.ComparisonContext ctx) { + if (ctx.operator() != null) { + int operator = ctx.operator().getStart().getType(); + if (operator == SearchParser.NEQUAL + || operator == SearchParser.NCEQUAL + || operator == SearchParser.NEEQUAL + || operator == SearchParser.NCEEQUAL + || operator == SearchParser.NREQUAL + || operator == SearchParser.NCREEQUAL) { + return null; + } + } + String term = SearchQueryConversion.unescapeSearchValue(ctx.searchValue()); + + // if not regex, escape the backslashes, because the highlighter uses regex + + // unfielded terms, check the search bar flags + if (ctx.FIELD() == null) { + if (!searchBarRegex) { + term = term.replace("\\", "\\\\"); + } + return new SearchQueryNode(Optional.empty(), term); + } + + String field = ctx.FIELD().getText().toLowerCase(Locale.ROOT); + + // Pseudo-fields + field = switch (field) { + case "key" -> InternalField.KEY_FIELD.getName(); + case "anykeyword" -> StandardField.KEYWORDS.getName(); + case "anyfield" -> "any"; + default -> field; + }; + + if (ctx.operator() != null) { + int operator = ctx.operator().getStart().getType(); + if (operator != SearchParser.REQUAL && operator != SearchParser.CREEQUAL) { + term = term.replace("\\", "\\\\"); + } + } + + if ("any".equals(field)) { + return new SearchQueryNode(Optional.empty(), term); + } + return new SearchQueryNode(Optional.of(FieldFactory.parseField(field)), term); + } +} + diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index f4a35fd3368..a659a1fbbfd 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -103,7 +103,7 @@ default void supportsYearRangeSearch() throws FetcherException { */ @Test default void supportsJournalSearch() throws FetcherException { - List result = getFetcher().performSearch("journal:\"" + getTestJournal() + "\""); + List result = getFetcher().performSearch("journal=\"" + getTestJournal() + "\""); FieldPreferences fieldPreferences = mock(FieldPreferences.class); when(fieldPreferences.getNonWrappableFields()).thenReturn(FXCollections.observableArrayList()); ImportCleanup.targeting(BibDatabaseMode.BIBTEX, fieldPreferences).doPostCleanup(result); @@ -111,8 +111,8 @@ default void supportsJournalSearch() throws FetcherException { assertFalse(result.isEmpty()); result.forEach(bibEntry -> { assertTrue(bibEntry.hasField(StandardField.JOURNAL)); - String journal = bibEntry.getField(StandardField.JOURNAL).orElse(""); - assertTrue(journal.contains(getTestJournal().replace("\"", ""))); + String journal = bibEntry.getField(StandardField.JOURNAL).orElse("").toLowerCase(); + assertTrue(journal.contains(getTestJournal().replace("\"", "").toLowerCase())); }); } From c890e92a116fc5c9dedd72adb6bd84e83dc88f44 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sun, 17 Aug 2025 23:45:24 +0300 Subject: [PATCH 07/37] Updated AbstractQueryTransformer to be more in line with the older code while still being compatible with Search.g4 parser --- .../AbstractQueryTransformer.java | 130 ++++++++++++------ 1 file changed, 88 insertions(+), 42 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java index afb474c1186..0f9c050fcec 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java @@ -1,9 +1,12 @@ package org.jabref.logic.importer.fetcher.transformers; -import java.util.List; import java.util.Optional; import java.util.StringJoiner; +import java.util.stream.Collectors; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.NotNode; +import org.jabref.model.search.query.OperatorNode; import org.jabref.model.search.query.SearchQueryNode; import org.jabref.model.strings.StringUtil; @@ -22,6 +25,29 @@ public abstract class AbstractQueryTransformer { protected int startYear = Integer.MAX_VALUE; protected int endYear = Integer.MIN_VALUE; + /** + * Transforms a and b and c to (a AND b AND c), where + * a, b, and c can be complex expressions. + */ + protected Optional transform(OperatorNode query) { + String delimiter; + if (query.op() == OperatorNode.Operator.OR) { + delimiter = getLogicalOrOperator(); + } else { + // We define the logical AND operator as the default implementation + delimiter = getLogicalAndOperator(); + } + + String result = query.children().stream() + .map(this::transform) + .flatMap(Optional::stream) + .collect(Collectors.joining(delimiter, "(", ")")); + if ("()".equals(result)) { + return Optional.empty(); + } + return Optional.of(result); + } + /** * Returns the logical AND operator used by the library * Note: whitespaces have to be included around the operator @@ -45,10 +71,55 @@ public abstract class AbstractQueryTransformer { */ protected abstract String getLogicalNotOperator(); + private Optional transform(SearchQueryNode query) { + String term = query.term(); + String field = NO_EXPLICIT_FIELD; + if (query.field().isPresent()) { + field = String.valueOf(query.field().get()).toLowerCase(); + } + switch (field) { + case "author" -> { + return Optional.of(handleAuthor(term)); + } + case "title" -> { + return Optional.of(handleTitle(term)); + } + case "journal" -> { + return Optional.of(handleJournal(term)); + } + case "year" -> { + String s = handleYear(term); + return s.isEmpty() ? Optional.empty() : Optional.of(s); + } + case "year-range" -> { + String s = handleYearRange(term); + return s.isEmpty() ? Optional.empty() : Optional.of(s); + } + case "doi" -> { + String s = handleDoi(term); + return s.isEmpty() ? Optional.empty() : Optional.of(s); + } + case NO_EXPLICIT_FIELD -> { + return handleUnFieldedTerm(term); + } + default -> { + // Just add unknown fields as default + return handleOtherField(field, term); + } + } + } + protected String handleDoi(String term) { return "doi:" + term; } + /** + * Handles the not modifier, all other cases are silently ignored + */ + private Optional transform(NotNode query) { + return transform(query.negatedNode()).map(s -> getLogicalNotOperator() + s); + } + /** * Return a string representation of the author fielded term */ @@ -128,60 +199,35 @@ protected Optional handleOtherField(String fieldAsString, String term) { return Optional.of(createKeyValuePair(fieldAsString, term)); } - protected Optional transform(List queryList) { - StringBuilder stringBuilder = new StringBuilder(); - for (SearchQueryNode node : queryList) { - Optional result; - String fieldString = "default"; - if (node.field().isPresent()) { - fieldString = node.field().get().getName(); + protected Optional transform(BaseQueryNode query) { + switch (query) { + case OperatorNode operatorQueryNode -> { + return transform(operatorQueryNode); + } + case SearchQueryNode searchQueryNode -> { + return transform(searchQueryNode); + } + case NotNode notQueryNode -> { + return transform(notQueryNode); } - switch (fieldString) { - case "author" -> { - result = Optional.of(handleAuthor(node.term())); - } - case "title" -> { - result = Optional.of(handleTitle(node.term())); - } - case "journal" -> { - result = Optional.of(handleJournal(node.term())); - } - case "year" -> { - String s = handleYear(node.term()); - result = s.isEmpty() ? Optional.empty() : Optional.of(s); - } - case "year-range" -> { - String s = handleYearRange(node.term()); - result = s.isEmpty() ? Optional.empty() : Optional.of(s); - } - case "doi" -> { - String s = handleDoi(node.term()); - result = s.isEmpty() ? Optional.empty() : Optional.of(s); - } - case NO_EXPLICIT_FIELD -> { - result = handleUnFieldedTerm(node.term()); - } - default -> { - // Just add unknown fields as default - result = handleOtherField(fieldString, node.term()); - } + case null, default -> { + LOGGER.error("Unsupported case when transforming the query:\n {}", query); + return Optional.empty(); } - result.ifPresent(s -> stringBuilder.append(s).append(" ")); } - return stringBuilder.toString().trim().describeConstable(); } /** * Parses the given query string into a complex query using lucene. * Note: For unique fields, the alphabetically and numerically first instance in the query string is used in the complex query. * - * @param queryList The list that contains the parsed nodes + * @param queryNode The first search node * @return A query string containing all fields that are contained in the original lucene query and * that are expressible in the library specific query language, other information either is discarded or * stored as part of the state of the transformer if it can be used e.g. as a URL parameter for the query. */ - public Optional transformSearchQuery(List queryList) { - Optional transformedQuery = transform(queryList); + public Optional transformSearchQuery(BaseQueryNode queryNode) { + Optional transformedQuery = transform(queryNode); transformedQuery = transformedQuery.map(this::removeOuterBraces); return transformedQuery; } From 954e1a4a4bbe68cb9b061abce2fd89561ceac975 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sun, 17 Aug 2025 23:45:51 +0300 Subject: [PATCH 08/37] Updated ACMPortalFetcher to use the new parser and node logic --- .../logic/importer/fetcher/ACMPortalFetcher.java | 13 ++++++------- .../importer/fetcher/ACMPortalFetcherTest.java | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index 54c1fa04c53..310cd51c1d9 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -5,7 +5,6 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.util.List; import java.util.Optional; import org.jabref.logic.help.HelpFile; @@ -13,7 +12,7 @@ import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.ACMPortalParser; -import org.jabref.model.search.query.SearchQueryNode; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; @@ -38,20 +37,20 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_ACM); } - private static String createQueryString(List queryList) { - return new DefaultQueryTransformer().transformSearchQuery(queryList).orElse(""); + private static String createQueryString(BaseQueryNode queryNode) { + return new DefaultQueryTransformer().transformSearchQuery(queryNode).orElse(""); } /** * Constructing the url for the searchpage. * - * @param queryList list that contains the parsed nodes + * @param queryNode the first query node * @return query URL */ @Override - public URL getURLForQuery(List queryList) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - uriBuilder.addParameter("AllField", createQueryString(queryList)); + uriBuilder.addParameter("AllField", createQueryString(queryNode)); return uriBuilder.build().toURL(); } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ACMPortalFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ACMPortalFetcherTest.java index 5038bee9d64..735fee29f99 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ACMPortalFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ACMPortalFetcherTest.java @@ -8,7 +8,7 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.fileformat.ACMPortalParser; -import org.jabref.logic.search.query.SearchQueryExtractorVisitor; +import org.jabref.logic.search.query.SearchQueryVisitor; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; @@ -63,7 +63,7 @@ void searchByQueryFindsEntry() throws FetcherException { void getURLForQuery() throws MalformedURLException, URISyntaxException { String testQuery = "test query url"; SearchQuery searchQueryObject = new SearchQuery(testQuery); - SearchQueryExtractorVisitor visitor = new SearchQueryExtractorVisitor(searchQueryObject.getSearchFlags()); + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); URL url = fetcher.getURLForQuery(visitor.visitStart(searchQueryObject.getContext())); String expected = "https://dl.acm.org/action/doSearch?AllField=test%20query%20url"; assertEquals(expected, url.toString()); From 3977901e44c8e59be6c465e820c0c4c7befcf350 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sun, 17 Aug 2025 23:46:23 +0300 Subject: [PATCH 09/37] Updated ArxivPortalFetcher to use the new parser and node logic --- .../logic/importer/fetcher/ArXivFetcher.java | 14 +++++++------- .../logic/importer/fetcher/ArXivFetcherTest.java | 14 +++++++------- .../transformers/ArXivQueryTransformerTest.java | 11 +++++++---- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java index 1232198748c..2432eb8157e 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ArXivFetcher.java @@ -45,11 +45,11 @@ import org.jabref.model.entry.identifier.DOI; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.paging.Page; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.OptionalUtil; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -332,13 +332,13 @@ private void inplaceAsyncInfuseArXivWithDoi(CompletableFuture * Constructs a complex query string using the field prefixes specified at https://arxiv.org/help/api/user-manual * and modify resulting BibEntries with additional info from the ArXiv-issued DOI * - * @param luceneQuery the root node of the lucene query + * @param queryNode the first search query node * @return A list of entries matching the complex query */ @Override - public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException { - Page result = arXiv.performSearchPaged(luceneQuery, pageNumber); + Page result = arXiv.performSearchPaged(queryNode, pageNumber); if (this.doiFetcher == null) { return result; } @@ -604,13 +604,13 @@ public Optional getHelpPage() { /** * Constructs a complex query string using the field prefixes specified at https://arxiv.org/help/api/user-manual * - * @param luceneQuery the root node of the lucene query + * @param queryNode the first search node * @return A list of entries matching the complex query */ @Override - public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException { ArXivQueryTransformer transformer = new ArXivQueryTransformer(); - String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); + String transformedQuery = transformer.transformSearchQuery(queryNode).orElse(""); List searchResult = searchForEntries(transformedQuery, pageNumber) .stream() .map(arXivEntry -> arXivEntry.toBibEntry(importFormatPreferences.bibEntryPreferences().getKeywordSeparator())) diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ArXivFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ArXivFetcherTest.java index 1eadb0a0b66..426453b58aa 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ArXivFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ArXivFetcherTest.java @@ -152,7 +152,7 @@ public PagedSearchBasedFetcher getPagedFetcher() { @Test @Override public void supportsAuthorSearch() throws FetcherException { - StringJoiner queryBuilder = new StringJoiner("\" AND author:\"", "author:\"", "\""); + StringJoiner queryBuilder = new StringJoiner("\" AND author=\"", "author=\"", "\""); getInputTestAuthors().forEach(queryBuilder::add); List result = getFetcher().performSearch(queryBuilder.toString()); @@ -300,13 +300,13 @@ void findFullTextTrustLevel() { @Test void searchEntryByPartOfTitle() throws FetcherException { assertEquals(List.of(mainResultPaper), - fetcher.performSearch("title:\"the architecture of mr. dLib's\"")); + fetcher.performSearch("title=\"the architecture of mr. dLib's\"")); } @Test void searchEntryByPartOfTitleWithAcuteAccent() throws FetcherException { assertEquals(List.of(sliceTheoremPaper), - fetcher.performSearch("title:\"slice theorem for Fréchet\"")); + fetcher.performSearch("title=\"slice theorem for Fréchet\"")); } @Test @@ -441,10 +441,10 @@ void supportsPhraseSearchAndMatchesExact() throws FetcherException { .withField(InternalField.KEY_FIELD, "https://doi.org/10.48550/arxiv.2009.10618") .withField(new UnknownField("copyright"), "arXiv.org perpetual, non-exclusive license"); - List resultWithPhraseSearch = fetcher.performSearch("title:\"Taxonomy of Distributed\""); + List resultWithPhraseSearch = fetcher.performSearch("title==\"A Survey and Taxonomy of Distributed Data Mining Research Studies: A Systematic Literature Review\""); - // There is only a single paper found by searching that contains the exact sequence "Taxonomy of Distributed" in the title. - assertEquals(List.of(expected), resultWithPhraseSearch); + // The first result should be the expected paper + assertEquals(expected, resultWithPhraseSearch.getFirst()); } @Test @@ -472,7 +472,7 @@ void supportsBooleanANDSearch() throws FetcherException { .withField(InternalField.KEY_FIELD, "B_scher_2020") .withField(new UnknownField("copyright"), "arXiv.org perpetual, non-exclusive license"); - List result = fetcher.performSearch("author:\"Tobias Büscher\" AND title:\"Instability and fingering of interfaces\""); + List result = fetcher.performSearch("author=\"Tobias Büscher\" AND title=\"Instability and fingering of interfaces\""); // There is only one paper authored by Tobias Büscher with that phrase in the title assertEquals(List.of(expected), result); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformerTest.java index 04487f81d3a..a20280a2bd5 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -41,8 +43,9 @@ public String getTitlePrefix() { public void convertYearField() throws QueryNodeParseException { ArXivQueryTransformer transformer = getTransformer(); String queryString = "year:2018"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional query = transformer.transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); assertEquals(Optional.of("2018"), query); assertEquals(Optional.of(2018), transformer.getStartYear()); assertEquals(Optional.of(2018), transformer.getEndYear()); From 60d203451c6daa025b48fdf8a7f00975e0fa8971 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sun, 17 Aug 2025 23:59:27 +0300 Subject: [PATCH 10/37] Updated BvbFetcher and BvbFetcherTest to use the new parser and node logic --- .../logic/importer/fetcher/BvbFetcher.java | 6 ++--- .../importer/fetcher/BvbFetcherTest.java | 24 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/BvbFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/BvbFetcher.java index a53803299f7..2cabc1c331f 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/BvbFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/BvbFetcher.java @@ -10,9 +10,9 @@ import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.MarcXmlParser; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /// Fetcher for [Bibliotheksverbund Bayern (BVB)](https://www.bib-bvb.de/) public class BvbFetcher implements SearchBasedParserFetcher { @@ -30,12 +30,12 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode query) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode query) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); uriBuilder.addParameter("recordSchema", "marcxml"); uriBuilder.addParameter("operation", "searchRetrieve"); - uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(query).orElse("")); + uriBuilder.addParameter("query", new DefaultQueryTransformer().transformSearchQuery(query).orElse("")); uriBuilder.addParameter("maximumRecords", "30"); return uriBuilder.build().toURL(); } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/BvbFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/BvbFetcherTest.java index 0940e1311a1..9c2eacf86b3 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/BvbFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/BvbFetcherTest.java @@ -6,19 +6,17 @@ import java.util.List; import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.search.query.SearchQueryVisitor; import org.jabref.logic.util.StandardFileType; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.SearchQuery; import org.jabref.testutils.category.FetcherTest; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; -import static org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer.NO_EXPLICIT_FIELD; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -53,7 +51,7 @@ class BvbFetcherTest { @Test void performTest() throws FetcherException { - String searchquery = "effective java author:bloch"; + String searchquery = "effective java author=bloch"; List result = fetcher.performSearch(searchquery); assertFalse(result.isEmpty()); @@ -64,18 +62,20 @@ void performTest() throws FetcherException { } @Test - void simpleSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException, QueryNodeParseException { + void simpleSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException { String query = "java jdk"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(query, NO_EXPLICIT_FIELD); - URL url = fetcher.getURLForQuery(luceneQuery); + SearchQuery searchQueryObject = new SearchQuery(query); + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); + URL url = fetcher.getURLForQuery(visitor.visitStart(searchQueryObject.getContext())); assertEquals("http://bvbr.bib-bvb.de:5661/bvb01sru?version=1.1&recordSchema=marcxml&operation=searchRetrieve&query=java%20jdk&maximumRecords=30", url.toString()); } @Test - void complexSearchQueryURLCorrect() throws QueryNodeParseException, MalformedURLException, URISyntaxException { - String query = "title:jdk"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(query, NO_EXPLICIT_FIELD); - URL url = fetcher.getURLForQuery(luceneQuery); + void complexSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException { + String query = "title=jdk"; + SearchQuery searchQueryObject = new SearchQuery(query); + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); + URL url = fetcher.getURLForQuery(visitor.visitStart(searchQueryObject.getContext())); assertEquals("http://bvbr.bib-bvb.de:5661/bvb01sru?version=1.1&recordSchema=marcxml&operation=searchRetrieve&query=jdk&maximumRecords=30", url.toString()); } From 2f9ea97ad8c1c8f1fd070c863895f55deb6a688f Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 00:26:16 +0300 Subject: [PATCH 11/37] Updated DBLPFetcher and DBLPQueryTransformerTest to use the new parser and node logic --- .../logic/importer/fetcher/DBLPFetcher.java | 6 +++--- .../DBLPQueryTransformerTest.java | 20 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java index 188dc6c9158..4640965d9e0 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java @@ -21,9 +21,9 @@ import org.jabref.logic.layout.format.RemoveLatexCommandsFormatter; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches BibTeX data from DBLP (dblp.org) @@ -43,9 +43,9 @@ public DBLPFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); - uriBuilder.addParameter("q", new DBLPQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("q", new DBLPQueryTransformer().transformSearchQuery(queryNode).orElse("")); uriBuilder.addParameter("h", String.valueOf(100)); // number of hits uriBuilder.addParameter("c", String.valueOf(0)); // no need for auto-completion uriBuilder.addParameter("f", String.valueOf(0)); // "from", index of first hit to download diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformerTest.java index c4826af85e8..0c7663690a1 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,10 +42,11 @@ public String getTitlePrefix() { @Test public void convertYearField() throws QueryNodeParseException { String queryString = "year:2015"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); DBLPQueryTransformer transformer = getTransformer(); - Optional searchQuery = transformer.transformLuceneQuery(luceneQuery); - assertEquals(Optional.empty(), searchQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); + assertEquals(Optional.empty(), query); assertEquals(Optional.of(2015), transformer.getStartYear()); assertEquals(Optional.of(2015), transformer.getEndYear()); } @@ -52,10 +55,11 @@ public void convertYearField() throws QueryNodeParseException { @Test public void convertYearRangeField() throws QueryNodeParseException { String queryString = "year-range:2012-2015"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); DBLPQueryTransformer transformer = getTransformer(); - Optional searchQuery = transformer.transformLuceneQuery(luceneQuery); - assertEquals(Optional.empty(), searchQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); + assertEquals(Optional.empty(), query); assertEquals(Optional.of(2012), transformer.getStartYear()); assertEquals(Optional.of(2015), transformer.getEndYear()); } From f3d40af66e929910acb07e8986569ae31afb2132 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 00:40:57 +0300 Subject: [PATCH 12/37] Updated DOABFetcher and DOABFetcherTest to use the new parser and node logic --- .../java/org/jabref/logic/importer/fetcher/DOABFetcher.java | 6 +++--- .../org/jabref/logic/importer/fetcher/DOABFetcherTest.java | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java index 74e73773414..92bd98b1e46 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOABFetcher.java @@ -16,13 +16,13 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.strings.StringUtil; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * fetches books from https://www.doabooks.org/ through @@ -37,9 +37,9 @@ public String getName() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder builder = new URIBuilder(SEARCH_URL); - String query = new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse(""); + String query = new DefaultQueryTransformer().transformSearchQuery(queryNode).orElse(""); // adding quotations for the query for more specified results // without the quotation the results returned are not relevant to the query query = "\"".concat(query).concat("\""); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/DOABFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/DOABFetcherTest.java index d4aba9c8254..8af1477e7fc 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/DOABFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/DOABFetcherTest.java @@ -71,7 +71,8 @@ public static Stream performSearch() { .withField(StandardField.LANGUAGE, "English") .withField(StandardField.KEYWORDS, "Religion, thema EDItEUR::Q Philosophy and Religion::QR Religion and beliefs::QRM Christianity::QRMF Christianity: sacred texts and revered writings::QRMF1 Bibles::QRMF13 New Testaments") .withField(StandardField.PUBLISHER, "Brill"), - "Four Kingdom Motifs before and beyond the Book of Daniel" + "\"Four Kingdom Motifs before and beyond the Book of Daniel\"" + // The title needs to be in quotes in order for and to be parsed correctly here ), Arguments.of( new BibEntry(StandardEntryType.Book) @@ -98,7 +99,8 @@ public static Stream performSearch() { .withField(StandardField.LANGUAGE, "English") .withField(StandardField.KEYWORDS, "agile, structural equation modelling, information technology, success, models, strategic alignment, complexity, waterfall, project management, quantitative, Agile software development, Change management, Deliverable, Exploratory factor analysis, South Africa, thema EDItEUR::U Computing and Information Technology::UB Information technology: general topics") .withField(StandardField.PUBLISHER, "AOSIS"), - "The symbiosis between information system project complexity and information system project success" + "\"The symbiosis between information system project complexity and information system project success\"" + // The title needs to be in quotes in order for and to be parsed correctly here ) ); } From 184a7212ef88318d6a6f264c442e41def67d7ef8 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 21:28:02 +0300 Subject: [PATCH 13/37] Updated DOAJFetcher to use the new parser and node logic --- .../java/org/jabref/logic/importer/fetcher/DOAJFetcher.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index 8ffd2820317..9c2e314f91e 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -21,12 +21,12 @@ import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.strings.StringUtil; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,9 +184,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - DOAJFetcher.addPath(uriBuilder, new DefaultLuceneQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + DOAJFetcher.addPath(uriBuilder, new DefaultLuceneQueryTransformer().transformSearchQuery(queryNode).orElse("")); // Number of results uriBuilder.addParameter("pageSize", "30"); // Page (not needed so far) From 3ce8865fc6b7e1592843a3f9b10a0b012d28d01d Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 22:02:58 +0300 Subject: [PATCH 14/37] Updated GoogleScholar and SearchBasedFetcherCapabilityTest to use the new parser and node logic --- .../org/jabref/logic/importer/fetcher/GoogleScholar.java | 6 +++--- .../importer/fetcher/SearchBasedFetcherCapabilityTest.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java index bb73400f75d..0b6560d709d 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java @@ -28,9 +28,9 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.paging.Page; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; @@ -179,9 +179,9 @@ private void obtainAndModifyCookie() throws FetcherException { } @Override - public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException { ScholarQueryTransformer queryTransformer = new ScholarQueryTransformer(); - String transformedQuery = queryTransformer.transformLuceneQuery(luceneQuery).orElse(""); + String transformedQuery = queryTransformer.transformSearchQuery(queryNode).orElse(""); obtainAndModifyCookie(); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index a659a1fbbfd..ef8c313b148 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -36,7 +36,7 @@ interface SearchBasedFetcherCapabilityTest { */ @Test default void supportsAuthorSearch() throws FetcherException { - StringJoiner queryBuilder = new StringJoiner("\" AND author:\"", "author:\"", "\""); + StringJoiner queryBuilder = new StringJoiner("\" AND author=\"", "author=\"", "\""); getTestAuthors().forEach(queryBuilder::add); List result = getFetcher().performSearch(queryBuilder.toString()); @@ -58,7 +58,7 @@ default void supportsAuthorSearch() throws FetcherException { */ @Test default void supportsYearSearch() throws FetcherException { - List result = getFetcher().performSearch("year:" + getTestYear()); + List result = getFetcher().performSearch("year=" + getTestYear()); FieldPreferences fieldPreferences = mock(FieldPreferences.class); when(fieldPreferences.getNonWrappableFields()).thenReturn(FXCollections.observableArrayList()); ImportCleanup.targeting(BibDatabaseMode.BIBTEX, fieldPreferences).doPostCleanup(result); @@ -79,7 +79,7 @@ default void supportsYearSearch() throws FetcherException { default void supportsYearRangeSearch() throws FetcherException { List yearsInYearRange = List.of("2018", "2019", "2020"); - List result = getFetcher().performSearch("year-range:2018-2020"); + List result = getFetcher().performSearch("year-range=2018-2020"); assertFalse(result.isEmpty()); FieldPreferences fieldPreferences = mock(FieldPreferences.class); From b28df9cdbb0725c4f71ea81133a420980650bc60 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 22:53:36 +0300 Subject: [PATCH 15/37] Updated GvkFetcher GvkFetcherTest and GVKQueryTransformerTest to use the new Search.g4 logic. Fixed how AbstractQueryTransformer handles fields. --- .../logic/importer/fetcher/GvkFetcher.java | 6 ++-- .../AbstractQueryTransformer.java | 4 +-- .../importer/fetcher/GvkFetcherTest.java | 28 +++++++++---------- .../transformers/GVKQueryTransformerTest.java | 11 +++++--- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index ae463ba3391..ad1f0d0d833 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -18,9 +18,9 @@ import org.jabref.logic.importer.fileformat.PicaXmlParser; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class GvkFetcher extends AbstractIsbnFetcher implements SearchBasedParserFetcher { @@ -47,11 +47,11 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); uriBuilder.addParameter("operation", "searchRetrieve"); - uriBuilder.addParameter("query", new GVKQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("query", new GVKQueryTransformer().transformSearchQuery(queryNode).orElse("")); uriBuilder.addParameter("maximumRecords", "50"); uriBuilder.addParameter("recordSchema", "picaxml"); uriBuilder.addParameter("sortKeys", "Year,,1"); diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java index 0f9c050fcec..82f954c8c0e 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java @@ -75,7 +75,7 @@ private Optional transform(SearchQueryNode query) { String term = query.term(); String field = NO_EXPLICIT_FIELD; if (query.field().isPresent()) { - field = String.valueOf(query.field().get()).toLowerCase(); + field = String.valueOf(query.field().get().getName()).toLowerCase(); } switch (field) { case "author" -> { @@ -184,7 +184,7 @@ protected Optional handleUnFieldedTerm(String term) { } protected String createKeyValuePair(String fieldAsString, String term) { - return createKeyValuePair(fieldAsString, term, ":"); + return createKeyValuePair(fieldAsString, term, "="); } protected String createKeyValuePair(String fieldAsString, String term, String separator) { diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java index 254b7a6d890..37e701cc2a2 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java @@ -7,16 +7,14 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer; +import org.jabref.logic.search.query.SearchQueryVisitor; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.SearchQuery; import org.jabref.testutils.category.FetcherTest; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Answers; @@ -68,37 +66,39 @@ void getName() { } @Test - void simpleSearchQueryURLCorrect() throws QueryNodeParseException, MalformedURLException, URISyntaxException { + void simpleSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException { String query = "java jdk"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - URL url = fetcher.getURLForQuery(luceneQuery); + SearchQuery searchQueryObject = new SearchQuery(query); + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); + URL url = fetcher.getURLForQuery(visitor.visitStart(searchQueryObject.getContext())); assertEquals("https://sru.k10plus.de/opac-de-627?version=1.1&operation=searchRetrieve&query=pica.all%3Djava%20and%20pica.all%3Djdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } @Test - void complexSearchQueryURLCorrect() throws QueryNodeParseException, MalformedURLException, URISyntaxException { - String query = "kon:java tit:jdk"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - URL url = fetcher.getURLForQuery(luceneQuery); + void complexSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException { + String query = "kon=java tit=jdk"; + SearchQuery searchQueryObject = new SearchQuery(query); + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); + URL url = fetcher.getURLForQuery(visitor.visitStart(searchQueryObject.getContext())); assertEquals("https://sru.k10plus.de/opac-de-627?version=1.1&operation=searchRetrieve&query=pica.kon%3Djava%20and%20pica.tit%3Djdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } @Test void performSearchMatchingMultipleEntries() throws FetcherException { - List searchResult = fetcher.performSearch("title:\"effective java\""); + List searchResult = fetcher.performSearch("title=\"effective java\""); assertTrue(searchResult.contains(bibEntryPPN591166003)); assertTrue(searchResult.contains(bibEntryPPN66391437X)); } @Test void performSearch591166003() throws FetcherException { - List searchResult = fetcher.performSearch("ppn:591166003"); + List searchResult = fetcher.performSearch("ppn=591166003"); assertEquals(List.of(bibEntryPPN591166003), searchResult); } @Test void performSearch66391437X() throws FetcherException { - List searchResult = fetcher.performSearch("ppn:66391437X"); + List searchResult = fetcher.performSearch("ppn=66391437X"); assertEquals(List.of(bibEntryPPN66391437X), searchResult); } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformerTest.java index fba68a3daab..8964f7c5e60 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -41,8 +43,9 @@ public String getTitlePrefix() { @Test public void convertYearField() throws QueryNodeParseException { String queryString = "year:2018"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional query = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("pica.erj=2018"); assertEquals(expected, query); From 9c978dc4d86a4893ec2cf1e3fb51678996ff48e5 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 22:59:55 +0300 Subject: [PATCH 16/37] Updated IEEE and IEEEQueryTransformerTest to use Search.g4 based parser --- .../jabref/logic/importer/fetcher/IEEE.java | 6 ++-- .../IEEEQueryTransformerTest.java | 28 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index 2a5e36b3a33..a4878bedf95 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -29,12 +29,12 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.identifier.DOI; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.strings.StringUtil; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -268,11 +268,11 @@ public String getTestUrl() { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode, int pageNumber) throws URISyntaxException, MalformedURLException { // transformer is stored globally, because we need to filter out the bib entries by the year manually // the transformer stores the min and max year transformer = new IEEEQueryTransformer(); - String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); + String transformedQuery = transformer.transformSearchQuery(queryNode).orElse(""); URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); importerPreferences.getApiKey(FETCHER_NAME).ifPresent(apiKey -> uriBuilder.addParameter("apikey", apiKey)); if (!transformedQuery.isBlank()) { diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java index 1ad8b2e1fb5..9eafcf8a5de 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java @@ -3,9 +3,11 @@ import java.util.Optional; import java.util.stream.Stream; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -46,8 +48,9 @@ public void convertJournalFieldPrefix() throws QueryNodeParseException { IEEEQueryTransformer transformer = getTransformer(); String queryString = "journal:Nature"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - transformer.transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + getTransformer().transformSearchQuery(searchQueryList); assertEquals(Optional.of("Nature"), transformer.getJournal()); } @@ -61,8 +64,9 @@ public void convertYearField() throws QueryNodeParseException { IEEEQueryTransformer transformer = getTransformer(); String queryString = "year:2021"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - transformer.transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + getTransformer().transformSearchQuery(searchQueryList); assertEquals(Optional.of(2021), transformer.getStartYear()); assertEquals(Optional.of(2021), transformer.getEndYear()); @@ -74,8 +78,9 @@ public void convertYearRangeField() throws QueryNodeParseException { IEEEQueryTransformer transformer = getTransformer(); String queryString = "year-range:2018-2021"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - transformer.transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + getTransformer().transformSearchQuery(searchQueryList); assertEquals(Optional.of(2018), transformer.getStartYear()); assertEquals(Optional.of(2021), transformer.getEndYear()); @@ -92,8 +97,9 @@ private static Stream getTitleTestData() { @ParameterizedTest @MethodSource("getTitleTestData") void stopWordRemoval(String expected, String queryString) throws QueryNodeParseException { - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional result = getTransformer().transformLuceneQuery(luceneQuery); - assertEquals(Optional.ofNullable(expected), result); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); + assertEquals(Optional.ofNullable(expected), query); } } From 60b6db9a2cce8adc1fc7fc945c9e898ef991b433 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 23:02:03 +0300 Subject: [PATCH 17/37] Updated InfixTransformerTest to use Search.g4 based parser. --- .../transformers/InfixTransformerTest.java | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/InfixTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/InfixTransformerTest.java index 42e099d024e..41ac49ca607 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/InfixTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/InfixTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -39,37 +41,41 @@ protected String getTitlePrefix() { @Test public void convertAuthorFieldPrefix() throws QueryNodeParseException { String queryString = "author:\"Igor Steinmacher\""; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\""); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void convertUnFieldedTermPrefix() throws QueryNodeParseException { String queryString = "\"default value\""; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of(getUnFieldedPrefix() + queryString); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void convertExplicitUnFieldedTermPrefix() throws QueryNodeParseException { String queryString = "default:\"default value\""; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of(getUnFieldedPrefix() + "\"default value\""); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void convertJournalFieldPrefix() throws QueryNodeParseException { String queryString = "journal:Nature"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of(getJournalPrefix() + "Nature"); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test @@ -81,27 +87,30 @@ public void convertJournalFieldPrefix() throws QueryNodeParseException { @Test public void convertMultipleValuesWithTheSameFieldPrefix() throws QueryNodeParseException { String queryString = "author:\"Igor Steinmacher\" author:\"Christoph Treude\""; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformer().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Treude\""); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void groupedOperationsPrefix() throws QueryNodeParseException { String queryString = "(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND title:test"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformer().getLogicalOrOperator() + "(" + getAuthorPrefix() + "\"Christoph Treude\"" + getTransformer().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Freunde\"))" + getTransformer().getLogicalAndOperator() + getTitlePrefix() + "test"); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void notOperatorPrefix() throws QueryNodeParseException { String queryString = "!(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\")"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of(getTransformer().getLogicalNotOperator() + "(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformer().getLogicalOrOperator() + getAuthorPrefix() + "\"Christoph Treude\")"); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } } From 98aa35f3d490a4737c0705a7881bd6b0f0a92d6a Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 23:06:03 +0300 Subject: [PATCH 18/37] Updated INSPIREFetcher to use Search.g4 based parser. --- .../org/jabref/logic/importer/fetcher/INSPIREFetcher.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index 8038ea2edd0..53707633b33 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -17,7 +17,7 @@ import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformers.DefaultLuceneQueryTransformer; +import org.jabref.logic.importer.fetcher.transformers.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.importer.util.MediaTypes; import org.jabref.logic.layout.format.LatexToUnicodeFormatter; @@ -25,9 +25,9 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the INSPIRE database. @@ -55,9 +55,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(INSPIRE_HOST); - uriBuilder.addParameter("q", new DefaultLuceneQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("q", new DefaultQueryTransformer().transformSearchQuery(queryNode).orElse("")); return uriBuilder.build().toURL(); } From efcd08fd8a4aa72df561d5bfc3472dfec126dd15 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 23:09:52 +0300 Subject: [PATCH 19/37] Updated ISIDOREFetcher and ISIDOREFetcherTest to use Search.g4 based parser --- .../org/jabref/logic/importer/fetcher/ISIDOREFetcher.java | 6 +++--- .../jabref/logic/importer/fetcher/ISIDOREFetcherTest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java index b4ec8f55206..569a261d590 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ISIDOREFetcher.java @@ -25,10 +25,10 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.EntryType; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import jakarta.ws.rs.core.MediaType; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.jooq.lambda.Unchecked; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,9 +96,9 @@ public URLDownload getUrlDownload(URL url) { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode, int pageNumber) throws URISyntaxException, MalformedURLException { ISIDOREQueryTransformer queryTransformer = new ISIDOREQueryTransformer(); - String transformedQuery = queryTransformer.transformLuceneQuery(luceneQuery).orElse(""); + String transformedQuery = queryTransformer.transformSearchQuery(queryNode).orElse(""); URIBuilder uriBuilder = new URIBuilder(SOURCE_WEB_SEARCH); uriBuilder.addParameter("q", transformedQuery); if (pageNumber > 1) { diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ISIDOREFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ISIDOREFetcherTest.java index 219645eb6c0..cd870ff8b99 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ISIDOREFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ISIDOREFetcherTest.java @@ -100,7 +100,7 @@ void noResults() throws FetcherException { @Test void author() throws FetcherException { - List actual = fetcher.performSearch("author:\"Adam Strange\""); + List actual = fetcher.performSearch("author=\"Adam Strange\""); assertEquals(List.of(new BibEntry(StandardEntryType.Article) .withField(StandardField.AUTHOR, "Howard Green and Karen Boyland and Adam Strange") .withField(StandardField.DOI, "doi:10.3406/htn.1990.2970") From 45245b80020e7b6af80a9f7282c828d07c480321 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 23:13:47 +0300 Subject: [PATCH 20/37] Updated JstorFetcher to use Search.g4 based fetcher --- .../logic/importer/fetcher/JstorFetcher.java | 6 +++--- .../logic/importer/fetcher/JstorFetcherTest.java | 2 +- .../transformers/JstorQueryTransformerTest.java | 16 ++++++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java index dbbe5fa2c24..94d26fd616f 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java @@ -25,9 +25,9 @@ import org.jabref.logic.util.URLUtil; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -49,9 +49,9 @@ public JstorFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST); - uriBuilder.addParameter("Query", new JstorQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("Query", new JstorQueryTransformer().transformSearchQuery(queryNode).orElse("")); return uriBuilder.build().toURL(); } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/JstorFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/JstorFetcherTest.java index bd737ac4e45..0c105692ca7 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/JstorFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/JstorFetcherTest.java @@ -60,7 +60,7 @@ public class JstorFetcherTest implements SearchBasedFetcherCapabilityTest { @Test void searchByTitle() throws FetcherException { - List entries = fetcher.performSearch("title: \"Test Anxiety Analysis of Chinese College Students in Computer-based Spoken English Test\""); + List entries = fetcher.performSearch("title= \"Test Anxiety Analysis of Chinese College Students in Computer-based Spoken English Test\""); assertEquals(List.of(bibEntry), entries); } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/JstorQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/JstorQueryTransformerTest.java index c72d030e93b..bee8a38b83e 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/JstorQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/JstorQueryTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,8 +42,9 @@ public String getTitlePrefix() { @Test public void convertYearField() throws QueryNodeParseException { String queryString = "year:2018"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional query = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); assertEquals(Optional.of("sd:2018 AND ed:2018"), query); } @@ -49,8 +52,9 @@ public void convertYearField() throws QueryNodeParseException { @Test public void convertYearRangeField() throws QueryNodeParseException { String queryString = "year-range:2018-2021"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional query = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); assertEquals(Optional.of("sd:2018 AND ed:2021"), query); } } From ee311be686e4e6b9e6fc4874ce81f1935fea4c4b Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 23:21:37 +0300 Subject: [PATCH 21/37] Updated LOBIDFetcher to use Search.g4 based fetcher --- .../org/jabref/logic/importer/fetcher/LOBIDFetcher.java | 8 ++++---- .../fetcher/transformers/AbstractQueryTransformer.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java index e7e1309b2cc..3958a62c89d 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java @@ -22,11 +22,11 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.EntryType; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,14 +46,14 @@ public class LOBIDFetcher implements PagedSearchBasedParserFetcher, IdBasedParse /** * Gets the query URL * - * @param luceneQuery the search query + * @param queryNode the list that contains the parsed nodes * @param pageNumber the number of the page indexed from 0 * @return URL */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("q", new LOBIDQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // search query + uriBuilder.addParameter("q", new LOBIDQueryTransformer().transformSearchQuery(queryNode).orElse("")); // search query uriBuilder.addParameter("from", String.valueOf(getPageSize() * pageNumber)); // from entry number, starts indexing at 0 uriBuilder.addParameter("size", String.valueOf(getPageSize())); uriBuilder.addParameter("format", "json"); diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java index 82f954c8c0e..524b269292c 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java @@ -184,7 +184,7 @@ protected Optional handleUnFieldedTerm(String term) { } protected String createKeyValuePair(String fieldAsString, String term) { - return createKeyValuePair(fieldAsString, term, "="); + return createKeyValuePair(fieldAsString, term, ":"); } protected String createKeyValuePair(String fieldAsString, String term, String separator) { From 59175ac84ee6482cc719f07a3fec67fe831a7a2d Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Mon, 18 Aug 2025 23:36:36 +0300 Subject: [PATCH 22/37] Updated MathSciNet to use Search.g4 based fetcher --- .../java/org/jabref/logic/importer/fetcher/MathSciNet.java | 6 +++--- .../org/jabref/logic/importer/fetcher/MathSciNetTest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java index 16428c7d9c0..43688de3266 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java @@ -33,6 +33,7 @@ import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.identifier.DOI; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.util.DummyFileUpdateMonitor; import kong.unirest.core.JsonNode; @@ -40,7 +41,6 @@ import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,9 +96,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/api/publications/search"); - uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // query + uriBuilder.addParameter("query", new DefaultQueryTransformer().transformSearchQuery(queryNode).orElse("")); // query uriBuilder.addParameter("currentPage", "1"); // start index uriBuilder.addParameter("pageSize", "100"); // page size return uriBuilder.build().toURL(); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/MathSciNetTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/MathSciNetTest.java index 1d7ffb513b2..742a7bfa4e8 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/MathSciNetTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/MathSciNetTest.java @@ -86,7 +86,7 @@ void searchByIdInEntryFindsEntry() throws FetcherException { @Test @DisabledOnCIServer("CI server has no subscription to MathSciNet and thus gets 401 response. One single call goes through, but subsequent calls fail.") void searchByQueryFindsEntry() throws FetcherException { - List fetchedEntries = fetcher.performSearch("Existence and uniqueness theorems Two-Dimensional Ericksen Leslie System"); + List fetchedEntries = fetcher.performSearch("\"Existence and uniqueness theorems Two-Dimensional Ericksen Leslie System\""); assertFalse(fetchedEntries.isEmpty()); BibEntry secondEntry = fetchedEntries.get(1); normalizeWhitespacesCleanup.cleanup(secondEntry); From bf0be13fadec582f61c0338e64447db1ebda1883 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Tue, 19 Aug 2025 21:10:18 +0300 Subject: [PATCH 23/37] Changed general exceptions to more specific ones according to feedback --- .../jabref/logic/importer/PagedSearchBasedFetcher.java | 8 ++++++-- .../org/jabref/logic/importer/SearchBasedFetcher.java | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index 2b3cfd7b13d..07a1760880c 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -9,6 +9,8 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; +import org.antlr.v4.runtime.misc.ParseCancellationException; + public interface PagedSearchBasedFetcher extends SearchBasedFetcher { /** @@ -31,8 +33,10 @@ default Page performSearchPaged(String searchQuery, int pageNumber) th SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); try { return this.performSearchPaged(visitor.visitStart(searchQueryObject.getContext()), pageNumber); - } catch (Exception e) { - throw new FetcherException("An error occurred during parsing of the query."); + } catch (ParseCancellationException e) { + throw new FetcherException("A syntax error occurred during parsing of the query"); + } catch (NullPointerException e) { + throw new FetcherException("The query string or a field in the query is empty"); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index 2741d106271..6460f96046d 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -7,6 +7,8 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; +import org.antlr.v4.runtime.misc.ParseCancellationException; + /** * Searches web resources for bibliographic information based on a free-text query. * May return multiple search hits. @@ -27,7 +29,7 @@ public interface SearchBasedFetcher extends WebFetcher { /** * Looks for hits which are matched by the given free-text query. * - * @param searchQuery query string that can be parsed into a search.g4 query + * @param searchQuery query string that can be parsed into a search query * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ default List performSearch(String searchQuery) throws FetcherException { @@ -40,8 +42,10 @@ default List performSearch(String searchQuery) throws FetcherException BaseQueryNode queryNode; try { queryNode = visitor.visitStart(searchQueryObject.getContext()); - } catch (Exception e) { - throw new FetcherException("An error occurred when parsing the query"); + } catch (ParseCancellationException e) { + throw new FetcherException("A syntax error occurred during parsing of the query"); + } catch (NullPointerException e) { + throw new FetcherException("The query string or a field is empty"); } return this.performSearch(queryNode); From 492456fbc80fe3a7566445a53c9630b55ad96236 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Tue, 19 Aug 2025 21:46:37 +0300 Subject: [PATCH 24/37] Removed null pointer exception --- .../java/org/jabref/logic/importer/PagedSearchBasedFetcher.java | 2 -- .../main/java/org/jabref/logic/importer/SearchBasedFetcher.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index 07a1760880c..2a740f74141 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -35,8 +35,6 @@ default Page performSearchPaged(String searchQuery, int pageNumber) th return this.performSearchPaged(visitor.visitStart(searchQueryObject.getContext()), pageNumber); } catch (ParseCancellationException e) { throw new FetcherException("A syntax error occurred during parsing of the query"); - } catch (NullPointerException e) { - throw new FetcherException("The query string or a field in the query is empty"); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index 6460f96046d..4d4eac7e3fb 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -44,8 +44,6 @@ default List performSearch(String searchQuery) throws FetcherException queryNode = visitor.visitStart(searchQueryObject.getContext()); } catch (ParseCancellationException e) { throw new FetcherException("A syntax error occurred during parsing of the query"); - } catch (NullPointerException e) { - throw new FetcherException("The query string or a field is empty"); } return this.performSearch(queryNode); From f561bbd3107beb05192433111905b18cc24f2151 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Tue, 19 Aug 2025 21:46:52 +0300 Subject: [PATCH 25/37] Removed null pointer exception --- .../org/jabref/logic/importer/fetcher/MedlineFetcher.java | 6 +++--- .../jabref/logic/importer/fetcher/MedlineFetcherTest.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java index eef636db451..6e73179dadb 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java @@ -33,9 +33,9 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -187,10 +187,10 @@ private List fetchMedline(List ids) throws FetcherException { } @Override - public List performSearch(QueryNode luceneQuery) throws FetcherException { + public List performSearch(BaseQueryNode queryNode) throws FetcherException { List entryList; MedlineQueryTransformer transformer = new MedlineQueryTransformer(); - Optional transformedQuery = transformer.transformLuceneQuery(luceneQuery); + Optional transformedQuery = transformer.transformSearchQuery(queryNode); if (transformedQuery.isEmpty() || transformedQuery.get().isBlank()) { return List.of(); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java index 395f4c1f211..885fa7b5a64 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java @@ -180,14 +180,14 @@ void multipleEntries() throws FetcherException { @Test void withLuceneQueryAuthorDate() throws FetcherException { - List entryList = fetcher.performSearch("author:vigmond AND year:2021"); + List entryList = fetcher.performSearch("author=vigmond AND year=2021"); entryList.forEach(entry -> entry.clearField(StandardField.ABSTRACT)); // Remove abstract due to copyright); assertEquals(18, entryList.size()); } @Test void withLuceneQueryAuthorDateRange() throws FetcherException { - List entryList = fetcher.performSearch("author:vigmond AND year-range:2020-2021"); + List entryList = fetcher.performSearch("author=vigmond AND year-range=2020-2021"); entryList.forEach(entry -> entry.clearField(StandardField.ABSTRACT)); // Remove abstract due to copyright); assertEquals(28, entryList.size()); } From d7d4a046607461d21ef39a95988d603c37be06ec Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Tue, 19 Aug 2025 21:56:35 +0300 Subject: [PATCH 26/37] Updated ScholarArchiveFetcher to use the ANTLR parser --- .../logic/importer/fetcher/ScholarArchiveFetcher.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java index 3c1521e1861..7a19dc3b91d 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarArchiveFetcher.java @@ -19,13 +19,13 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.EntryType; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import jakarta.ws.rs.core.MediaType; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,14 +40,14 @@ public class ScholarArchiveFetcher implements PagedSearchBasedParserFetcher { /** * Gets the query URL by luceneQuery and pageNumber. * - * @param luceneQuery the search query + * @param queryNode the first node from the parsed query * @param pageNumber the number of the page indexed from 0 * @return URL */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("q", new ScholarArchiveQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("q", new ScholarArchiveQueryTransformer().transformSearchQuery(queryNode).orElse("")); uriBuilder.addParameter("from", String.valueOf(getPageSize() * pageNumber)); uriBuilder.addParameter("size", String.valueOf(getPageSize())); uriBuilder.addParameter("format", "json"); From 9d8b24d901d879c57bab9135d2934f6faa9dee6b Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Tue, 19 Aug 2025 22:22:18 +0300 Subject: [PATCH 27/37] Updated SpringerFetcher to use the ANTLR parser --- .../importer/fetcher/SpringerFetcher.java | 8 ++++---- .../SpringerQueryTransformerTest.java | 20 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index 71cce64e126..d29731aa9d4 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -23,12 +23,12 @@ import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import com.google.common.base.Strings; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -177,14 +177,14 @@ public String getTestUrl() { /** * Gets the query URL * - * @param luceneQuery the search query + * @param queryNode the search query * @param pageNumber the number of the page indexed from 0 * @return URL */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("q", new SpringerQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Search query + uriBuilder.addParameter("q", new SpringerQueryTransformer().transformSearchQuery(queryNode).orElse("")); // Search query importerPreferences.getApiKey(getName()).ifPresent(key -> uriBuilder.addParameter("api_key", key)); // API key uriBuilder.addParameter("s", String.valueOf(getPageSize() * pageNumber + 1)); // Start entry, starts indexing at 1 uriBuilder.addParameter("p", String.valueOf(getPageSize())); // Page size diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SpringerQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SpringerQueryTransformerTest.java index af6b55cfb82..57306f48c55 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SpringerQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SpringerQueryTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,21 +42,23 @@ public String getTitlePrefix() { @Test public void convertYearField() throws QueryNodeParseException { String queryString = "year:2015"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("date:2015*"); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Override @Test public void convertYearRangeField() throws QueryNodeParseException { String queryString = "year-range:2012-2015"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("date:2012* OR date:2013* OR date:2014* OR date:2015*"); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } } From 879e5c153b7eac6727057a8d4b2af0a076690260 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Tue, 19 Aug 2025 22:56:35 +0300 Subject: [PATCH 28/37] Updated rest of the classes needed for compiling --- .../fetcher/AstrophysicsDataSystem.java | 19 +++--- .../importer/fetcher/BiodiversityLibrary.java | 6 +- .../logic/importer/fetcher/CiteSeer.java | 12 ++-- ...fComputerScienceBibliographiesFetcher.java | 6 +- .../fetcher/CompositeSearchBasedFetcher.java | 6 +- .../logic/importer/fetcher/CrossRef.java | 6 +- .../logic/importer/fetcher/ResearchGate.java | 14 ++-- .../importer/fetcher/SemanticScholar.java | 6 +- .../jabref/logic/importer/fetcher/ZbMATH.java | 6 +- ...puterScienceBibliographiesFetcherTest.java | 11 +-- .../importer/fetcher/ResearchGateTest.java | 12 ++-- .../importer/fetcher/SemanticScholarTest.java | 9 +-- .../CiteSeerQueryTransformerTest.java | 45 +++++++------ ...nceBibliographiesQueryTransformerTest.java | 16 +++-- .../transformers/SuffixTransformerTest.java | 67 +++++++++++-------- ...rRangeByFilteringQueryTransformerTest.java | 13 ++-- ...rRangeByFilteringQueryTransformerTest.java | 13 ++-- .../ZbMathQueryTransformerTest.java | 24 ++++--- 18 files changed, 160 insertions(+), 131 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 57cc590f901..03328690c82 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -33,13 +33,13 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; import org.jabref.model.paging.Page; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.strings.StringUtil; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the SAO/NASA Astrophysics Data System (https://ui.adsabs.harvard.edu/) @@ -81,13 +81,14 @@ public String getName() { } /** - * @param luceneQuery query string, matching the apache solr format + * @param queryList the list that contains the parsed nodes * @return URL which points to a search request for given query */ @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryList, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + builder.addParameter("q", new DefaultQueryTransformer().transformSearchQuery(queryList).orElse("")); + builder.addParameter("q", ""); builder.addParameter("fl", "bibcode"); builder.addParameter("rows", String.valueOf(getPageSize())); builder.addParameter("start", String.valueOf(getPageSize() * pageNumber)); @@ -278,10 +279,10 @@ private List performSearchByIds(Collection identifiers) throws } @Override - public List performSearch(QueryNode luceneQuery) throws FetcherException { + public List performSearch(BaseQueryNode queryList) throws FetcherException { URL urlForQuery; try { - urlForQuery = getURLForQuery(luceneQuery); + urlForQuery = getURLForQuery(queryList); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); } @@ -290,17 +291,17 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti } @Override - public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(BaseQueryNode queryList, int pageNumber) throws FetcherException { URL urlForQuery; try { - urlForQuery = getURLForQuery(luceneQuery, pageNumber); + urlForQuery = getURLForQuery(queryList, pageNumber); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI is malformed", e); } // This is currently just interpreting the complex query as a default string query List bibCodes = fetchBibcodes(urlForQuery); Collection results = performSearchByIds(bibCodes); - return new Page<>(luceneQuery.toString(), pageNumber, results); + return new Page<>(queryList.toString(), pageNumber, results); } @Override diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java index eb2612ad245..7bff7e15b57 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java @@ -20,13 +20,13 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import com.google.common.annotations.VisibleForTesting; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -207,12 +207,12 @@ public Parser getParser() { } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode query) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(getBaseURL().toURI()); BiodiversityLibraryTransformer transformer = new BiodiversityLibraryTransformer(); uriBuilder.addParameter("op", "PublicationSearch"); uriBuilder.addParameter("searchtype", "C"); - uriBuilder.addParameter("searchterm", transformer.transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("searchterm", transformer.transformSearchQuery(query).orElse("")); return uriBuilder.build().toURL(); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java index bd71e84a355..1857ca11e6c 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java @@ -17,13 +17,13 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.http.SimpleHttpResponse; +import org.jabref.model.search.query.BaseQueryNode; import kong.unirest.core.HttpResponse; import kong.unirest.core.JsonNode; import kong.unirest.core.Unirest; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONElement; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,10 +53,10 @@ public Optional getHelpPage() { } @Override - public List performSearch(QueryNode luceneQuery) throws FetcherException { + public List performSearch(BaseQueryNode queryNode) throws FetcherException { // ADR-0014 try { - JSONElement payload = getPayloadJSON(luceneQuery); + JSONElement payload = getPayloadJSON(queryNode); HttpResponse httpResponse = Unirest.post(API_URL) .header("authority", BASE_URL) .header("accept", "application/json, text/plain, */*") @@ -77,7 +77,7 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti .map(response -> response.optJSONArray("response")); if (jsonResponse.isEmpty()) { - LOGGER.debug("No entries found for query: {}", luceneQuery); + LOGGER.debug("No entries found for query: {}", queryNode); return List.of(); } @@ -89,9 +89,9 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti } } - private JSONElement getPayloadJSON(QueryNode luceneQuery) { + private JSONElement getPayloadJSON(BaseQueryNode searchQueryList) throws ParseException { transformer = new CiteSeerQueryTransformer(); - String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); + String transformedQuery = transformer.transformSearchQuery(searchQueryList).orElse(""); return transformer.getJSONPayload(); } diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java index 018a9b4c9c5..5a7734251c7 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java @@ -19,9 +19,9 @@ import org.jabref.model.entry.field.FieldFactory; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class CollectionOfComputerScienceBibliographiesFetcher implements SearchBasedParserFetcher { @@ -34,9 +34,9 @@ public CollectionOfComputerScienceBibliographiesFetcher(ImportFormatPreferences } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode query) throws URISyntaxException, MalformedURLException { return new URIBuilder(BASIC_SEARCH_URL) - .addParameter("query", new CollectionOfComputerScienceBibliographiesQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")) + .addParameter("query", new CollectionOfComputerScienceBibliographiesQueryTransformer().transformSearchQuery(query).orElse("")) .addParameter("sort", "score") .build() .toURL(); diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java index ba1b287119d..28a919479f6 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java @@ -11,8 +11,8 @@ import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.SearchBasedFetcher; import org.jabref.model.entry.BibEntry; +import org.jabref.model.search.query.BaseQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,12 +52,12 @@ public Optional getHelpPage() { } @Override - public List performSearch(QueryNode luceneQuery) throws FetcherException { + public List performSearch(BaseQueryNode queryList) throws FetcherException { // All entries have to be converted into one format, this is necessary for the format conversion return fetchers.parallelStream() .flatMap(searchBasedFetcher -> { try { - return searchBasedFetcher.performSearch(luceneQuery).stream(); + return searchBasedFetcher.performSearch(queryList).stream(); } catch (FetcherException e) { LOGGER.warn("%s API request failed".formatted(searchBasedFetcher.getName()), e); return Stream.empty(); diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index 9859460429e..16a05ba2617 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -27,13 +27,13 @@ import org.jabref.model.entry.identifier.DOI; import org.jabref.model.entry.types.EntryType; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.util.OptionalUtil; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * A class for fetching DOIs from CrossRef @@ -65,9 +65,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("query", new DefaultQueryTransformer().transformSearchQuery(queryNode).orElse("")); return uriBuilder.build().toURL(); } diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java index dff0b43a0fb..4f0a56b324f 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java @@ -27,9 +27,9 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.identifier.DOI; +import org.jabref.model.search.query.BaseQueryNode; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -184,11 +184,11 @@ private Document getPage(URL url) throws IOException { * Extract the numerical internal ID and add it to the URL to receive a link to a {@link BibEntry} *

* - * @param luceneQuery the search query. + * @param queryNode the search query. * @return A URL that lets us download a .bib file */ - private static URL getUrlForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { - String query = new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse(""); + private static URL getUrlForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { + String query = new DefaultQueryTransformer().transformSearchQuery(queryNode).orElse(""); URIBuilder source = new URIBuilder(SEARCH); source.addParameter("type", "publication"); source.addParameter("query", query); @@ -203,17 +203,17 @@ public TrustLevel getTrustLevel() { /** * This method is used to send complex queries using fielded search. * - * @param luceneQuery the root node of the lucene query + * @param queryNode the first node from the Search.g4 parser * @return a list of {@link BibEntry}, which are matched by the query (maybe empty) * @throws FetcherException if the ResearchGate refuses to serve the page */ @Override - public List performSearch(QueryNode luceneQuery) throws FetcherException { + public List performSearch(BaseQueryNode queryNode) throws FetcherException { Document html; URL url; try { - url = getUrlForQuery(luceneQuery); + url = getUrlForQuery(queryNode); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Invalid URL", e); } diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java index 41afcbbf225..65d2e82a234 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/SemanticScholar.java @@ -27,13 +27,13 @@ import org.jabref.model.entry.identifier.ArXivIdentifier; import org.jabref.model.entry.identifier.DOI; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.strings.StringUtil; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -135,9 +135,9 @@ String getURLBySource(String source) throws IOException, FetcherException { } @Override - public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SOURCE_WEB_SEARCH); - uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); + uriBuilder.addParameter("query", new DefaultQueryTransformer().transformSearchQuery(queryNode).orElse("")); uriBuilder.addParameter("offset", String.valueOf(pageNumber * getPageSize())); uriBuilder.addParameter("limit", String.valueOf(Math.min(getPageSize(), 10000 - pageNumber * getPageSize()))); // All fields need to be specified diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java index b02c19d9025..0e4a732c518 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java @@ -23,13 +23,13 @@ import org.jabref.model.entry.field.AMSField; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.search.query.BaseQueryNode; import kong.unirest.core.HttpResponse; import kong.unirest.core.JsonNode; import kong.unirest.core.Unirest; import kong.unirest.core.json.JSONArray; import org.apache.hc.core5.net.URIBuilder; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the Zentralblatt Math (https://www.zbmath.org/) @@ -104,9 +104,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(BaseQueryNode queryNode) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); - uriBuilder.addParameter("q", new ZbMathQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // search all fields + uriBuilder.addParameter("q", new ZbMathQueryTransformer().transformSearchQuery(queryNode).orElse("")); // search all fields uriBuilder.addParameter("start", "0"); // start index uriBuilder.addParameter("count", "200"); // should return up to 200 items (instead of default 100) return uriBuilder.build().toURL(); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java index 37e80316379..51a4b6b3ba7 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java @@ -9,15 +9,14 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer; +import org.jabref.logic.search.query.SearchQueryVisitor; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.SearchQuery; import org.jabref.testutils.category.FetcherTest; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -45,9 +44,11 @@ void getNameReturnsCorrectName() { } @Test - void getUrlForQueryReturnsCorrectUrl() throws MalformedURLException, URISyntaxException, QueryNodeParseException { + void getUrlForQueryReturnsCorrectUrl() throws MalformedURLException, URISyntaxException { String query = "java jdk"; - URL url = fetcher.getURLForQuery(new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD)); + SearchQuery searchQueryObject = new SearchQuery(query); + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); + URL url = fetcher.getURLForQuery(visitor.visit(searchQueryObject.getContext())); assertEquals("http://liinwww.ira.uka.de/bibliography/rss?query=java+jdk&sort=score", url.toString()); } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java index 98189897a22..10221a85ec2 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ResearchGateTest.java @@ -6,22 +6,21 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.search.query.SearchQueryVisitor; import org.jabref.logic.util.URLUtil; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.entry.types.UnknownEntryType; +import org.jabref.model.search.query.SearchQuery; import org.jabref.support.DisabledOnCIServer; import org.jabref.testutils.category.FetcherTest; import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Answers; -import static org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer.NO_EXPLICIT_FIELD; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -100,9 +99,10 @@ void performSearchWithLuceneQuery() throws QueryNodeParseException, FetcherExcep .withField(StandardField.AUTHOR, "Petruzzi, Leonardo and Campaniello, Daniela and Corbo," + " Maria and Speranza, Barbara and Altieri, Clelia and Sinigaglia, Milena and Bevilacqua, Antonio"); - QueryNode queryNode = new StandardSyntaxParser().parse("Wine Microbiology and Predictive " + - "Microbiology: A Short Overview on Application, and Perspectives", NO_EXPLICIT_FIELD); - assertEquals(Optional.of(master), fetcher.performSearch(queryNode).stream().findFirst()); + SearchQuery searchQueryObject = new SearchQuery("Wine Microbiology and Predictive " + + "Microbiology: A Short Overview on Application, and Perspectives"); + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); + assertEquals(Optional.of(master), fetcher.performSearch(visitor.visitStart(searchQueryObject.getContext())).stream().findFirst()); } @Test diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/SemanticScholarTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/SemanticScholarTest.java index bfb5d19748f..7351f45473a 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/SemanticScholarTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/SemanticScholarTest.java @@ -12,16 +12,16 @@ import org.jabref.logic.importer.ImporterPreferences; import org.jabref.logic.importer.PagedSearchBasedFetcher; import org.jabref.logic.importer.fetcher.citation.semanticscholar.SemanticScholarCitationFetcher; +import org.jabref.logic.search.query.SearchQueryVisitor; import org.jabref.logic.util.BuildInfo; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.SearchQuery; import org.jabref.support.DisabledOnCIServer; import org.jabref.testutils.category.FetcherTest; import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -127,8 +127,9 @@ public PagedSearchBasedFetcher getPagedFetcher() { @Test void getURLForQueryWithLucene() throws QueryNodeParseException, MalformedURLException, URISyntaxException { String query = "Software engineering"; - SyntaxParser parser = new StandardSyntaxParser(); - URL url = fetcher.getURLForQuery(parser.parse(query, "default"), 0); + SearchQuery searchQueryObject = new SearchQuery(query); + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); + URL url = fetcher.getURLForQuery(visitor.visitStart(searchQueryObject.getContext()), 0); assertEquals("https://api.semanticscholar.org/graph/v1/paper/search?query=Software%20engineering&offset=0&limit=20&fields=paperId%2CexternalIds%2Curl%2Ctitle%2Cabstract%2Cvenue%2Cyear%2Cauthors", url.toString()); } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CiteSeerQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CiteSeerQueryTransformerTest.java index 27143e76001..04544b76968 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CiteSeerQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CiteSeerQueryTransformerTest.java @@ -6,12 +6,13 @@ import java.util.Optional; import java.util.stream.Stream; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; import org.jabref.model.strings.StringUtil; import kong.unirest.core.json.JSONObject; import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -30,9 +31,10 @@ protected CiteSeerQueryTransformer getTransformer() { @Test public void convertYearField() throws QueryNodeParseException { String queryString = "year:2023"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); - transformer.transformLuceneQuery(luceneQuery); + transformer.transformSearchQuery(searchQueryList); Optional start = Optional.of(transformer.getJSONPayload().getInt("yearStart")); Optional end = Optional.of(transformer.getJSONPayload().getInt("yearEnd")); @@ -44,9 +46,10 @@ public void convertYearField() throws QueryNodeParseException { @Test public void convertYearRangeField() throws QueryNodeParseException { String queryString = "year-range:2019-2023"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); - transformer.transformLuceneQuery(luceneQuery); + transformer.transformSearchQuery(searchQueryList); Optional start = Optional.of(transformer.getJSONPayload().getInt("yearStart")); Optional end = Optional.of(transformer.getJSONPayload().getInt("yearEnd")); @@ -57,9 +60,10 @@ public void convertYearRangeField() throws QueryNodeParseException { @Test void convertPageField() throws QueryNodeParseException { String queryString = "page:2"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); - transformer.transformLuceneQuery(luceneQuery); + transformer.transformSearchQuery(searchQueryList); Optional page = Optional.of(transformer.getJSONPayload().getInt("page")); assertEquals(Optional.of(2), page); @@ -68,9 +72,10 @@ void convertPageField() throws QueryNodeParseException { @Test void convertPageSizeField() throws QueryNodeParseException { String queryString = "pageSize:20"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); - transformer.transformLuceneQuery(luceneQuery); + transformer.transformSearchQuery(searchQueryList); Optional pageSize = Optional.of(transformer.getJSONPayload().getInt("pageSize")); assertEquals(Optional.of(20), pageSize); @@ -79,9 +84,10 @@ void convertPageSizeField() throws QueryNodeParseException { @Test void convertSortByField() throws QueryNodeParseException { String queryString = "sortBy:relevance"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); - transformer.transformLuceneQuery(luceneQuery); + transformer.transformSearchQuery(searchQueryList); Optional sortBy = Optional.of(transformer.getJSONPayload().get("sortBy").toString()); assertEquals(Optional.of("relevance"), sortBy); @@ -90,9 +96,10 @@ void convertSortByField() throws QueryNodeParseException { @Test void convertMultipleAuthors() throws QueryNodeParseException { String queryString = "author:\"Wang Wei\" author:\"Zhang Pingwen\" author:\"Zhang Zhifei\""; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); - transformer.transformLuceneQuery(luceneQuery); + transformer.transformSearchQuery(searchQueryList); List authorsActual = transformer.getJSONPayload().getJSONArray("author").toList(); List authorsExpected = List.of("Wang Wei", "Zhang Pingwen", "Zhang Zhifei"); @@ -115,14 +122,10 @@ private static Stream getJSONWithYearVariations() { List actualJSONObjects = new ArrayList<>(); withYearAndYearRange.forEach(requestStr -> { - QueryNode luceneQuery; - try { - luceneQuery = new StandardSyntaxParser().parse(requestStr, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - } catch (QueryNodeParseException e) { - throw new RuntimeException(e); - } + SearchQuery searchQuery = new SearchQuery(requestStr); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = new CiteSeerQueryTransformer(); - transformer.transformLuceneQuery(luceneQuery); + transformer.transformSearchQuery(searchQueryList); actualJSONObjects.add(transformer.getJSONPayload()); }); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java index 69187f412be..744b53baf49 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,8 +42,9 @@ public String getTitlePrefix() { @Test public void convertYearField() throws QueryNodeParseException { String queryString = "year:2018"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional query = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); assertEquals(Optional.of("year:2018"), query); } @@ -49,8 +52,9 @@ public void convertYearField() throws QueryNodeParseException { @Test public void convertYearRangeField() throws QueryNodeParseException { String queryString = "year-range:2018-2021"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional query = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); assertEquals(Optional.of("year:2018 OR year:2019 OR year:2020 OR year:2021"), query); } } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SuffixTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SuffixTransformerTest.java index faf8fc7ecb8..9f6fa9a920f 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SuffixTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SuffixTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -26,38 +28,42 @@ public abstract class SuffixTransformerTest @Test public void convertAuthorFieldSuffix() throws QueryNodeParseException { - String queryString = "author:\"Igor Steinmacher\""; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + String queryString = "author=\"Igor Steinmacher\""; + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("\"Igor Steinmacher\"" + getAuthorSuffix()); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void convertUnFieldedTermSuffix() throws QueryNodeParseException { String queryString = "\"default value\""; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of(queryString + getUnFieldedSuffix()); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void convertExplicitUnFieldedTermSuffix() throws QueryNodeParseException { - String queryString = "default:\"default value\""; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + String queryString = "default=\"default value\""; + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("\"default value\"" + getUnFieldedSuffix()); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void convertJournalFieldSuffix() throws QueryNodeParseException { - String queryString = "journal:Nature"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + String queryString = "journal=Nature"; + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("Nature" + getJournalSuffix()); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test @@ -68,28 +74,31 @@ public void convertJournalFieldSuffix() throws QueryNodeParseException { @Test public void convertMultipleValuesWithTheSameSuffix() throws QueryNodeParseException { - String queryString = "author:\"Igor Steinmacher\" author:\"Christoph Treude\""; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + String queryString = "author=\"Igor Steinmacher\" author=\"Christoph Treude\""; + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("\"Igor Steinmacher\"" + getAuthorSuffix() + getTransformer().getLogicalAndOperator() + "\"Christoph Treude\"" + getAuthorSuffix()); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void groupedOperationsSuffix() throws QueryNodeParseException { - String queryString = "(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND title:test"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + String queryString = "(author=\"Igor Steinmacher\" OR author=\"Christoph Treude\" AND author=\"Christoph Freunde\") AND title=test"; + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("(" + "\"Igor Steinmacher\"" + getAuthorSuffix() + getTransformer().getLogicalOrOperator() + "(" + "\"Christoph Treude\"" + getAuthorSuffix() + getTransformer().getLogicalAndOperator() + "\"Christoph Freunde\"" + getAuthorSuffix() + "))" + getTransformer().getLogicalAndOperator() + "test" + getTitleSuffix()); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Test public void notOperatorSufix() throws QueryNodeParseException { - String queryString = "!(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\")"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + String queryString = "!(author=\"Igor Steinmacher\" OR author=\"Christoph Treude\")"; + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of(getTransformer().getLogicalNotOperator() + "(" + "\"Igor Steinmacher\"" + getAuthorSuffix() + getTransformer().getLogicalOrOperator() + "\"Christoph Treude\")" + getAuthorSuffix()); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearAndYearRangeByFilteringQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearAndYearRangeByFilteringQueryTransformerTest.java index 8811acd59c4..0f07265398f 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearAndYearRangeByFilteringQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearAndYearRangeByFilteringQueryTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -14,9 +16,10 @@ public abstract class YearAndYearRangeByFilteringQueryTransformerTest query = transformer.transformLuceneQuery(luceneQuery); + String queryString = "year=2021"; + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); assertEquals(Optional.empty(), query); assertEquals(Optional.of(2021), transformer.getStartYear()); assertEquals(Optional.of(2021), transformer.getEndYear()); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearRangeByFilteringQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearRangeByFilteringQueryTransformerTest.java index 5f56c79d17d..4764c943f81 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearRangeByFilteringQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearRangeByFilteringQueryTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -17,13 +19,14 @@ public void convertYearRangeField() throws QueryNodeParseException { YearRangeByFilteringQueryTransformer transformer = getTransformer(); String queryString = "year-range:2018-2021"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional result = transformer.transformLuceneQuery(luceneQuery); + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); // The API does not support querying for a year range // The implementation of the fetcher filters the results manually - assertEquals(Optional.empty(), result); + assertEquals(Optional.empty(), query); // The implementation sets the start year and end year values according to the query assertEquals(Optional.of(2018), transformer.getStartYear()); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformerTest.java index 583a0b56280..d166a14fdbf 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformerTest.java @@ -2,9 +2,11 @@ import java.util.Optional; +import org.jabref.logic.search.query.SearchQueryVisitor; +import org.jabref.model.search.query.BaseQueryNode; +import org.jabref.model.search.query.SearchQuery; + import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -39,20 +41,22 @@ public String getTitlePrefix() { @Override @Test public void convertYearField() throws QueryNodeParseException { - String queryString = "year:2015"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + String queryString = "year=2015"; + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("py:2015"); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } @Override @Test public void convertYearRangeField() throws QueryNodeParseException { - String queryString = "year-range:2012-2015"; - QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional searchQuery = getTransformer().transformLuceneQuery(luceneQuery); + String queryString = "year-range=2012-2015"; + SearchQuery searchQuery = new SearchQuery(queryString); + BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); + Optional query = getTransformer().transformSearchQuery(searchQueryList); Optional expected = Optional.of("py:2012-2015"); - assertEquals(expected, searchQuery); + assertEquals(expected, query); } } From 179b3a0c561e06e95ca2fd8adf772196c154f57a Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sat, 23 Aug 2025 20:41:19 +0300 Subject: [PATCH 29/37] Updated ANTLR parser grammar and updated the transformer testers to be compatible with ANTLR-based parser logic. --- .../main/antlr/org/jabref/search/Search.g4 | 2 +- .../ArXivQueryTransformerTest.java | 8 ++-- .../CiteSeerQueryTransformerTest.java | 32 +++++++-------- ...nceBibliographiesQueryTransformerTest.java | 8 ++-- .../DBLPQueryTransformerTest.java | 14 +++---- .../transformers/GVKQueryTransformerTest.java | 6 +-- .../IEEEQueryTransformerTest.java | 22 +++++----- .../transformers/InfixTransformerTest.java | 41 +++++++++++-------- .../JstorQueryTransformerTest.java | 10 ++--- .../SpringerQueryTransformerTest.java | 10 ++--- ...rRangeByFilteringQueryTransformerTest.java | 6 +-- ...rRangeByFilteringQueryTransformerTest.java | 8 ++-- 12 files changed, 87 insertions(+), 80 deletions(-) diff --git a/jablib/src/main/antlr/org/jabref/search/Search.g4 b/jablib/src/main/antlr/org/jabref/search/Search.g4 index ca5d3b570a8..6fdf09ee099 100644 --- a/jablib/src/main/antlr/org/jabref/search/Search.g4 +++ b/jablib/src/main/antlr/org/jabref/search/Search.g4 @@ -39,7 +39,7 @@ CONTAINS: 'CONTAINS'; MATCHES: 'MATCHES'; NOT: 'NOT'; -FIELD: [A-Z]+; +FIELD: [A-Z]([A-Z] | '-' | '_')*; // field name should allow for - or _ STRING_LITERAL: '"' ('\\"' | ~["])* '"'; // " should be escaped with a backslash TERM: ('\\' [=!~()] | ~[ \t\n\r=!~()])+; // =!~() should be escaped with a backslash diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformerTest.java index a20280a2bd5..fddf7e0acf3 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,12 +40,12 @@ public String getTitlePrefix() { @Override @Test - public void convertYearField() throws QueryNodeParseException { + public void convertYearField() throws ParseCancellationException { ArXivQueryTransformer transformer = getTransformer(); - String queryString = "year:2018"; + String queryString = "year=2018"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); - Optional query = getTransformer().transformSearchQuery(searchQueryList); + Optional query = transformer.transformSearchQuery(searchQueryList); assertEquals(Optional.of("2018"), query); assertEquals(Optional.of(2018), transformer.getStartYear()); assertEquals(Optional.of(2018), transformer.getEndYear()); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CiteSeerQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CiteSeerQueryTransformerTest.java index 04544b76968..89788e8e0bc 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CiteSeerQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CiteSeerQueryTransformerTest.java @@ -12,7 +12,7 @@ import org.jabref.model.strings.StringUtil; import kong.unirest.core.json.JSONObject; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -29,8 +29,8 @@ protected CiteSeerQueryTransformer getTransformer() { @Override @Test - public void convertYearField() throws QueryNodeParseException { - String queryString = "year:2023"; + public void convertYearField() throws ParseCancellationException { + String queryString = "year=2023"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); @@ -44,8 +44,8 @@ public void convertYearField() throws QueryNodeParseException { @Override @Test - public void convertYearRangeField() throws QueryNodeParseException { - String queryString = "year-range:2019-2023"; + public void convertYearRangeField() throws ParseCancellationException { + String queryString = "year-range=2019-2023"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); @@ -58,8 +58,8 @@ public void convertYearRangeField() throws QueryNodeParseException { } @Test - void convertPageField() throws QueryNodeParseException { - String queryString = "page:2"; + void convertPageField() throws ParseCancellationException { + String queryString = "page=2"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); @@ -70,8 +70,8 @@ void convertPageField() throws QueryNodeParseException { } @Test - void convertPageSizeField() throws QueryNodeParseException { - String queryString = "pageSize:20"; + void convertPageSizeField() throws ParseCancellationException { + String queryString = "pageSize=20"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); @@ -82,8 +82,8 @@ void convertPageSizeField() throws QueryNodeParseException { } @Test - void convertSortByField() throws QueryNodeParseException { - String queryString = "sortBy:relevance"; + void convertSortByField() throws ParseCancellationException { + String queryString = "sortBy=relevance"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); @@ -94,8 +94,8 @@ void convertSortByField() throws QueryNodeParseException { } @Test - void convertMultipleAuthors() throws QueryNodeParseException { - String queryString = "author:\"Wang Wei\" author:\"Zhang Pingwen\" author:\"Zhang Zhifei\""; + void convertMultipleAuthors() throws ParseCancellationException { + String queryString = "author=\"Wang Wei\" author=\"Zhang Pingwen\" author=\"Zhang Zhifei\""; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); CiteSeerQueryTransformer transformer = getTransformer(); @@ -107,10 +107,10 @@ void convertMultipleAuthors() throws QueryNodeParseException { } private static Stream getJSONWithYearVariations() { - String baseString = "title:Ericksen-Leslie page:1 pageSize:20 must_have_pdf:false sortBy:relevance"; + String baseString = "title=Ericksen-Leslie page=1 pageSize=20 must_have_pdf=false sortBy=relevance"; List withYearAndYearRange = List.of( - StringUtil.join(new String[]{baseString, "year:2020"}, " ", 0, 2), - StringUtil.join(new String[]{baseString, "year-range:2019-2023"}, " ", 0, 2) + StringUtil.join(new String[]{baseString, "year=2020"}, " ", 0, 2), + StringUtil.join(new String[]{baseString, "year-range=2019-2023"}, " ", 0, 2) ); JSONObject expectedJson = new JSONObject(); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java index 744b53baf49..7d7ab8a7ce9 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,7 +40,7 @@ public String getTitlePrefix() { @Override @Test - public void convertYearField() throws QueryNodeParseException { + public void convertYearField() throws ParseCancellationException { String queryString = "year:2018"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); @@ -50,8 +50,8 @@ public void convertYearField() throws QueryNodeParseException { @Override @Test - public void convertYearRangeField() throws QueryNodeParseException { - String queryString = "year-range:2018-2021"; + public void convertYearRangeField() throws ParseCancellationException { + String queryString = "year-range=2018-2021"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformerTest.java index 0c7663690a1..cebc1704529 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,12 +40,12 @@ public String getTitlePrefix() { @Override @Test - public void convertYearField() throws QueryNodeParseException { - String queryString = "year:2015"; + public void convertYearField() throws ParseCancellationException { + String queryString = "year=2015"; DBLPQueryTransformer transformer = getTransformer(); SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); - Optional query = getTransformer().transformSearchQuery(searchQueryList); + Optional query = transformer.transformSearchQuery(searchQueryList); assertEquals(Optional.empty(), query); assertEquals(Optional.of(2015), transformer.getStartYear()); assertEquals(Optional.of(2015), transformer.getEndYear()); @@ -53,12 +53,12 @@ public void convertYearField() throws QueryNodeParseException { @Override @Test - public void convertYearRangeField() throws QueryNodeParseException { - String queryString = "year-range:2012-2015"; + public void convertYearRangeField() throws ParseCancellationException { + String queryString = "year-range=2012-2015"; DBLPQueryTransformer transformer = getTransformer(); SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); - Optional query = getTransformer().transformSearchQuery(searchQueryList); + Optional query = transformer.transformSearchQuery(searchQueryList); assertEquals(Optional.empty(), query); assertEquals(Optional.of(2012), transformer.getStartYear()); assertEquals(Optional.of(2015), transformer.getEndYear()); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformerTest.java index 8964f7c5e60..7395ab0d620 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -41,8 +41,8 @@ public String getTitlePrefix() { @Override @Test - public void convertYearField() throws QueryNodeParseException { - String queryString = "year:2018"; + public void convertYearField() throws ParseCancellationException { + String queryString = "year=2018"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java index 9eafcf8a5de..932c0444389 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java @@ -7,7 +7,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -44,29 +44,29 @@ public String getTitlePrefix() { @Override @Test - public void convertJournalFieldPrefix() throws QueryNodeParseException { + public void convertJournalFieldPrefix() throws ParseCancellationException { IEEEQueryTransformer transformer = getTransformer(); - String queryString = "journal:Nature"; + String queryString = "journal=Nature"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); - getTransformer().transformSearchQuery(searchQueryList); + transformer.transformSearchQuery(searchQueryList); assertEquals(Optional.of("Nature"), transformer.getJournal()); } @Override @Test - public void convertYearField() throws QueryNodeParseException { + public void convertYearField() throws ParseCancellationException { // IEEE does not support year range // Thus, a generic test does not work IEEEQueryTransformer transformer = getTransformer(); - String queryString = "year:2021"; + String queryString = "year=2021"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); - getTransformer().transformSearchQuery(searchQueryList); + transformer.transformSearchQuery(searchQueryList); assertEquals(Optional.of(2021), transformer.getStartYear()); assertEquals(Optional.of(2021), transformer.getEndYear()); @@ -74,13 +74,13 @@ public void convertYearField() throws QueryNodeParseException { @Override @Test - public void convertYearRangeField() throws QueryNodeParseException { + public void convertYearRangeField() throws ParseCancellationException { IEEEQueryTransformer transformer = getTransformer(); - String queryString = "year-range:2018-2021"; + String queryString = "year-range=2018-2021"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); - getTransformer().transformSearchQuery(searchQueryList); + transformer.transformSearchQuery(searchQueryList); assertEquals(Optional.of(2018), transformer.getStartYear()); assertEquals(Optional.of(2021), transformer.getEndYear()); @@ -96,7 +96,7 @@ private static Stream getTitleTestData() { @ParameterizedTest @MethodSource("getTitleTestData") - void stopWordRemoval(String expected, String queryString) throws QueryNodeParseException { + void stopWordRemoval(String expected, String queryString) throws ParseCancellationException { SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/InfixTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/InfixTransformerTest.java index 41ac49ca607..482e8feefb4 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/InfixTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/InfixTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -39,8 +39,8 @@ protected String getTitlePrefix() { } @Test - public void convertAuthorFieldPrefix() throws QueryNodeParseException { - String queryString = "author:\"Igor Steinmacher\""; + public void convertAuthorFieldPrefix() throws ParseCancellationException { + String queryString = "author=\"Igor Steinmacher\""; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); @@ -49,7 +49,7 @@ public void convertAuthorFieldPrefix() throws QueryNodeParseException { } @Test - public void convertUnFieldedTermPrefix() throws QueryNodeParseException { + public void convertUnFieldedTermPrefix() throws ParseCancellationException { String queryString = "\"default value\""; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); @@ -59,8 +59,8 @@ public void convertUnFieldedTermPrefix() throws QueryNodeParseException { } @Test - public void convertExplicitUnFieldedTermPrefix() throws QueryNodeParseException { - String queryString = "default:\"default value\""; + public void convertExplicitUnFieldedTermPrefix() throws ParseCancellationException { + String queryString = "default=\"default value\""; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); @@ -69,8 +69,8 @@ public void convertExplicitUnFieldedTermPrefix() throws QueryNodeParseException } @Test - public void convertJournalFieldPrefix() throws QueryNodeParseException { - String queryString = "journal:Nature"; + public void convertJournalFieldPrefix() throws ParseCancellationException { + String queryString = "journal=Nature"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); @@ -79,14 +79,14 @@ public void convertJournalFieldPrefix() throws QueryNodeParseException { } @Test - public abstract void convertYearField() throws QueryNodeParseException; + public abstract void convertYearField() throws ParseCancellationException; @Test - public abstract void convertYearRangeField() throws QueryNodeParseException; + public abstract void convertYearRangeField() throws ParseCancellationException; @Test - public void convertMultipleValuesWithTheSameFieldPrefix() throws QueryNodeParseException { - String queryString = "author:\"Igor Steinmacher\" author:\"Christoph Treude\""; + public void convertMultipleValuesWithTheSameFieldPrefix() throws ParseCancellationException { + String queryString = "author=\"Igor Steinmacher\" author=\"Christoph Treude\""; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); @@ -95,8 +95,8 @@ public void convertMultipleValuesWithTheSameFieldPrefix() throws QueryNodeParseE } @Test - public void groupedOperationsPrefix() throws QueryNodeParseException { - String queryString = "(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND title:test"; + public void groupedOperationsPrefix() throws ParseCancellationException { + String queryString = "(author=\"Igor Steinmacher\" OR author=\"Christoph Treude\" AND author=\"Christoph Freunde\") AND title=test"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); @@ -105,12 +105,19 @@ public void groupedOperationsPrefix() throws QueryNodeParseException { } @Test - public void notOperatorPrefix() throws QueryNodeParseException { - String queryString = "!(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\")"; + public void notOperatorPrefix() throws ParseCancellationException { + String queryString = "NOT (author=\"Igor Steinmacher\" OR author=\"Christoph Treude\")"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); - Optional expected = Optional.of(getTransformer().getLogicalNotOperator() + "(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformer().getLogicalOrOperator() + getAuthorPrefix() + "\"Christoph Treude\")"); + + // If there is no not operator for this transformer, the parentheses will be at the start and end of the string and will be removed + Optional expected; + if (getTransformer().getLogicalNotOperator().isEmpty()) { + expected = Optional.of(getTransformer().getLogicalNotOperator() + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformer().getLogicalOrOperator() + getAuthorPrefix() + "\"Christoph Treude\""); + } else { + expected = Optional.of(getTransformer().getLogicalNotOperator() + "(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformer().getLogicalOrOperator() + getAuthorPrefix() + "\"Christoph Treude\")"); + } assertEquals(expected, query); } } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/JstorQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/JstorQueryTransformerTest.java index bee8a38b83e..d0490820b72 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/JstorQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/JstorQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,8 +40,8 @@ public String getTitlePrefix() { @Override @Test - public void convertYearField() throws QueryNodeParseException { - String queryString = "year:2018"; + public void convertYearField() throws ParseCancellationException { + String queryString = "year=2018"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); @@ -50,8 +50,8 @@ public void convertYearField() throws QueryNodeParseException { @Override @Test - public void convertYearRangeField() throws QueryNodeParseException { - String queryString = "year-range:2018-2021"; + public void convertYearRangeField() throws ParseCancellationException { + String queryString = "year-range=2018-2021"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SpringerQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SpringerQueryTransformerTest.java index 57306f48c55..20c248a2927 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SpringerQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/SpringerQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,8 +40,8 @@ public String getTitlePrefix() { @Override @Test - public void convertYearField() throws QueryNodeParseException { - String queryString = "year:2015"; + public void convertYearField() throws ParseCancellationException { + String queryString = "year=2015"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); @@ -52,8 +52,8 @@ public void convertYearField() throws QueryNodeParseException { @Override @Test - public void convertYearRangeField() throws QueryNodeParseException { - String queryString = "year-range:2012-2015"; + public void convertYearRangeField() throws ParseCancellationException { + String queryString = "year-range=2012-2015"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearAndYearRangeByFilteringQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearAndYearRangeByFilteringQueryTransformerTest.java index 0f07265398f..849202dee50 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearAndYearRangeByFilteringQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearAndYearRangeByFilteringQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -14,12 +14,12 @@ public abstract class YearAndYearRangeByFilteringQueryTransformerTest extends YearRangeByFilteringQueryTransformerTest { @Override @Test - public void convertYearField() throws QueryNodeParseException { + public void convertYearField() throws ParseCancellationException { YearAndYearRangeByFilteringQueryTransformer transformer = getTransformer(); String queryString = "year=2021"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); - Optional query = getTransformer().transformSearchQuery(searchQueryList); + Optional query = transformer.transformSearchQuery(searchQueryList); assertEquals(Optional.empty(), query); assertEquals(Optional.of(2021), transformer.getStartYear()); assertEquals(Optional.of(2021), transformer.getEndYear()); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearRangeByFilteringQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearRangeByFilteringQueryTransformerTest.java index 4764c943f81..b2bb893e1d1 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearRangeByFilteringQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/YearRangeByFilteringQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -15,13 +15,13 @@ public abstract class YearRangeByFilteringQueryTransformerTest query = getTransformer().transformSearchQuery(searchQueryList); + Optional query = transformer.transformSearchQuery(searchQueryList); // The API does not support querying for a year range // The implementation of the fetcher filters the results manually From b78d8cc88f0d47df4f9256e7c3f6bd75a59ae386 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sat, 23 Aug 2025 20:51:03 +0300 Subject: [PATCH 30/37] Updated ZbMathQueryTransformerTest to throw the appropriate exception --- .../fetcher/transformers/ZbMathQueryTransformerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformerTest.java index d166a14fdbf..ebc22b2367f 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.jabref.model.search.query.BaseQueryNode; import org.jabref.model.search.query.SearchQuery; -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.antlr.v4.runtime.misc.ParseCancellationException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -40,7 +40,7 @@ public String getTitlePrefix() { @Override @Test - public void convertYearField() throws QueryNodeParseException { + public void convertYearField() throws ParseCancellationException { String queryString = "year=2015"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); @@ -51,7 +51,7 @@ public void convertYearField() throws QueryNodeParseException { @Override @Test - public void convertYearRangeField() throws QueryNodeParseException { + public void convertYearRangeField() throws ParseCancellationException { String queryString = "year-range=2012-2015"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); From 1bf6d9983330ed29aca2e2416b165468e5364eef Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sat, 23 Aug 2025 21:56:25 +0300 Subject: [PATCH 31/37] IEEEQueryTransformerTest wont check for the case where the string is just "and" as this is not valid syntax and throws an exception when parsing --- .../fetcher/transformers/IEEEQueryTransformerTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java index 932c0444389..2d5caf7cdf2 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java @@ -89,8 +89,7 @@ public void convertYearRangeField() throws ParseCancellationException { private static Stream getTitleTestData() { return Stream.of( Arguments.of("Overcoming AND Open AND Source AND Project AND Entry AND Barriers AND Portal AND Newcomers", "Overcoming Open Source Project Entry Barriers with a Portal for Newcomers"), - Arguments.of("Overcoming AND Open AND Source AND Project AND Entry AND Barriers", "Overcoming Open Source Project Entry Barriers"), - Arguments.of(null, "and") + Arguments.of("Overcoming AND Open AND Source AND Project AND Entry AND Barriers", "Overcoming Open Source Project Entry Barriers") ); } From a56066a3abe654af48cd1577641bd507a2568047 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sat, 23 Aug 2025 22:15:03 +0300 Subject: [PATCH 32/37] SearchBasedFetcher performSearch function now throws fetcher exception if query is not valid --- .../java/org/jabref/logic/importer/SearchBasedFetcher.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index 4d4eac7e3fb..461bf76ea23 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -38,8 +38,11 @@ default List performSearch(String searchQuery) throws FetcherException } SearchQuery searchQueryObject = new SearchQuery(searchQuery); - SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); + if (!searchQueryObject.isValid()) { + throw new FetcherException("The query is not valid"); + } BaseQueryNode queryNode; + SearchQueryVisitor visitor = new SearchQueryVisitor(searchQueryObject.getSearchFlags()); try { queryNode = visitor.visitStart(searchQueryObject.getContext()); } catch (ParseCancellationException e) { From 70988c1d700ad323d7118c1aee1687df003f777d Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sat, 23 Aug 2025 22:44:49 +0300 Subject: [PATCH 33/37] Changed description for function getURLForQuery --- .../java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java index 3958a62c89d..cc4edaf359f 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/LOBIDFetcher.java @@ -46,7 +46,7 @@ public class LOBIDFetcher implements PagedSearchBasedParserFetcher, IdBasedParse /** * Gets the query URL * - * @param queryNode the list that contains the parsed nodes + * @param queryNode the first parsed node * @param pageNumber the number of the page indexed from 0 * @return URL */ From 138746f9293d1ccb8caa5ce427b0c5cd9e7b60b1 Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sat, 23 Aug 2025 23:31:58 +0300 Subject: [PATCH 34/37] Changed how "not comparison expressions" are handled --- .../search/query/SearchQueryVisitor.java | 27 ++++++++++--------- ...nceBibliographiesQueryTransformerTest.java | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/search/query/SearchQueryVisitor.java b/jablib/src/main/java/org/jabref/logic/search/query/SearchQueryVisitor.java index 529f87e9f36..a21a828a2b1 100644 --- a/jablib/src/main/java/org/jabref/logic/search/query/SearchQueryVisitor.java +++ b/jablib/src/main/java/org/jabref/logic/search/query/SearchQueryVisitor.java @@ -72,21 +72,8 @@ public BaseQueryNode visitComparisonExpression(SearchParser.ComparisonExpression @Override public BaseQueryNode visitComparison(SearchParser.ComparisonContext ctx) { - if (ctx.operator() != null) { - int operator = ctx.operator().getStart().getType(); - if (operator == SearchParser.NEQUAL - || operator == SearchParser.NCEQUAL - || operator == SearchParser.NEEQUAL - || operator == SearchParser.NCEEQUAL - || operator == SearchParser.NREQUAL - || operator == SearchParser.NCREEQUAL) { - return null; - } - } String term = SearchQueryConversion.unescapeSearchValue(ctx.searchValue()); - // if not regex, escape the backslashes, because the highlighter uses regex - // unfielded terms, check the search bar flags if (ctx.FIELD() == null) { if (!searchBarRegex) { @@ -115,6 +102,20 @@ public BaseQueryNode visitComparison(SearchParser.ComparisonContext ctx) { if ("any".equals(field)) { return new SearchQueryNode(Optional.empty(), term); } + + if (ctx.operator() != null) { + int operator = ctx.operator().getStart().getType(); + if (operator == SearchParser.NEQUAL + || operator == SearchParser.NCEQUAL + || operator == SearchParser.NEEQUAL + || operator == SearchParser.NCEEQUAL + || operator == SearchParser.NREQUAL + || operator == SearchParser.NCREEQUAL) { + // All of these will be treated as != + SearchQueryNode negatedNode = new SearchQueryNode(Optional.of(FieldFactory.parseField(field)), term); + return new NotNode(negatedNode); + } + } return new SearchQueryNode(Optional.of(FieldFactory.parseField(field)), term); } } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java index 7d7ab8a7ce9..36939b9a823 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformerTest.java @@ -41,7 +41,7 @@ public String getTitlePrefix() { @Override @Test public void convertYearField() throws ParseCancellationException { - String queryString = "year:2018"; + String queryString = "year=2018"; SearchQuery searchQuery = new SearchQuery(queryString); BaseQueryNode searchQueryList = new SearchQueryVisitor(searchQuery.getSearchFlags()).visitStart(searchQuery.getContext()); Optional query = getTransformer().transformSearchQuery(searchQueryList); From e83c1fede092e39be327037101b58c357b60d9ee Mon Sep 17 00:00:00 2001 From: RunningEscaping Date: Sun, 24 Aug 2025 21:09:20 +0300 Subject: [PATCH 35/37] Added the change to CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 576b6b29c87..a619c91a942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv - We introduced walkthrough functionality [#12664](https://github.com/JabRef/jabref/issues/12664) - The Welcome tab now has a responsive layout. [#12664](https://github.com/JabRef/jabref/issues/12664) - We introduced a donation prompt in the Welcome tab. [#12664](https://github.com/JabRef/jabref/issues/12664) +- We changed the web-search fetchers to use an ANTLR based parser instead of lucene parser [#13607](https://github.com/JabRef/jabref/issues/13607) ### Fixed From b8c190f834707c0a6d62bf9163582bf863e072ff Mon Sep 17 00:00:00 2001 From: turhantolgaunal Date: Mon, 25 Aug 2025 02:06:35 +0300 Subject: [PATCH 36/37] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a619c91a942..4e3084fbfef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,7 +75,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv - We introduced walkthrough functionality [#12664](https://github.com/JabRef/jabref/issues/12664) - The Welcome tab now has a responsive layout. [#12664](https://github.com/JabRef/jabref/issues/12664) - We introduced a donation prompt in the Welcome tab. [#12664](https://github.com/JabRef/jabref/issues/12664) -- We changed the web-search fetchers to use an ANTLR based parser instead of lucene parser [#13607](https://github.com/JabRef/jabref/issues/13607) +- We changed to syntax for the websearch to the one of the main search bar. [#13607](https://github.com/JabRef/jabref/issues/13607) ### Fixed From 9f4953d316bcde047831de90e0bff626c7eb0f55 Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Mon, 25 Aug 2025 22:13:43 +0200 Subject: [PATCH 37/37] Reword --- .../java/org/jabref/logic/importer/fetcher/ResearchGate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java index 4f0a56b324f..d4bd1e561aa 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ResearchGate.java @@ -203,7 +203,7 @@ public TrustLevel getTrustLevel() { /** * This method is used to send complex queries using fielded search. * - * @param queryNode the first node from the Search.g4 parser + * @param queryNode the first node from the search parser * @return a list of {@link BibEntry}, which are matched by the query (maybe empty) * @throws FetcherException if the ResearchGate refuses to serve the page */