diff --git a/documentation/src/main/resources/openapi/sources/paths/search/things-count.yml b/documentation/src/main/resources/openapi/sources/paths/search/things-count.yml index fc61698003..3fd9a6bbe5 100644 --- a/documentation/src/main/resources/openapi/sources/paths/search/things-count.yml +++ b/documentation/src/main/resources/openapi/sources/paths/search/things-count.yml @@ -13,14 +13,11 @@ get: description: |- This resource can be used to count things. - The query parameter `filter` is not mandatory. If it is not set there is - returned the total amount of things which the logged in user is allowed - to read. + The query parameter `filter` is not mandatory. If it is not set there is returned the total amount of things + which the logged in user is allowed to read. - To search for nested properties, we use JSON Pointer notation - (RFC-6901). See the following example how to search for the sub property - `location` of the parent property `attributes` with a forward slash as - separator: + To search for nested properties, we use JSON Pointer notation (RFC-6901). See the following example how to search + for the sub property `location` of the parent property `attributes` with a forward slash as separator: ```eq(attributes/location,"kitchen")``` parameters: @@ -38,8 +35,7 @@ get: type: integer '400': description: |- - The request could not be completed. A provided parameter is in a - wrong format. + The request could not be completed. A provided parameter is in a wrong format. content: application/json: schema: diff --git a/documentation/src/main/resources/openapi/sources/paths/search/things.yml b/documentation/src/main/resources/openapi/sources/paths/search/things.yml index dc9b2a7c66..b26476fa3c 100644 --- a/documentation/src/main/resources/openapi/sources/paths/search/things.yml +++ b/documentation/src/main/resources/openapi/sources/paths/search/things.yml @@ -20,11 +20,10 @@ get: spelling of value of the namespace, name, attribute, feature etc. is, use the *like* notation instead of *eq* for filtering. - * The resource supports sorting and paging. If paging is not explicitly - specified by means of the `size` option, a default count of `25` - documents is returned. + * The resource supports sorting and paging. If paging is not explicitly specified by means of the `size` option, + a default count of `25` documents is returned. - * The internal search index is "eventually consistent". Consistency with the latest + * The internal search index is "eventually consistent". Consistency with the latest thing updates should recover within milliseconds. parameters: - $ref: '../../parameters/searchFilter.yml' @@ -48,9 +47,6 @@ get: quotation marks. Cursor IDs are given in search responses and mark the position after the last entry of the previous search. The meaning of cursor IDs is unspecified and may change without notice. - The paging option `limit({offset},{count})` is deprecated. - It may result in slow queries or timeouts and will be removed eventually. - #### Examples: * ```sort(+thingId)``` @@ -67,7 +63,6 @@ get: ```size(200),cursor(LOREMIPSUM)``` - The deprecated paging option `limit` may not be combined with the other paging options `size` and `cursor`. required: false schema: type: string @@ -82,8 +77,7 @@ get: $ref: '../../schemas/search/searchResultThings.yml' '400': description: |- - The request could not be completed. A provided parameter is in a - wrong format. + The request could not be completed. A provided parameter is in a wrong format. content: application/json: schema: diff --git a/documentation/src/main/resources/pages/ditto/basic-search.md b/documentation/src/main/resources/pages/ditto/basic-search.md index 91dbfb7d6d..73aaec9c3d 100644 --- a/documentation/src/main/resources/pages/ditto/basic-search.md +++ b/documentation/src/main/resources/pages/ditto/basic-search.md @@ -17,8 +17,8 @@ The functionality is available for the following APIs. ## Search index Ditto's microservice [things-search](architecture-services-things-search.html) automatically consumes all -[events](basic-signals-event.html) which are emitted for changes to `Things` and `Policies` and updates an for search -optimized representation of the `Thing` data into its own database. +[events](basic-signals-event.html) which are emitted for changes to `Things` and `Policies` and updates the search +index which contains an optimized representation of the `Thing` data. The search service has its own database. No custom indexes have to be defined as the structure in the database is "flattened" so that all data contained in [Things](basic-thing.html) can be searched for efficiently. @@ -58,7 +58,7 @@ Paging: size(5),cursor(CURSOR_ID) ## Search count queries The same syntax applies for search count queries - only the [sorting](basic-rql.html#rql-sorting) and -[paging](#rql-paging-deprecated) makes no sense here, so there are not necessary to specify. +[paging](#rql-paging-deprecated) makes no sense here, so they are not necessary to specify. ## Namespaces @@ -107,32 +107,3 @@ of the cursor. Otherwise, the request is rejected. option=size(10),cursor() ``` -## RQL paging (deprecated) - -{% include note.html content="The limit option is deprecated, it may be removed in future releases. Use [cursor-based -paging](basic-search.html#sorting-and-paging-options) instead." %} - -The RQL limiting part specifies which part (or "page") should be returned of a large search result set. - -``` -limit(,) -``` - -Limits the search results to `` items, starting with the item at index ``. -* if the paging option is not explicitly specified, the **default** value `limit(0,25)` is used, - i.e. the first `25` results are returned. -* the **maximum** allowed count is `200`. - -**Example - return the first ten items** -``` -limit(0,10) -``` - -**Example - return the items 11 to 20** -``` -limit(10,10) -``` -i.e. Return the next ten items (from index 11 to 20) - -{% include note.html content="We recommend **not to use high offsets** (e.g. higher than 10000) for paging - because of potential performance degradations." %} diff --git a/documentation/src/main/resources/pages/ditto/httpapi-search.md b/documentation/src/main/resources/pages/ditto/httpapi-search.md index cc9d7bd6af..038a50fd43 100644 --- a/documentation/src/main/resources/pages/ditto/httpapi-search.md +++ b/documentation/src/main/resources/pages/ditto/httpapi-search.md @@ -8,11 +8,11 @@ permalink: httpapi-search.html The [search aspect](basic-search.html) of Ditto can be accessed via an HTTP API. {% include note.html content="Find the HTTP API reference at the - [Search resources](http-api-doc.html?urls.primaryName=api2#/Search)." %} +[Search resources](http-api-doc.html?urls.primaryName=api2#/Search)." %} -The concepts of the [RQL expression](basic-rql.html#rql-filter), [RQL sorting](basic-rql.html#rql-sorting) and -[RQL paging](basic-search.html#rql-paging-deprecated) are mapped to HTTP as query parameters which are added to -`GET` requests to the search endpoint: +The concepts of the [RQL expression](basic-rql.html#rql-filter) and +[RQL sorting and paging options](basic-search.html#sorting-and-paging-options) are mapped to HTTP as query parameters +which are added to `GET` requests to the search endpoint: ``` http://localhost:8080/api/2/search/things @@ -32,8 +32,7 @@ Default values of each option is documented [here](basic-search.html#sorting-and Complex example: ``` -GET .../search/things?filter=eq(attributes/location,"living-room")&option=sort(+thingId),limit(0,5)&namespaces=org -.eclipse.ditto,foo.bar +GET .../search/things?filter=eq(attributes/location,"living-room")&option=sort(+thingId),cursor(CURSOR_ID),size(10)&namespaces=org.eclipse.ditto,foo.bar ``` Another Complex example with the `namespaces` parameter: diff --git a/documentation/src/main/resources/pages/ditto/protocol-specification-things-search.md b/documentation/src/main/resources/pages/ditto/protocol-specification-things-search.md index ba4bdee30e..4960957a55 100644 --- a/documentation/src/main/resources/pages/ditto/protocol-specification-things-search.md +++ b/documentation/src/main/resources/pages/ditto/protocol-specification-things-search.md @@ -97,9 +97,9 @@ In particular: - When given in the `options` field, `sort(<+"->, ...)` sets the order of search results. If not given, the default `sort(+thingId)` is used. -The paging options `cursor` and `limit` of the [HTTP-API](httpapi-search.html) are not supported here, because -they are not meaningful for the search protocol. For the HTTP-API, those options are for iterating through large -numbers of search results over many HTTP requests in a stateless manner. +The paging option `cursor` of the [HTTP-API](httpapi-search.html) is not supported here, because +it is not meaningful for the search protocol. For the HTTP-API, the option is for iterating through a large +number of search results over many HTTP requests in a stateless manner. The search protocol is not stateless and does not require the client to keep track of any cursor or offset; results of any size are streamed over an arbitrarily long period of time. diff --git a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/proxy/actors/QueryThingsPerRequestActor.java b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/proxy/actors/QueryThingsPerRequestActor.java index 57915f8a24..797aa6b776 100644 --- a/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/proxy/actors/QueryThingsPerRequestActor.java +++ b/gateway/service/src/main/java/org/eclipse/ditto/gateway/service/proxy/actors/QueryThingsPerRequestActor.java @@ -145,7 +145,6 @@ public Receive createReceive() { final JsonArray retrievedEntitiesWithFieldSelection = getEntitiesWithSelectedFields(rtrEntity); final SearchResult resultWithRetrievedItems = SearchModelFactory.newSearchResultBuilder() .addAll(retrievedEntitiesWithFieldSelection) - .nextPageOffset(queryThingsResponse.getSearchResult().getNextPageOffset().orElse(null)) .cursor(queryThingsResponse.getSearchResult().getCursor().orElse(null)) .build(); final QueryThingsResponse theQueryThingsResponse = diff --git a/internal/utils/config/src/main/resources/ditto-limits.conf b/internal/utils/config/src/main/resources/ditto-limits.conf index b0d00f053b..0a09692a6d 100644 --- a/internal/utils/config/src/main/resources/ditto-limits.conf +++ b/internal/utils/config/src/main/resources/ditto-limits.conf @@ -34,13 +34,13 @@ ditto.limits { auth-subjects-count = ${?LIMITS_MESSAGES_AUTH_SUBJECTS_COUNT} } - # limiations for the "things-search" service + # limitations for the "things-search" service search { default-page-size = 25 default-page-size = ${?LIMITS_THINGS_SEARCH_DEFAULT_PAGE_SIZE} # the allowed maximum page size limit - e.g. specified when doing a search via HTTP: - # /api/1/search/things?filter=...&option=limit(0,200) + # /api/2/search/things?filter=...&option=size(200) max-page-size = 200 max-page-size = ${?LIMITS_THINGS_SEARCH_MAX_PAGE_SIZE} } diff --git a/internal/utils/search/src/test/java/org/eclipse/ditto/internal/utils/search/SearchSourceBuilderTest.java b/internal/utils/search/src/test/java/org/eclipse/ditto/internal/utils/search/SearchSourceBuilderTest.java index 78d6001c2e..e0f81789dc 100644 --- a/internal/utils/search/src/test/java/org/eclipse/ditto/internal/utils/search/SearchSourceBuilderTest.java +++ b/internal/utils/search/src/test/java/org/eclipse/ditto/internal/utils/search/SearchSourceBuilderTest.java @@ -73,11 +73,6 @@ public void failWithMissingDittoHeaders() { .build(); } - @Test(expected = InvalidOptionException.class) - public void failOnLimitOption() { - SearchSource.newBuilder().options("limit(0,1)"); - } - @Test(expected = InvalidOptionException.class) public void failOnCursorOption() { SearchSource.newBuilder().options("cursor(ABC)"); diff --git a/rql/query/pom.xml b/rql/query/pom.xml index 52a2e4f0a1..fb274dc022 100644 --- a/rql/query/pom.xml +++ b/rql/query/pom.xml @@ -109,6 +109,10 @@ org.eclipse.ditto.rql.query.LikeHelper + org.eclipse.ditto.rql.query.Query#getLimit() + org.eclipse.ditto.rql.query.Query#getSkip() + org.eclipse.ditto.rql.query.QueryBuilder#limit(long) + org.eclipse.ditto.rql.query.QueryBuilder#skip(long) diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/Query.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/Query.java index 5790b9543a..4774fbaf94 100644 --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/Query.java +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/Query.java @@ -32,14 +32,9 @@ public interface Query { List getSortOptions(); /** - * @return the number of results to which this query is limited. + * @return the number of results for this query. */ - int getLimit(); - - /** - * @return the number of results which are discarded from the beginning of this query. - */ - int getSkip(); + int getSize(); /** * Replace criteria of this query by another. @@ -49,4 +44,5 @@ public interface Query { * @since 1.1.0 */ Query withCriteria(Criteria criteria); + } diff --git a/rql/query/src/main/java/org/eclipse/ditto/rql/query/QueryBuilder.java b/rql/query/src/main/java/org/eclipse/ditto/rql/query/QueryBuilder.java index d3e59cd18f..86d2f0aca5 100644 --- a/rql/query/src/main/java/org/eclipse/ditto/rql/query/QueryBuilder.java +++ b/rql/query/src/main/java/org/eclipse/ditto/rql/query/QueryBuilder.java @@ -28,14 +28,6 @@ public interface QueryBuilder { */ QueryBuilder sort(List sortOptions); - /** - * Limits the number of elements returned by the query. - * - * @param n the (maximum) number of elements to be returned - * @return this builder - */ - QueryBuilder limit(long n); - /** * Limits the number of elements returned by the query. * @@ -44,14 +36,6 @@ public interface QueryBuilder { */ QueryBuilder size(long n); - /** - * Discards the given number of elements at the beginning of the query result. - * - * @param n the number of elements to be skipped - * @return this builder - */ - QueryBuilder skip(long n); - /** * Builds the Query. * diff --git a/rql/search-option-parser/src/main/scala/org/eclipse/ditto/rql/parser/thingsearch/internal/RqlOptionParser.scala b/rql/search-option-parser/src/main/scala/org/eclipse/ditto/rql/parser/thingsearch/internal/RqlOptionParser.scala index 1cd3936731..2fef7ad2d4 100644 --- a/rql/search-option-parser/src/main/scala/org/eclipse/ditto/rql/parser/thingsearch/internal/RqlOptionParser.scala +++ b/rql/search-option-parser/src/main/scala/org/eclipse/ditto/rql/parser/thingsearch/internal/RqlOptionParser.scala @@ -17,7 +17,7 @@ import org.eclipse.ditto.rql.model.ParserException import org.eclipse.ditto.rql.parser.internal.RqlParserBase import org.eclipse.ditto.rql.parser.thingsearch.OptionParser import org.eclipse.ditto.thingsearch.model -import org.eclipse.ditto.thingsearch.model.{LimitOption, SearchModelFactory, SortOption, SortOptionEntry} +import org.eclipse.ditto.thingsearch.model.{SearchModelFactory, SortOption, SortOptionEntry} import java.util import scala.collection.JavaConverters @@ -27,11 +27,10 @@ import scala.util.{Failure, Success} * RQL Parser. Parses options in the RQL "standard" according to https://github.com/persvr/rql with the following EBNF: * 
   * Options                    = Option, { ',', Option }
-  * Option                     = Sort | Limit
+  * Option                     = Sort | Cursor | Size
   * Sort                       = "sort", '(', SortProperty, { ',', SortProperty }, ')'
   * SortProperty               = SortOrder, PropertyLiteral
   * SortOrder                  = '+' | '-'
-  * Limit                      = "limit", '(', IntegerLiteral, ',', IntegerLiteral, ')'
   *
*/ private class RqlOptionParser(override val input: ParserInput) extends RqlParserBase(input) { @@ -51,10 +50,10 @@ private class RqlOptionParser(override val input: ParserInput) extends RqlParser } /** - * Option = Sort | Limit | Cursor | Size + * Option = Sort | Cursor | Size */ private def Option: Rule1[model.Option] = rule { - Sort | Limit | Cursor | Size + Sort | Cursor | Size } /** @@ -89,14 +88,6 @@ private class RqlOptionParser(override val input: ParserInput) extends RqlParser '-' ~ push(SortOptionEntry.SortOrder.DESC) } - /** - * Limit = "limit", '(', IntegerLiteral, ',', IntegerLiteral, ')' - */ - private def Limit: Rule1[LimitOption] = rule { - "limit" ~ '(' ~ LongLiteral ~ ',' ~ LongLiteral ~ ')' ~> ((offset: java.lang.Long, count: java.lang.Long) => - SearchModelFactory.newLimitOption(offset.toInt, count.toInt)) - } - /** * Cursor = "cursor", '(', StringLiteral, ')' */ @@ -114,6 +105,7 @@ private class RqlOptionParser(override val input: ParserInput) extends RqlParser } private def CursorString: Rule1[String] = PropertyLiteral + } /** @@ -141,4 +133,5 @@ object RqlOptionParser extends OptionParser { } private def rqlOptionsParser(string: String): RqlOptionParser = new RqlOptionParser(ParserInput.apply(string)) + } diff --git a/rql/search-option-parser/src/test/java/org/eclipse/ditto/rql/parser/thingsearch/options/rql/RqlOptionsParserTest.java b/rql/search-option-parser/src/test/java/org/eclipse/ditto/rql/parser/thingsearch/options/rql/RqlOptionsParserTest.java index ccd9477083..0a644bb21c 100755 --- a/rql/search-option-parser/src/test/java/org/eclipse/ditto/rql/parser/thingsearch/options/rql/RqlOptionsParserTest.java +++ b/rql/search-option-parser/src/test/java/org/eclipse/ditto/rql/parser/thingsearch/options/rql/RqlOptionsParserTest.java @@ -21,7 +21,6 @@ import org.eclipse.ditto.rql.parser.thingsearch.OptionParser; import org.eclipse.ditto.rql.parser.thingsearch.RqlOptionParser; import org.eclipse.ditto.thingsearch.model.CursorOption; -import org.eclipse.ditto.thingsearch.model.LimitOption; import org.eclipse.ditto.thingsearch.model.Option; import org.eclipse.ditto.thingsearch.model.SizeOption; import org.eclipse.ditto.thingsearch.model.SortOption; @@ -109,56 +108,34 @@ public void invalidSortMissingOrderAtSecondProperty() throws ParserException { parser.parse("sort(-username,owner)"); } - @Test - public void parseLimitSuccess() throws ParserException { - final List