diff --git a/modules/fts/pages/fts-quickstart-guide.adoc b/modules/fts/pages/fts-quickstart-guide.adoc index ba31a1698c..04e5b02e65 100644 --- a/modules/fts/pages/fts-quickstart-guide.adoc +++ b/modules/fts/pages/fts-quickstart-guide.adoc @@ -43,6 +43,5 @@ Refer to xref:n1ql:n1ql-language-reference/searchfun.adoc[Search Functions] for [#establishing-demonstration-indexes] == Accessing the Search service via the Java SDK -The Java SDK code-example provided in xref:java-sdk:howtos:full-text-searching-with-sdk.adoc[Searching from the Java SDK] contains multiple demonstration calls — each featuring a different query-combination — and makes use of three different index-definitions, related to the `travel-sample` bucket: for the code example to run successfully, the three indexes must be appropriately pre-established. -//The definitions are provided in xref:fts-demonstration-indexes.adoc[Demonstration Indexes]. -Instructions on how to use the Couchbase REST API to establish the definitions refer to xref:fts-creating-index-with-rest-api.adoc[Index Creation with REST API]. +The Java SDK code-example provided in xref:java-sdk:howtos:full-text-searching-with-sdk.adoc[Searching from the SDK] contains multiple demonstration calls — each featuring a different query-combination — and makes use of three different index-definitions, related to the `travel-sample` bucket: for the code example to run successfully, the three indexes must be appropriately pre-established. +Instructions on how to use the Couchbase REST API to establish the definitions refer to xref:fts-creating-index-with-rest-api.adoc[Creating Index with REST API]. diff --git a/modules/fts/pages/fts-searching-from-N1QL.adoc b/modules/fts/pages/fts-searching-from-N1QL.adoc index 9b77e89aa4..f46fe877f0 100644 --- a/modules/fts/pages/fts-searching-from-N1QL.adoc +++ b/modules/fts/pages/fts-searching-from-N1QL.adoc @@ -5,11 +5,57 @@ Search functions enable you to use full text search queries directly within a N1 == Prerequisites To use any of the search functions, the Search service must be available on the cluster. -It is also recommended, but not required, that you should create suitable full text indexes for the searches that you need to perform. +It is required for you to create suitable full text indexes for the searches that you want to perform. [NOTE] -- -The examples in this page all assume that demonstration full text indexes have been created. +The examples in this page assume that you have a full text search index created with this definition, unless specified otherwise .. +[source,json] +---- +{ + "name": "travel-sample-inventory-hotels", + "type": "fulltext-index", + "params": { + "doc_config": { + "mode": "scope.collection.type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": false, + "index_dynamic": true, + "store_dynamic": false, + "type_field": "_type", + "types": { + "inventory.hotel": { + "dynamic": true, + "enabled": true + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "", + "sourceParams": {}, + "planParams": { + "indexPartitions": 1 + }, + "uuid": "" +} +---- + -- === Authorization @@ -194,6 +240,11 @@ Order pushdown is possible only if query ORDER BY has only <> on t Offset and Limit pushdown is possible if the query only has a SEARCH() predicate, using a single search index -- no IntersectScan or OrderIntersectScan. Group aggregates and projection are not pushed. +[NOTE] +-- +If the "index" setting isn't specified within the options argument of the _SEARCH_ function, N1QL can pick any FTS index available in the system that qualifies for the query. If a number of FTS indexes qualify, then one that is defined most precise will be chosen. +-- + === Examples .Search using a query string @@ -203,14 +254,14 @@ The following queries are equivalent: [source,n1ql] ---- SELECT META(t1).id -FROM `travel-sample`.inventory.airline AS t1 +FROM `travel-sample`.inventory.hotel AS t1 WHERE SEARCH(t1.country, "+United +States"); ---- [source,n1ql] ---- SELECT META(t1).id -FROM `travel-sample`.inventory.airline AS t1 +FROM `travel-sample`.inventory.hotel AS t1 WHERE SEARCH(t1, "country:\"United States\""); ---- @@ -218,18 +269,14 @@ WHERE SEARCH(t1, "country:\"United States\""); [source,json] ---- [ - { - "id": "airline_10" + "id": "hotel_26215" }, { - "id": "airline_10123" + "id": "hotel_7396" }, { - "id": "airline_10226" - }, - { - "id": "airline_10748" + "id": "hotel_32177" }, ... ] @@ -350,8 +397,51 @@ WHERE t1.type = "hotel" AND SEARCH(t1.description, "amazing"); ---- If the full text search index being queried has its default mapping disabled and has a custom type mapping defined, the query needs to specify the type explicitly. - -//The above query uses the demonstration index xref:fts:fts-demonstration-indexes.adoc#travel-sample-index-hotel-description[travel-sample-index-hotel-description], which has the custom type mapping "hotel". +The above query uses the following index definition .. +[source,json] +---- +{ + "name": "travel-sample-hotels", + "type": "fulltext-index", + "params": { + "doc_config": { + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "index_dynamic": true, + "store_dynamic": true, + "type_field": "type", + "types": { + "hotel": { + "dynamic": true, + "enabled": true + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "", + "sourceParams": {}, + "planParams": { + "indexPartitions": 1 + }, + "uuid": "" +} +---- For more information on defining custom type mappings within the full text search index, refer to xref:fts:fts-type-mappings.adoc[Type Mappings]. Note that for N1QL queries, only full text search indexes with one type mapping are searchable. @@ -506,6 +596,10 @@ AND SEARCH(t1, { ---- [ { + "meta": { + "id": "hotel_17598", + "score": 2.1256278997816835 + }, "name": "Marina del Rey Marriott" } ] @@ -569,16 +663,19 @@ LIMIT 3; ---- [ { - "description": "3 Star Hotel next to the Mountain Railway terminus and set in 30 acres of grounds which include Dolbadarn Castle", - "name": "The Royal Victoria Hotel" + "description": "370 guest rooms offering both water and mountain view.", + "name": "Marina del Rey Marriott", + "score": 2.1256278997816835 }, { - "description": "370 guest rooms offering both water and mountain view.", - "name": "Marina del Rey Marriott" + "description": "Log cabin glamping in a rural setting with panoramic views toward the Clwydian Mountain Range.", + "name": "Clwydian Holidays", + "score": 1.6956645086702617 }, { - "description": "This small family run hotel captures the spirit of Mull and is a perfect rural holiday retreat. The mountain and sea blend together to give fantastic, panoramic views from the hotel which is in an elevated position on the shoreline. Panoramic views are also available from the bar and restaurant which serves local produce 7 days a week.", - "name": "The Glenforsa Hotel" + "description": "3 Star Hotel next to the Mountain Railway terminus and set in 30 acres of grounds which include Dolbadarn Castle", + "name": "The Royal Victoria Hotel", + "score": 1.5030458987111712 } ] ---- @@ -682,4 +779,4 @@ WHERE reviews.content LIKE "%travel%"; SELECT META().id FROM `travel-sample`.`inventory`.`hotel` t USE INDEX(USING FTS) WHERE content LIKE "%travel%"; ----- \ No newline at end of file +---- diff --git a/modules/fts/pages/fts-supported-queries-geo-point-distance.adoc b/modules/fts/pages/fts-supported-queries-geo-point-distance.adoc index 5c6158525a..a018b66a12 100644 --- a/modules/fts/pages/fts-supported-queries-geo-point-distance.adoc +++ b/modules/fts/pages/fts-supported-queries-geo-point-distance.adoc @@ -1,8 +1,7 @@ = Creating a Query: Radius-Based -Note that a detailed example for Geopoint index creation and also executing queries can be found at xref:fts-supported-queries-geopoint-spatial.adoc#creating_a_geospatial_geopoint_index[Geopoint Index Creation] and running queries xref:fts-supported-queries-geopoint-spatial.adoc#creating_geopoint_rest_query_radius_based[Geopoint Radius Queries]. - -In addition detailed information on performing queries with the Search REST API can be found in xref:fts-searching-with-curl-http-requests.adoc[Searching with the REST API]; which shows how to use the full `curl` command and how to incorporate query-bodies into your cURL requests. +This section and those following, provide examples of the query-bodies required to make geospatial queries with the Couchbase REST API. +Note that more detailed information on performing queries with the Couchbase REST API can be found in xref:fts-searching-with-curl-http-requests.adoc[Searching with the REST API]; which shows how to use the full `curl` command and how to incorporate query-bodies into it. The following query-body specifies a longitude of `-2.235143` and a latitude of `53.482358`. The target-field `geo` is specified, as is a `distance` of `100` miles: this is the radius within which target-locations must reside for their documents to be returned. diff --git a/modules/fts/pages/fts-supported-queries-geopoint-spatial.adoc b/modules/fts/pages/fts-supported-queries-geopoint-spatial.adoc index 075cfd5bb1..131d67d491 100644 --- a/modules/fts/pages/fts-supported-queries-geopoint-spatial.adoc +++ b/modules/fts/pages/fts-supported-queries-geopoint-spatial.adoc @@ -232,10 +232,61 @@ Detailed instructions for setting up indexes, and specifying type mappings, are For initial experimentation with geospatial geopoint querying (based on the type geopoint), the `geo` field of documents within the `travel-sample` bucket can be specified as a child field of the `default` type mapping (keyspace `travel-sample._default._default` as follows: -include::partial$fts-creating-geopoint-common.adoc[] +The following query examples in this page will require the following index definition .. -The index once created can also be accessed by means of the Search REST API -see xref:fts-searching-with-curl-http-requests.adoc[Searching with the REST API]. Furthermore the index could have been created in the first place via the Search REST API see xref:fts-creating-index-with-rest-api.adoc[Index Creation with REST API] for more information on using the Search REST API syntax. +[source,json] +---- +{ + "type": "fulltext-index", + "name": "geoIndex", + "uuid": "", + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "", + "planParams": { + "indexPartitions": 1 + }, + "params": { + "doc_config": { + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": true, + "properties": { + "geo": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "geo", + "store": true, + "type": "geopoint" + } + ] + } + } + }, + "default_type": "_default", + "index_dynamic": true, + "store_dynamic": false + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceParams": {} +} +---- [#creating_geospatial_rest_query_radius_based] [#creating_geopoint_rest_query_radius_based] diff --git a/modules/n1ql/pages/n1ql-language-reference/searchfun.adoc b/modules/n1ql/pages/n1ql-language-reference/searchfun.adoc index b1a11929e9..35f0901017 100644 --- a/modules/n1ql/pages/n1ql-language-reference/searchfun.adoc +++ b/modules/n1ql/pages/n1ql-language-reference/searchfun.adoc @@ -6,20 +6,263 @@ {description} -[float] -=== Prerequisites +[#prerequisities,float] +== Prerequisites To use any of the search functions, the Search service must be available on the cluster. -It is also recommended, but not required, that you should create suitable full text indexes for the searches that you need to perform. -For more information, refer to xref:fts:fts-quickstart-guide.adoc#preparing-for-full-text-searches[Preparing for Full Text Searches]. + +To run the examples on this page, you will need to create three indexes. + +To create the indexes, you can use: + +* The index editor built into the web console. +* The REST API +* The command line utility. + +For more information, see xref:server:search:create-search-indexes.adoc[Creating search indexes]. + +For this exercise, the easiest way is to use the web console to import the index definitions below: + +[tabs] +==== +travel-sample-index-unstored:: ++ +-- +[source,json] +---- + +{ + "name": "travel-sample-index-unstored", + "type": "fulltext-index", + "params": { + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "analysis": { + "analyzers": { + "singleAnalyzer": { + "tokenizer": "single", + "type": "custom" + } + } + }, + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": true + }, + "default_type": "_default", + "docvalues_dynamic": false, + "index_dynamic": true, + "store_dynamic": false, + "type_field": "_type" + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "94387e8a4cefe4abeb51d92e28a696c6", + "sourceParams": {}, + "planParams": { + "maxPartitionsPerPIndex": 1024, + "indexPartitions": 1, + "numReplicas": 0 + }, + "uuid": "5e26bbd6c2c87069" +} +---- +-- + +travel-sample-index-stored:: ++ +-- +[source,json] +---- +{ + "name": "travel-sample-index-stored", + "type": "fulltext-index", + "params": { + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "analysis": { + "analyzers": { + "letterAnalyzer": { + "tokenizer": "letter", + "type": "custom" + } + } + }, + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": true + }, + "default_type": "_default", + "docvalues_dynamic": false, + "index_dynamic": true, + "store_dynamic": true, + "type_field": "_type" + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "94387e8a4cefe4abeb51d92e28a696c6", + "sourceParams": {}, + "planParams": { + "maxPartitionsPerPIndex": 1024, + "indexPartitions": 1, + "numReplicas": 0 + }, + "uuid": "16d6602d3193b526" +} +---- + +-- + +geoIndex:: ++ +-- + +[source,json] +---- + +{ + "name": "geoIndex", + "type": "fulltext-index", + "params": { + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": true, + "properties": { + "geo": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "include_in_all": true, + "index": true, + "name": "geo", + "store": true, + "type": "geopoint" + } + ] + } + } + }, + "default_type": "_default", + "docvalues_dynamic": false, + "index_dynamic": true, + "store_dynamic": false, + "type_field": "_type" + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15, + "spatialPlugin": "s2" + } + }, + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "94387e8a4cefe4abeb51d92e28a696c6", + "sourceParams": {}, + "planParams": { + "maxPartitionsPerPIndex": 1024, + "indexPartitions": 1, + "numReplicas": 0 + }, + "uuid": "445bceb9301d0d74" +} +---- + + +-- + + +==== [NOTE] -- -The examples in this page all assume that demonstration full text indexes have been created, as described in xref:fts:fts-demonstration-indexes.adoc[Demonstration Indexes]. +The examples in this page assume that you have a full text search index created with this definition, unless specified otherwise .. +[source,json] +---- +{ + "name": "travel-sample-inventory-hotels", + "type": "fulltext-index", + "params": { + "doc_config": { + "mode": "scope.collection.type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": false, + "index_dynamic": true, + "store_dynamic": false, + "type_field": "_type", + "types": { + "inventory.hotel": { + "dynamic": true, + "enabled": true + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "", + "sourceParams": {}, + "planParams": { + "indexPartitions": 1 + }, + "uuid": "" +} +---- + -- [float] -=== Authorization +== Authorization You do not need credentials for the FTS service to be able to use the search functions in a query. The role *Data Admin* must be assigned to those who intend to create indexes; and the role *Data Reader* to those who intend to perform searches. @@ -88,7 +331,7 @@ For more details, refer to xref:fts:fts-query-types.adoc#query-string-query-synt | object | The query object within a full text search request. -For more details, refer to xref:fts-supported-queries.adoc[Supported Queries]. +For more details, refer to xref:fts:fts-supported-queries.adoc[Supported Queries]. | object | A complete full text search request, including sort and pagination options, and so on. @@ -96,9 +339,10 @@ For more details, refer to xref:fts:fts-sorting.adoc[Sorting Query Results]. [NOTE] ==== -When specifying a complete full text search request with the N1QL SEARCH() function, if the value of the `size` parameter is greater than the xref:fts:fts-response-object-schema.adoc#request[maximum number of full text search results], the query ignores the `size` parameter and returns all matching results. - -This is different to the behavior of a complete full text search request in the Search service, where the query returns an error if the value of the `size` parameter is greater than the maximum number of full text search results. +* If size/from (pagination) settings aren't set, the search index will "stream" all qualifying results to N1QL which can perform further filtering. +* If `size/limit`, `from/offset` are set within the search request within the SEARCH function, FTS handles the pagination. +* N1QL can hand down LIMIT/OFFSET settings to FTS. If this `limit+offset <= bleveMaxResultWindow` - then the FTS index is deemed pageable and N1QL hands down these settings to FTS. If not, then FTS will stream results to N1QL and N1QL will be responsible for pagination and ordering. +* The default value for the cluster wide xref:fts:fts-advanced-settings-bleveMaxResultWindow.adoc[bleveMaxResultWindow] setting is 10000. ==== |=== + @@ -202,6 +446,11 @@ Order pushdown is possible only if query ORDER BY has only <> on t Offset and Limit pushdown is possible if the query only has a SEARCH() predicate, using a single search index -- no IntersectScan or OrderIntersectScan. Group aggregates and projection are not pushed. +[NOTE] +-- +If the "index" setting isn't specified within the options argument of the _SEARCH_ function, N1QL can pick any FTS index available in the system that qualifies for the query. If a number of FTS indexes qualify, then one that is defined most precise will be chosen. +-- + === Examples .Search using a query string @@ -211,14 +460,14 @@ The following queries are equivalent: [source,n1ql] ---- SELECT META(t1).id -FROM `travel-sample`.inventory.airline AS t1 +FROM `travel-sample`.inventory.hotel AS t1 WHERE SEARCH(t1.country, "+United +States"); ---- [source,n1ql] ---- SELECT META(t1).id -FROM `travel-sample`.inventory.airline AS t1 +FROM `travel-sample`.inventory.hotel AS t1 WHERE SEARCH(t1, "country:\"United States\""); ---- @@ -226,18 +475,14 @@ WHERE SEARCH(t1, "country:\"United States\""); [source,json] ---- [ - - { - "id": "airline_10" - }, { - "id": "airline_10123" + "id": "hotel_26215" }, { - "id": "airline_10226" + "id": "hotel_7396" }, { - "id": "airline_10748" + "id": "hotel_32177" }, ... ] @@ -334,7 +579,7 @@ As an alternative, you could limit the number of results and order them using th [source,n1ql] ---- SELECT META(t1).id -FROM `travel-sample`.inventory.hotel AS t1 +FROM `travel-sample` AS t1 WHERE t1.type = "hotel" AND SEARCH(t1.description, "amazing"); ---- @@ -358,7 +603,51 @@ WHERE t1.type = "hotel" AND SEARCH(t1.description, "amazing"); ---- If the full text search index being queried has its default mapping disabled and has a custom type mapping defined, the query needs to specify the type explicitly. -The above query uses the demonstration index xref:fts:fts-demonstration-indexes.adoc#travel-sample-index-hotel-description[travel-sample-index-hotel-description], which has the custom type mapping "hotel". +The above query uses the following index definition: +[source,json] +---- +{ + "name": "travel-sample-hotels", + "type": "fulltext-index", + "params": { + "doc_config": { + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "index_dynamic": true, + "store_dynamic": true, + "type_field": "type", + "types": { + "hotel": { + "dynamic": true, + "enabled": true + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "", + "sourceParams": {}, + "planParams": { + "indexPartitions": 1 + }, + "uuid": "" +} +---- For more information on defining custom type mappings within the full text search index, refer to xref:fts:fts-creating-indexes.adoc#specifying-type-mappings[Specifying Type Mappings]. Note that for N1QL queries, only full text search indexes with one type mapping are searchable. @@ -513,6 +802,10 @@ AND SEARCH(t1, { ---- [ { + "meta": { + "id": "hotel_17598", + "score": 2.1256278997816835 + }, "name": "Marina del Rey Marriott" } ] @@ -576,16 +869,19 @@ LIMIT 3; ---- [ { - "description": "3 Star Hotel next to the Mountain Railway terminus and set in 30 acres of grounds which include Dolbadarn Castle", - "name": "The Royal Victoria Hotel" + "description": "370 guest rooms offering both water and mountain view.", + "name": "Marina del Rey Marriott", + "score": 2.1256278997816835 }, { - "description": "370 guest rooms offering both water and mountain view.", - "name": "Marina del Rey Marriott" + "description": "Log cabin glamping in a rural setting with panoramic views toward the Clwydian Mountain Range.", + "name": "Clwydian Holidays", + "score": 1.6956645086702617 }, { - "description": "This small family run hotel captures the spirit of Mull and is a perfect rural holiday retreat. The mountain and sea blend together to give fantastic, panoramic views from the hotel which is in an elevated position on the shoreline. Panoramic views are also available from the bar and restaurant which serves local produce 7 days a week.", - "name": "The Glenforsa Hotel" + "description": "3 Star Hotel next to the Mountain Railway terminus and set in 30 acres of grounds which include Dolbadarn Castle", + "name": "The Royal Victoria Hotel", + "score": 1.5030458987111712 } ] ----