Add numeric and date fields, gt lt gte lte operators (#76)

* Support colons in search box & metadata search

* remove logging

* Add support for other operators in search api & search box

* Clean up != behavior, include gte lte in UI

* Clean up regex and parsing for exists and missing queries

* remove space sensitivity to searches

* improve documentation

* Fixed spacing check

* handle colons on contains matches better

* Add numeric detection to mapping

* disable lt gt lte gte

* update help syntax

* Simplify help

* colon warning

* Add lt gt lte gte back to available operators

* enable date_detection explicitly

* Update CHANGELOG.md

* omit redundant date_detection flag

* update search help descriptions

Co-authored-by: Luigi Marini <[email protected]>

Author: max-zilla

CHANGELOG.md

@@ -20,8 +20,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## 1.11.1 - 2020-09-29
 
+**_Warning:_ This update modifies information stored in Elasticsearch used for text based searching. To take advantage 
+of these changes a reindex of Elasticsearch is required. A reindex can be started by an admin from the Admin menu.**
+
 ### Added
-- Added healtz endpoint that is cheap and quick to return, useful for kubernetes live/ready checks.
+- added healtz endpoint that is cheap and quick to return, useful for kubernetes live/ready checks.
+- added support for parsing of Date and Numeric data in new metadata fields. New search operators <, >, <=, >= have been
+added to search API now that those data are properly compared.
 
 ### Fixed
 - Fixed health check script when using custom path prefix.

app/services/ElasticsearchPlugin.scala

@@ -49,8 +49,7 @@ class ElasticsearchPlugin(application: Application) extends Plugin {
   val nameOfIndex = play.api.Play.configuration.getString("elasticsearchSettings.indexNamePrefix").getOrElse("clowder")
   val maxResults = play.api.Play.configuration.getInt("elasticsearchSettings.maxResults").getOrElse(240)
 
-  // TODO: Removed gt lt gte lte operators until numeric_detection can be enabled on the dynamic mapper
-  val mustOperators = List("==", ":") // "<=", ">=", "<", ">", ":")
+  val mustOperators = List("==", "<=", ">=", "<", ">", ":")
   val mustNotOperators = List("!=")
 
 
@@ -696,9 +695,8 @@ class ElasticsearchPlugin(application: Application) extends Plugin {
      * as strings for datatypes besides Objects. In the future, this could
      * be removed, but only once the Search API better supports those data types (e.g. Date).
      */
-    // TODO: Enable "numeric_detection": true alongside date_detection
     """{"clowder_object": {
-          |"date_detection": false,
+          |"numeric_detection": true,
           |"properties": {
             |"name": {"type": "string"},
             |"description": {"type": "string"},
@@ -847,8 +845,7 @@ class ElasticsearchPlugin(application: Application) extends Plugin {
 
     // Use regex to split string into a list, preserving quoted phrases as single value
     val matches = ListBuffer[String]()
-    val m = Pattern.compile("([^\":= ]+|\".+?\")").matcher(query)
-    //val m = Pattern.compile("([^\":=<> ]+|\".+?\")").matcher(query)
+    val m = Pattern.compile("([^\":=<> ]+|\".+?\")").matcher(query)
     while (m.find()) {
       var mat = m.group(1).replace("\"", "").replace("__", " ").trim
       if (mat.length>0) {

app/views/metadatald/search.scala.html

@@ -205,10 +205,10 @@ <h1>Search Metadata within Space: "@space.name"</h1>
                         "<option value='=='>equals</option>" +
                         "<option value='!='>does not equal</option>" +
                         "<option value=':'>contains</option>" +
-                        //"<option value='>'>greater than</option>" +
-                        //"<option value='<'>less than</option>" +
-                        //"<option value='>='>greater or equal to</option>" +
-                        //"<option value='<='>less or equal to</option>" +
+                        "<option value='>'>greater than</option>" +
+                        "<option value='<'>less than</option>" +
+                        "<option value='>='>greater or equal to</option>" +
+                        "<option value='<='>less or equal to</option>" +
                     "</select></div>" +
                     <!-- VALUE FIELD -->
                     "<div class='form-group col-lg-4 col-md-4'>" +

app/views/searchResults.scala.html

@@ -32,19 +32,19 @@ <h1>Search</h1>
                 <div class="panel-body">
                     <ul>
                         <li>@Messages("help.search.overview1")</li>
+                        <li>@Messages("help.search.overview4")<br/><t/>@Messages("help.search.overview6")</li>
                         <li>@Messages("help.search.overview2")</li>
                         <li>@Messages("help.search.overview3")</li>
+                        <li>@Messages("help.search.overview5")</li>
                     </ul>
                     <table style="width:100%;">
                         <tr><td width="120"><b>operator</b></td>
                             <td width="400"></td></tr>
                         <tr><td><b>==</b></td><td>equals</td></tr>
                         <tr><td><b>!=</b></td><td>does not equal</td></tr>
                         <tr><td><b>:</b></td><td>contains</td></tr>
-                        <!--
                         <tr><td><b>< ></b></td><td>less than / greater than</td></tr>
                         <tr><td><b><= >=</b></td><td>less or equal to / greater or equal to </td></tr>
-                        -->
                     </table></br>
                     <table style="width:100%;">
                         <tr><td width="120"><b>@Messages("help.search.col1")</b></td>

conf/messages

@@ -62,11 +62,13 @@ securesocial.login.useEmailAndPassword=Or login using an email and password.
 securesocial.login.useEmailAndPasswordOnly=Login using an email and password.
 
 #Search help text
-help.search.toggle=Search Syntax
+help.search.toggle=Search Syntax Help
+help.search.overview4=When searching for extractor-generated metadata, the entire extractor path with periods replaced as underscores must be included as a prefix.
+help.search.overview6=e.g.: "https://mysite_com/clowder/extractors/my_extractor.field"<20
 help.search.overview1=If the term or metadata field contains spaces or colons, use double quotes to enclose the phrase.
 help.search.overview2=You can use * as a wildcard character.
 help.search.overview3=Case does not matter unless noted.
-help.search.overview5=(Elasticsearch regular expression syntax help)
+help.search.overview5=When searching date fields, the date formats must match (YYYY-MM-DD will not match YYYY/MM/DD)
 help.search.col1=field
 help.search.col2=example
 help.search.allfields=name, description, creator name, and tag