COMPASS-287: distinct numeric types (#584)

* COMPASS-287 upgrade old language-model to pegjs

* fix bar selection rounding bug for negative, very large numbers and 0.

* don’t promote values.

* allow dotted field names and fix type switching bug

* remove debug statements.

* handle number ranges correctly for unpromoted bson types

* fix query building for negative long values.

Author: rueckstiess

package.json

@@ -98,7 +98,7 @@
     "d3-timer": "^1.0.3",
     "debug": "mongodb-js/debug#v2.2.3",
     "debug-menu": "^0.3.0",
-    "detect-coordinates": "^0.1.0",
+    "detect-coordinates": "^0.2.0",
     "electron-squirrel-startup": "^1.0.0",
     "font-awesome": "https://github.com/FortAwesome/Font-Awesome/archive/v4.4.0.tar.gz",
     "get-object-path": "azer/get-object-path#74eb42de0cfd02c14ffdd18552f295aba723d394",
@@ -125,15 +125,15 @@
     "mongodb": "^2.2.8",
     "mongodb-collection-model": "^0.3.1",
     "mongodb-connection-model": "^6.3.1",
-    "mongodb-data-service": "^2.1.0",
+    "mongodb-data-service": "^2.1.1",
     "mongodb-database-model": "^0.1.2",
     "mongodb-explain-plan-model": "^0.2.2",
-    "mongodb-extended-json": "^1.7.0",
+    "mongodb-extended-json": "^1.8.0",
     "mongodb-instance-model": "^3.3.0",
     "mongodb-js-metrics": "^1.5.2",
-    "mongodb-language-model": "^0.3.3",
+    "mongodb-language-model": "^1.0.2",
     "mongodb-ns": "^1.0.3",
-    "mongodb-schema": "^5.0.0",
+    "mongodb-schema": "^6.0.2",
     "ms": "^0.7.1",
     "ncp": "^2.0.0",
     "numeral": "^1.5.3",

src/app/home/collection.js

@@ -213,7 +213,7 @@ var MongoDBCollectionView = View.extend({
       this.loadIndexesAction();
       this.fetchExplainPlanAction();
     });
-    Action.filterChanged(app.queryOptions.query.serialize());
+    Action.filterChanged(app.queryOptions.query);
     this.switchView(this.activeView);
   },
   onCollectionFetched: function(model) {

src/app/models/editable-query.js

@@ -1,7 +1,6 @@
 var ms = require('ms');
 var Model = require('ampersand-model');
 var EJSON = require('mongodb-extended-json');
-var Query = require('mongodb-language-model').Query;
 // var debug = require('debug')('mongodb-compass:models:query-options');
 
 var DEFAULT_QUERY = {};
@@ -12,12 +11,6 @@ var DEFAULT_SIZE = 1000;
 var DEFAULT_SKIP = 0;
 var DEFAULT_MAX_TIME_MS = ms('10 seconds');
 
-var getDefaultQuery = function() {
-  return new Query(DEFAULT_QUERY, {
-    parse: true
-  });
-};
-
 /**
  * Options for reading a collection of documents from MongoDB.
  */
@@ -26,7 +19,7 @@ module.exports = Model.extend({
     query: {
       type: 'state',
       default: function() {
-        return getDefaultQuery();
+        return DEFAULT_QUERY;
       }
     },
     sort: {
@@ -53,18 +46,18 @@ module.exports = Model.extend({
       deps: ['query'],
       cache: false,
       fn: function() {
-        return EJSON.stringify(this.query.serialize());
+        return EJSON.stringify(this.query);
       }
     }
   },
   serialize: function() {
     var res = Model.prototype.serialize.call(this);
-    res.query = this.query.serialize();
+    res.query = this.query;
     return res;
   },
   reset: function() {
     this.set({
-      query: getDefaultQuery(),
+      query: DEFAULT_QUERY,
       sort: DEFAULT_SORT,
       size: DEFAULT_SIZE,
       skip: DEFAULT_SKIP,

src/app/models/query-options.js

@@ -27,7 +27,7 @@ const LoadMoreDocumentsStore = Reflux.createStore({
    * @param {Integer} skip - The number of documents to skip.
    */
   loadMoreDocuments: function(skip) {
-    const filter = app.queryOptions.query.serialize();
+    const filter = app.queryOptions.query;
     const options = { skip: skip, limit: 20, sort: [[ '_id', 1 ]], readPreference: READ };
     app.dataService.find(NamespaceStore.ns, filter, options, (error, documents) => {
       if (!error) {

src/internal-packages/crud/lib/store/load-more-documents-store.js

@@ -1,6 +1,6 @@
 const React = require('react');
 const Action = require('../action/index-actions');
-const Query = require('mongodb-language-model').Query;
+const accepts = require('mongodb-language-model').accepts;
 const EJSON = require('mongodb-extended-json');
 const _ = require('lodash');
 
@@ -69,20 +69,17 @@ class CreateIndexTextField extends React.Component {
    * @returns {Object|Boolean}        false if invalid, otherwise the query
    */
   _validateQueryString(queryString) {
-    let parsed;
     try {
-      // is it valid eJSON?
       const cleaned = this._cleanQueryString(queryString);
-      parsed = EJSON.parse(cleaned);
-      // is it a valid parsable Query according to the language?
-      /* eslint no-unused-vars: 0 */
-      const query = new Query(parsed, {
-        parse: true
-      });
+      // is it valid eJSON?
+      const parsed = EJSON.parse(cleaned);
+      // can it be serialized to JSON?
+      const stringified = JSON.stringify(parsed);
+      // is it a valid MongoDB query according to the language?
+      return accepts(stringified);
     } catch (e) {
       return false;
     }
-    return parsed;
   }
 
   /**
@@ -107,7 +104,7 @@ class CreateIndexTextField extends React.Component {
     if (this.props.units) inputClassName += ' inline-option-field';
     if (this.props.option === 'partialFilterExpression') {
       inputClassName += ' partial-filter-input';
-      const valid = Boolean(this._validateQueryString(this.state.value));
+      const valid = this._validateQueryString(this.state.value);
       if (!valid) groupClassName += ' has-error';
     }
     return (

src/internal-packages/indexes/lib/component/create-index-text-field.jsx

@@ -4,7 +4,7 @@ const NamespaceStore = require('hadron-reflux-store').NamespaceStore;
 const StateMixin = require('reflux-state-mixin');
 const QueryAction = require('../action');
 const EJSON = require('mongodb-extended-json');
-const Query = require('mongodb-language-model').Query;
+const accepts = require('mongodb-language-model').accepts;
 const _ = require('lodash');
 const hasDistinctValue = require('../util').hasDistinctValue;
 const filterChanged = require('hadron-action').filterChanged;
@@ -115,23 +115,18 @@ const QueryStore = Reflux.createStore({
    * validates whether a string is a valid query.
    *
    * @param  {Object} queryString    a string to validate
-   * @return {Object|Boolean}        false if invalid, otherwise the query
+   * @returns {Object|Boolean}        false if invalid, otherwise the query
    */
   _validateQueryString(queryString) {
-    let parsed;
     try {
-      // is it valid eJSON?
       const cleaned = this._cleanQueryString(queryString);
-      parsed = EJSON.parse(cleaned);
-      // is it a valid parsable Query according to the language?
-      /* eslint no-unused-vars: 0 */
-      const query = new Query(parsed, {
-        parse: true
-      });
+      // is it valid eJSON?
+      const parsed = EJSON.parse(cleaned);
+      // is it a valid MongoDB query according to the language?
+      return accepts(cleaned) ? parsed : false;
     } catch (e) {
       return false;
     }
-    return parsed;
   },
 
   _validateFeatureFlag(queryString) {

src/internal-packages/query/lib/store/index.js

@@ -1,11 +1,17 @@
 const _ = require('lodash');
-// const debug = require('debug')('mongodb-compass:schema:test');
+// const debug = require('debug')('mongodb-compass:query:utils');
 
 function bsonEqual(value, other) {
   const bsontype = _.get(value, '_bsontype', undefined);
   if (bsontype === 'ObjectID') {
     return value.equals(other);
   }
+  if (_.includes(['Decimal128', 'Long'], bsontype)) {
+    return value.toString() === other.toString();
+  }
+  if (_.includes(['Int32', 'Double'], bsontype)) {
+    return value.value === other.value;
+  }
   // for all others, use native comparisons
   return undefined;
 }
@@ -27,7 +33,9 @@ function hasDistinctValue(field, value) {
     if (_.has(field, '$in')) {
       // check if $in array contains the value
       const inArray = field.$in;
-      return (_.contains(inArray, value));
+      return (_.some(inArray, (other) => {
+        return _.isEqual(value, other, bsonEqual);
+      }));
     }
   }
   // it is not a $in operator, check value directly
@@ -111,8 +119,10 @@ function inValueRange(field, d) {
   }
   const dx = _.get(d, 'dx', null);
 
-  // extract bound(s)
-  const bounds = dx === null ? [d.value] : _.uniq([d.value, d.value + dx]);
+  // extract bound(s): if a dx value is set, create a 2-element array of
+  // upper and lower bound. otherwise create a single value array of the
+  // original bson type (if available) or the extracted value.
+  const bounds = dx ? _.uniq([d.value, d.value + dx]) : [d.bson || d.value];
 
   /*
    * Logic to determine if the query covers the value (or value range)
@@ -129,10 +139,10 @@ function inValueRange(field, d) {
    */
   const results = _.map(bounds, function(bound, i) {
     // adjust the upper bound slightly as it represents an exclusive bound
-    // getting this right would require a lot more code to check for all 4
-    // edge cases.
+    // getting this perfectly right would require a lot more code to check for
+    // all 4 edge cases.
     if (i > 0) {
-      bound *= 0.999999;
+      bound -= (0.00001 * Math.abs(bound)) + 0.00001;
     }
     return _.every(_.map(conditions, function(cond) {
       return cond(bound);

src/internal-packages/query/lib/util/index.js

@@ -8,9 +8,7 @@ const D3Component = require('./d3component');
 const vizFns = require('../d3');
 const Actions = require('../action');
 
-const STRING = 'String';
-const NUMBER = 'Number';
-const DECIMAL_128 = 'Decimal128';
+const { STRING, DECIMAL_128, DOUBLE, LONG, INT_32 } = require('../helpers');
 
 const Minichart = React.createClass({
 
@@ -72,17 +70,20 @@ const Minichart = React.createClass({
   },
 
   minichartFactory() {
-    /* eslint camelcase: 0 */
-    const typeName = this.props.type.name;
+    // cast all numeric types to Number minichart
+    const typeName = _.includes([ DECIMAL_128, DOUBLE, INT_32 ],
+      this.props.type.name) ? 'Number' : this.props.type.name;
+
     const fieldName = this.props.fieldName;
     const queryClause = this.state.query[fieldName];
-    const has_duplicates = this.props.type.has_duplicates;
+    const hasDuplicates = this.props.type.has_duplicates;
     const fn = vizFns[typeName.toLowerCase()];
     const width = this.state.containerWidth;
 
-    if (_.includes([ STRING, NUMBER, DECIMAL_128 ], typeName) && !has_duplicates) {
+    if (_.includes([ STRING, LONG ], typeName) && !hasDuplicates) {
       return (
         <UniqueMinichart
+          key={typeName}
           fieldName={fieldName}
           query={queryClause}
           type={this.props.type}