Support more aggregate options and Improve Editor UI (#33)

* Support more aggregate options; Update UI

* Add link to legacy plugin

* Improve language selection UI

* Add SetBypassDocumentValidation in the backend

* Improve loggings

* Change JavaScriptShadow label in test

* Remove \t from label

* Update editor layout

* Update screenshot

* Update README

Author: haohanyang

README.md

@@ -1,20 +1,20 @@
 # Grafana MongoDB Data Source
 
-Integrate MongoDB to Grafana
+Integrate MongoDB to Grafana. A free, open source, community-driven alternative to Grafana Lab's MongoDB enterprise plugin and MongoDB Atlas Charts.
 
 ![ci](https://github.com/haohanyang/mongodb-datasource/actions/workflows/ci.yml/badge.svg?branch=master)
 
 This plugin enables you to query and visualize data from your MongoDB databases directly within Grafana. Leverage the flexibility of MongoDB's aggregation pipeline to create insightful dashboards and panels.
 
-![screenshot](/static/screenshot.png)
+![screenshot](/static/screenshot-2.png)
 
 ## Features
 
 - **Flexible Querying:** Query data using MongoDB's aggregation pipeline syntax in JSON or JavaScript. Support query variables to create dynamic dashboards.
 - **Time Series & Table Data:** Visualize time-based data or display results in tabular format for various Grafana panels.
 - **MongoDB Atlas Support** Connect to MongoDB Atlas Services.
 - **Grafana Alerting Support** Set up alerting rules based on query result
-- **Legacy Plugin Compatibility:** Easy migrate from the legacy plugin with support for its query syntax.
+- **[Legacy Plugin](https://github.com/JamesOsgood/mongodb-grafana) Compatibility:** Easy migrate from the [legacy plugin](https://github.com/JamesOsgood/mongodb-grafana) with support for its query syntax.
 
 ## Authentication methods
 * No authentication
@@ -73,7 +73,7 @@ Provide the collection name and your MongoDB aggregation pipeline in standard JS
 
 ### JavaScript (Legacy & ShadowRealm)
 
-- **Legacy:** Maintain compatibility with the older plugin's syntax: 
+- **Legacy:** Maintain compatibility with the [legacy plugin](https://github.com/JamesOsgood/mongodb-grafana)'s syntax: 
     ```javascript
     db.listingsAndReviews.aggregate([ /* Your aggregation pipeline (JSON) */ ]); 
     ```

pkg/plugin/datasource.go

@@ -31,7 +31,7 @@ func NewDatasource(ctx context.Context, source backend.DataSourceInstanceSetting
 
 	config, err := models.LoadPluginSettings(source)
 	if err != nil {
-		backend.Logger.Error(fmt.Sprintf("Failed to load plugin settings: %s", err.Error()))
+		backend.Logger.Error("Failed to load plugin settings", "error", err)
 		return nil, err
 	}
 
@@ -84,8 +84,9 @@ func (d *Datasource) QueryData(ctx context.Context, req *backend.QueryDataReques
 }
 
 func (d *Datasource) query(ctx context.Context, _ backend.PluginContext, query backend.DataQuery) backend.DataResponse {
+	backend.Logger.Debug("Executing query", "refId", query.RefID, "json", query.JSON)
+
 	var response backend.DataResponse
-	backend.Logger.Debug("Raw query", query.JSON)
 	var qm queryModel
 	db := d.client.Database(d.database)
 
@@ -105,23 +106,58 @@ func (d *Datasource) query(ctx context.Context, _ backend.PluginContext, query b
 		return backend.ErrDataResponse(backend.StatusBadRequest, fmt.Sprintf("Failed to unmarshal JsonExt: %v", err.Error()))
 	}
 
+	// Set aggregate options
 	aggregateOpts := options.AggregateOptions{}
-	if qm.Timeout > 0 {
-		aggregateOpts.MaxTime = pointer(time.Hour * time.Duration(qm.Timeout))
-		backend.Logger.Debug("Aggregate timeout was set", "timeout", qm.Timeout)
+
+	if qm.AggregateMaxTimeMS > 0 {
+		aggregateOpts.SetMaxTime(time.Hour * time.Duration(qm.AggregateMaxTimeMS))
+
+		backend.Logger.Debug("Aggregate option was set", "maxTime", qm.AggregateMaxTimeMS)
+	}
+
+	if qm.AggregateComment != "" {
+		aggregateOpts.SetComment(qm.AggregateComment)
+
+		backend.Logger.Debug("Aggregate option was set", "comment", qm.AggregateComment)
+	}
+
+	if qm.AggregateBatchSize > 0 {
+		aggregateOpts.SetBatchSize(qm.AggregateBatchSize)
+
+		backend.Logger.Debug("Aggregate option was set", "batchSize", qm.AggregateBatchSize)
+	}
+
+	if qm.AggregateAllowDiskUse {
+		aggregateOpts.SetAllowDiskUse(qm.AggregateAllowDiskUse)
+
+		backend.Logger.Debug("Aggregate option was set", "allowDiskUse", qm.AggregateAllowDiskUse)
+	}
+
+	if qm.AggregateMaxAwaitTime > 0 {
+		aggregateOpts.SetMaxAwaitTime(time.Hour * time.Duration(qm.AggregateMaxAwaitTime))
+
+		backend.Logger.Debug("Aggregate option was set", "maxAwaitTime", qm.AggregateMaxAwaitTime)
+	}
+
+	if qm.AggregateBypassDocumentValidation {
+		aggregateOpts.SetBypassDocumentValidation(qm.AggregateBypassDocumentValidation)
+
+		backend.Logger.Debug("Aggregate option was set", "bypassDocumentValidation", qm.AggregateBypassDocumentValidation)
 	}
 
 	cursor, err := db.Collection(qm.Collection).Aggregate(ctx, pipeline, &aggregateOpts)
 	if err != nil {
-		return backend.ErrDataResponse(backend.StatusBadRequest, fmt.Sprintf("Failed to query: %v", err.Error()))
+		backend.Logger.Error("Failed to execute aggregate", "error", err)
 
+		return backend.ErrDataResponse(backend.StatusBadRequest, fmt.Sprintf("Failed to query: %v", err.Error()))
 	}
+
 	defer cursor.Close(ctx)
 
 	if qm.QueryType == "table" {
 		frame, err := CreateTableFramesFromQuery(ctx, query.RefID, cursor)
 		if err != nil {
-			backend.Logger.Error(err.Error())
+			backend.Logger.Error("Failed to create data frame from query", "error", err)
 			return backend.ErrDataResponse(backend.StatusBadRequest, fmt.Sprintf("Failed to query: %v", err.Error()))
 		}
 
@@ -130,7 +166,8 @@ func (d *Datasource) query(ctx context.Context, _ backend.PluginContext, query b
 	} else {
 		frames, err := CreateTimeSeriesFramesFromQuery(ctx, cursor)
 		if err != nil {
-			backend.Logger.Error(err.Error())
+			backend.Logger.Error("Failed to create time series frames from query", "error", err)
+
 			return backend.ErrDataResponse(backend.StatusBadRequest, fmt.Sprintf("Failed to query: %v", err.Error()))
 		}
 
@@ -148,10 +185,13 @@ func (d *Datasource) query(ctx context.Context, _ backend.PluginContext, query b
 // a datasource is working as expected.
 func (d *Datasource) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) {
 	res := &backend.CheckHealthResult{}
+
 	backend.Logger.Debug("Checking health")
 
 	config, err := models.LoadPluginSettings(*req.PluginContext.DataSourceInstanceSettings)
 	if err != nil {
+		backend.Logger.Error("Failed to load settings", "error", err)
+
 		res.Status = backend.HealthStatusError
 		res.Message = "Unable to load settings"
 		return res, nil
@@ -168,12 +208,13 @@ func (d *Datasource) CheckHealth(ctx context.Context, req *backend.CheckHealthRe
 	if err != nil {
 		res.Status = backend.HealthStatusError
 		res.Message = err.Error()
-		backend.Logger.Error(err.Error())
+
+		backend.Logger.Error("Failed to connect to db", "error", err)
 		return res, nil
 	}
 	defer func() {
 		if err = client.Disconnect(ctx); err != nil {
-			backend.Logger.Error(fmt.Sprintf("Failed to disconnect db: %s", err.Error()))
+			backend.Logger.Error("Failed to disconnect to db", "error", err)
 		}
 	}()
 

pkg/plugin/types.go

@@ -18,9 +18,15 @@ type queryModel struct {
 	Collection    string `json:"collection"`
 	QueryType     string `json:"queryType"`
 	QueryLanguage string `json:"queryLanguage"`
-	Timeout       int    `json:"timeout"`
-}
 
+	// Aggregate options
+	AggregateComment                  string `json:"aggregateComment"`
+	AggregateMaxTimeMS                int    `json:"aggregateMaxTimeMS"`
+	AggregateBatchSize                int32  `json:"aggregateBatchSize"`
+	AggregateAllowDiskUse             bool   `json:"aggregateAllowDiskUse"`
+	AggregateMaxAwaitTime             int    `json:"aggregateMaxAwaitTime"`
+	AggregateBypassDocumentValidation bool   `json:"aggregateBypassDocumentValidation"`
+}
 type TimeSeriesRow[T any] struct {
 	Timestamp time.Time `bson:"ts"`
 	Name      string    `bson:"name"`

src/components/ConfigEditor.tsx

@@ -2,7 +2,6 @@ import React, { ChangeEvent } from "react";
 import { Divider, Field, FieldSet, InlineField, InlineFieldRow, Input, RadioButtonGroup, SecretInput } from "@grafana/ui";
 import { DataSourcePluginOptionsEditorProps, SelectableValue } from "@grafana/data";
 import { MongoDataSourceOptions, MySecureJsonData, MongoDBAuthMethod, ConnectionStringScheme } from "../types";
-;
 
 interface Props extends DataSourcePluginOptionsEditorProps<MongoDataSourceOptions, MySecureJsonData> { }
 

src/components/QueryEditor.tsx

@@ -3,45 +3,52 @@ import {
   ActionMeta,
   Button,
   CodeEditor,
-  Divider,
   Field,
   InlineField,
   InlineFieldRow,
   Input,
   Select,
-
+  ControlledCollapse,
+  InlineSwitch,
+  RadioButtonGroup
 } from "@grafana/ui";
 import { QueryEditorProps, SelectableValue } from "@grafana/data";
 import { DataSource } from "../datasource";
 import { MongoDataSourceOptions, MongoQuery, QueryLanguage, QueryType, DEFAULT_QUERY } from "../types";
-import { parseJsQuery, parseJsQueryLegacy, validateJsonQueryText, validateTimeout } from "../utils";
+import { parseJsQuery, parseJsQueryLegacy, validateJsonQueryText, validatePositiveNumber } from "../utils";
 import * as monacoType from "monaco-editor/esm/vs/editor/editor.api";
 
 type Props = QueryEditorProps<DataSource, MongoQuery, MongoDataSourceOptions>;
 
 const queryTypes: Array<SelectableValue<string>> = [
   {
     label: "Time series",
-    value: QueryType.TIMESERIES
+    value: QueryType.TIMESERIES,
+    icon: "chart-line"
   },
   {
-    label: "Table",
-    value: QueryType.TABLE
+    label: "Data Table",
+    value: QueryType.TABLE,
+    icon: "table"
   }
 ];
 
+const languageOptions: Array<SelectableValue<string>> = [
+  { label: "JSON", value: QueryLanguage.JSON },
+  { label: "JavaScript", value: QueryLanguage.JAVASCRIPT, description: "JavaScript Legacy" },
+  { label: "JavaScript Shadow", value: QueryLanguage.JAVASCRIPT_SHADOW, description: "JavaScript with Evaluation" }
+];
+
 
 export function QueryEditor({ query, onChange, onRunQuery }: Props) {
 
   const codeEditorRef = useRef<monacoType.editor.IStandaloneCodeEditor | null>(null);
   const [queryTextError, setQueryTextError] = useState<string | null>(null);
-  const [timeoutText, setTimeoutText] = useState<string>(query.timeout ? query.timeout.toString() : "");
+  const [isOpen, setIsOpen] = useState(false);
 
-  const optionsLanguage = [
-    { label: "JSON", value: QueryLanguage.JSON },
-    { label: "JavaScript", value: QueryLanguage.JAVASCRIPT, description: "javascript legacy" },
-    { label: "JavaScriptShadow", value: QueryLanguage.JAVASCRIPT_SHADOW, description: "javascript with evaluation" }
-  ];
+  const [maxTimeMSText, setMaxTimeMSText] = useState<string>(query.aggregateMaxTimeMS ? query.aggregateMaxTimeMS.toString() : "");
+  const [maxAwaitTimeMSText, setMaxAwaitTimeMSText] = useState<string>(query.aggregateMaxAwaitTime ? query.aggregateMaxAwaitTime.toString() : "");
+  const [batchSizeText, setBatchSizeText] = useState<string>(query.aggregateBatchSize ? query.aggregateBatchSize.toString() : "");
 
   const onQueryTextChange = (queryText: string) => {
     if (query.queryLanguage === QueryLanguage.JAVASCRIPT || query.queryLanguage === QueryLanguage.JAVASCRIPT_SHADOW) {
@@ -67,24 +74,56 @@ export function QueryEditor({ query, onChange, onRunQuery }: Props) {
     onChange({ ...query, queryLanguage: value.value });
   };
 
-  const onQueryTypeChange = (sv: SelectableValue<string>) => {
-    onChange({ ...query, queryType: sv.value });
+  const onQueryTypeChange = (value: string) => {
+    onChange({ ...query, queryType: value });
   };
 
   const onCollectionChange = (event: ChangeEvent<HTMLInputElement>) => {
     onChange({ ...query, collection: event.target.value });
   };
 
-  const onTimeoutChange = (event: ChangeEvent<HTMLInputElement>) => {
-    setTimeoutText(event.target.value);
+  const onMaxTimeMSChange = (event: ChangeEvent<HTMLInputElement>) => {
+    setMaxTimeMSText(event.target.value);
     console.log(event.target.value);
     if (!event.target.value) {
-      onChange({ ...query, timeout: undefined });
-    } else if (validateTimeout(event.target.value)) {
-      onChange({ ...query, timeout: parseInt(event.target.value, 10) });
+      onChange({ ...query, aggregateMaxTimeMS: undefined });
+    } else if (validatePositiveNumber(event.target.value)) {
+      onChange({ ...query, aggregateMaxTimeMS: parseInt(event.target.value, 10) });
+    }
+  };
+
+  const onMaxAwaitTimeMSChange = (event: ChangeEvent<HTMLInputElement>) => {
+    setMaxAwaitTimeMSText(event.target.value);
+    if (!event.target.value) {
+      onChange({ ...query, aggregateMaxAwaitTime: undefined });
+    } else if (validatePositiveNumber(event.target.value)) {
+      onChange({ ...query, aggregateMaxAwaitTime: parseInt(event.target.value, 10) });
     }
   };
 
+  const onAllowDiskUseChange = (event: ChangeEvent<HTMLInputElement>) => {
+    onChange({
+      ...query, aggregateAllowDiskUse: event.target.checked
+    });
+  };
+
+  const onBatchSizeChange = (event: ChangeEvent<HTMLInputElement>) => {
+    setBatchSizeText(event.target.value);
+    if (!event.target.value) {
+      onChange({ ...query, aggregateBatchSize: undefined });
+    } else if (validatePositiveNumber(event.target.value)) {
+      onChange({ ...query, aggregateBatchSize: parseInt(event.target.value, 10) });
+    }
+  };
+
+  const onBypassDocumentValidationChange = (event: ChangeEvent<HTMLInputElement>) => {
+    onChange({ ...query, aggregateBypassDocumentValidation: event.target.checked });
+  };
+
+  const onCommentChange = (event: ChangeEvent<HTMLInputElement>) => {
+    onChange({ ...query, aggregateComment: event.target.value });
+  };
+
   const onCodeEditorDidMount = (e: monacoType.editor.IStandaloneCodeEditor) => {
     codeEditorRef.current = e;
   };
@@ -101,23 +140,51 @@ export function QueryEditor({ query, onChange, onRunQuery }: Props) {
 
   return (
     <>
+      <Field label="Query Type" description="Choose to query time series or table">
+        <RadioButtonGroup id="query-editor-query-type" options={queryTypes} onChange={onQueryTypeChange} value={query.queryType || QueryType.TIMESERIES} />
+      </Field>
       <InlineFieldRow>
-        <InlineField label="Query Type">
-          <Select id="query-editor-query-type" options={queryTypes} value={query.queryType || QueryType.TIMESERIES} onChange={onQueryTypeChange}></Select>
-        </InlineField>
-        <InlineField label="Collection" tooltip="Enter the collection to query"
-          error="Please enter the collection" invalid={!query.collection}>
-          <Input id="query-editor-collection" onChange={onCollectionChange} value={query.collection} required />
+        <InlineField label="Collection" error="Collection is required" invalid={query.queryLanguage !== QueryLanguage.JAVASCRIPT && !query.collection} tooltip="Name of the MongoDB collection to query">
+          <Input width={25} id="query-editor-collection" onChange={onCollectionChange} value={query.collection} disabled={query.queryLanguage === QueryLanguage.JAVASCRIPT} />
         </InlineField>
-        <InlineField label="Timeout" tooltip="(Optional) The maximum amount of time (in milisecond) that the query can run on the server."
-          error="Invalid timeout" invalid={timeoutText !== "" && !validateTimeout(timeoutText)}>
-          <Input id="query-editor-timeout" onChange={onTimeoutChange} value={timeoutText} />
+        <InlineField label="Query language">
+          <Select id="query-editor-query-language" onChange={onQueryLanguageChange} options={languageOptions} value={query.queryLanguage} width={25} />
         </InlineField>
       </InlineFieldRow>
-      <Divider />
-      <InlineField label="Query language">
-        <Select id="query-editor-use-js-query" onChange={onQueryLanguageChange} options={optionsLanguage} value={query.queryLanguage} />
-      </InlineField>
+      <ControlledCollapse label="Aggregate Options" isOpen={isOpen} onToggle={() => setIsOpen(!isOpen)}>
+        <InlineFieldRow>
+          <InlineField label="Max time(ms)" tooltip="The maximum amount of time that the query can run on the server. The default value is nil, meaning that there is no time limit for query execution."
+            error="Invalid time" invalid={maxTimeMSText !== "" && !validatePositiveNumber(maxTimeMSText)}>
+            <Input id="query-editor-max-time-ms" onChange={onMaxTimeMSChange} value={maxTimeMSText} />
+          </InlineField>
+        </InlineFieldRow>
+        <InlineFieldRow>
+          <InlineField label="Max Await Time(ms)" tooltip="The maximum amount of time that the server should wait for new documents to satisfy a tailable cursor query.">
+            <Input id="query-editor-max-await-time-ms" onChange={onMaxAwaitTimeMSChange} value={maxAwaitTimeMSText} />
+          </InlineField>
+        </InlineFieldRow>
+        <InlineFieldRow>
+          <InlineField label="Comment" tooltip="A string that will be included in server logs, profiling logs, and currentOp queries to help trace the operation.">
+            <Input id="query-editor-comment" onChange={onCommentChange} value={query.aggregateComment} />
+          </InlineField>
+        </InlineFieldRow>
+        <InlineFieldRow>
+          <InlineField label="Batch Size" tooltip="The maximum number of documents to be included in each batch returned by the server."
+            error="Invalid batch size" invalid={batchSizeText !== "" && !validatePositiveNumber(batchSizeText)}>
+            <Input id="query-editor-batch-size" onChange={onBatchSizeChange} value={batchSizeText} />
+          </InlineField>
+        </InlineFieldRow>
+        <InlineFieldRow>
+          <InlineField label="Allow Disk Use" tooltip="If true, the operation can write to temporary files in the _tmp subdirectory of the database directory path on the server. The default value is false.">
+            <InlineSwitch id="query-editor-allow-disk-use" onChange={onAllowDiskUseChange} value={query.aggregateAllowDiskUse} />
+          </InlineField>
+        </InlineFieldRow>
+        <InlineFieldRow>
+          <InlineField label="Bypass Document Validation" tooltip="If true, writes executed as part of the operation will opt out of document-level validation on the server. This option is valid for MongoDB versions >= 3.2 and is ignored for previous server versions. The default value is false.">
+            <InlineSwitch id="query-editor-bypass-document-validation" onChange={onBypassDocumentValidationChange} value={query.aggregateBypassDocumentValidation} />
+          </InlineField>
+        </InlineFieldRow>
+      </ControlledCollapse>
       <Field label="Query Text" description={`Enter the Mongo Aggregation Pipeline (${query.queryLanguage})`}
         error={queryTextError} invalid={queryTextError != null}>
         <CodeEditor onEditorDidMount={onCodeEditorDidMount} width="100%" height={300} language={query.queryLanguage === QueryLanguage.JAVASCRIPT || query.queryLanguage === QueryLanguage.JAVASCRIPT_SHADOW ? "javascript" : "json"}

src/types.ts

@@ -6,7 +6,15 @@ export interface MongoQuery extends DataQuery {
   collection?: string;
   queryType?: string;
   queryLanguage?: string;
-  timeout?: number;
+
+  // Aggregate options
+  aggregateMaxTimeMS?: number;
+  aggregateComment?: string;
+  aggregateBatchSize?: number;
+  aggregateAllowDiskUse?: boolean;
+  aggregateMaxAwaitTime?: number;
+  aggregateBypassDocumentValidation?: boolean;
+  
 }
 
 export const QueryType = {