Dataframe parsing bug fixes (openobserve#16)

* fix: getField
* Fix bug which skips the first column regardless of if it is a timestamp
* date(timestamp + "Z") was breaking timestamps. Use whatever O2 gives us, unless it's microseconds which needs converting to MS.
* Only add Time column if it's part of the request [breaking for queries relying on this implicit field, but deduplicating for the queries that manually request it including example requests]

* fix: getGraphDataFrame

* [bugfix] No longer skip first requested column
* [bugfix] Remove type: number for non-time fields which breaks string responses
* Detect timestamp column in request

* fix: Add getColumnsFromQuery quote stripping

[big] The dataframe parser would fail to assign fields wrapped in quotes i.e. select 1 as "server"

* change: Utilize datasource defined timestamp column automatically in data frame parser

* Implement openobserve#14 fix in openobserve#15 to pass tests

Author: jealouscloud

package.json

@@ -57,7 +57,7 @@
     "eslint-plugin-deprecation": "^2.0.0"
   },
   "engines": {
-    "node": ">=16"
+    "node": ">=18"
   },
   "dependencies": {
     "@emotion/css": "^11.1.3",

src/components/QueryEditor.tsx

@@ -32,12 +32,6 @@ export const QueryEditor = ({ query, onChange, onRunQuery, datasource, app, data
     setIsLoading(isLoading.slice(1));
   };
 
-  const isInDashboard = useMemo(() => app === 'panel-editor', [app]);
-
-  const getTimeStampColumnName = () => {
-    return datasource.instanceSettings?.jsonData?.timestamp_column || '_timestamp';
-  };
-
   useEffect(() => {
     startLoading();
     getOrganizations({ url: datasource.url, page_num: 0, page_size: 1000, sort_by: 'id' })

src/datasource.ts

@@ -72,20 +72,20 @@ export class DataSource
 
       // As we don't show histogram for sql mode in explore
       if (options.app === 'explore' && target?.refId?.includes(REF_ID_STARTER_LOG_VOLUME) && target.sqlMode) {
-        return getGraphDataFrame([], target, options.app);
+        return getGraphDataFrame([], target, options.app, this.timestampColumn);
       }
 
       this.cachedQuery.requestQuery = JSON.stringify(reqData);
       this.cachedQuery.isFetching = true;
       return this.doRequest(target, reqData)
         .then((response) => {
           if (options.app === 'panel-editor' || options.app === 'dashboard') {
-            return getGraphDataFrame(response.hits, target, options.app);
+            return getGraphDataFrame(response.hits, target, options.app, this.timestampColumn);
           }
 
           const logsDataFrame = getLogsDataFrame(response.hits, target, this.streamFields, this.timestampColumn);
 
-          const graphDataFrame = getGraphDataFrame(response?.aggs?.histogram || [], target, options.app);
+          const graphDataFrame = getGraphDataFrame(response?.aggs?.histogram || [], target, options.app, this.timestampColumn);
 
           this.cachedQuery.promise?.resolve({ graph: graphDataFrame, logs: logsDataFrame });
 

src/features/log/queryResponseBuilder.ts

@@ -36,7 +36,12 @@ export const getLogsDataFrame = (
   return logsData;
 };
 
-export const getGraphDataFrame = (data: any, target: MyQuery, app: string) => {
+export const getGraphDataFrame = (
+  data: any,
+  target: MyQuery,
+  app: string,
+  timestampColumn = '_timestamp'
+) => {
   const graphData = getDefaultDataFrame(target.refId, 'graph');
 
   let fields = ['zo_sql_key', 'zo_sql_num'];
@@ -49,39 +54,52 @@ export const getGraphDataFrame = (data: any, target: MyQuery, app: string) => {
     }
   }
 
-  graphData.addField({
-    config: {
-      filterable: true,
-    },
-    name: 'Time',
-    type: FieldType.time,
-  });
-
-  for (let i = 1; i < fields.length; i++) {
-    graphData.addField({
-      name: fields[i],
-      type: FieldType.number,
-    });
+  for (let i = 0; i < fields.length; i++) {
+    if (fields[i] === timestampColumn) {
+      graphData.addField({
+        config: {
+          filterable: true,
+        },
+        name: 'Time',
+        type: FieldType.time,
+      });
+    } else {
+      graphData.addField({
+        name: fields[i],
+      });
+    }
   }
 
   if (!data.length) {
     return graphData;
   }
 
   data.forEach((log: any) => {
-    graphData.add(getField(log, fields));
+    graphData.add(getField(log, fields, timestampColumn));
   });
 
   return graphData;
 };
 
-const getField = (log: any, columns: any) => {
-  let field: any = {
-    Time: new Date(log[columns[0]] + 'Z').getTime(),
-  };
-
-  for (let i = 1; i < columns.length; i++) {
-    field[columns[i]] = log[columns[i]];
+const getField = (log: any, columns: any, timestampColumn: string) => {
+  let field: any = {};
+
+  for (let i = 0; i < columns.length; i++) {
+    let col_name = columns[i];
+    let col_value = log[col_name]
+    if (col_name === timestampColumn) {
+      // We have to convert microseconds if we receive them
+      // 500 billion / year 17814 is probably a good threshold for milliseconds
+      if (col_value > 500_000_000_000) {
+        col_value = convertTimeToMs(col_value);
+        field["Time"] = col_value;
+      } else {
+        // Convert any other date fmt
+        field["Time"] = new Date(col_value).getTime();
+      }
+    } else {
+      field[col_name] = log[col_name];
+    }
   }
 
   return field;
@@ -127,7 +145,9 @@ const getColumnsFromQuery = (query: string) => {
 
     // If alias exists, use that, otherwise use column name
     if (aliasMatch) {
-      columnNames.push(aliasMatch[1]);
+      // SQL alias may have quotes, strip those.
+      let stripped = aliasMatch[1].replace(/^['"]|['"]$/g, '');
+      columnNames.push(stripped);
     } else {
       columnNames.push(column);
     }