diff --git a/packages/cubejs-backend-shared/src/track.ts b/packages/cubejs-backend-shared/src/track.ts index 96932cb7f71d1..c53a231bb563f 100644 --- a/packages/cubejs-backend-shared/src/track.ts +++ b/packages/cubejs-backend-shared/src/track.ts @@ -5,14 +5,17 @@ import { internalExceptions } from './errors'; export type BaseEvent = { event: string, + // It's possible to fill timestamp at the place of logging, otherwise, it will be filled in automatically + timestamp?: string, [key: string]: any, }; -export type Event = BaseEvent & { +export type Event = { id: string, clientTimestamp: string, anonymousId: string, platform: string, + arch: string, nodeVersion: string, sentFrom: 'backend'; }; @@ -79,8 +82,8 @@ export async function track(opts: BaseEvent) { trackEvents.push({ ...opts, + clientTimestamp: opts.timestamp || new Date().toJSON(), id: crypto.randomBytes(16).toString('hex'), - clientTimestamp: new Date().toJSON(), platform: process.platform, arch: process.arch, nodeVersion: process.version, diff --git a/packages/cubejs-server-core/src/core/DevServer.ts b/packages/cubejs-server-core/src/core/DevServer.ts index bcd791c9ce44e..024a8c71f1a58 100644 --- a/packages/cubejs-server-core/src/core/DevServer.ts +++ b/packages/cubejs-server-core/src/core/DevServer.ts @@ -11,7 +11,7 @@ import jwt from 'jsonwebtoken'; import isDocker from 'is-docker'; import type { Application as ExpressApplication, Request, Response } from 'express'; import type { ChildProcess } from 'child_process'; -import { executeCommand, getEnv, keyByDataSource, packageExists } from '@cubejs-backend/shared'; +import { executeCommand, getAnonymousId, getEnv, keyByDataSource, packageExists } from '@cubejs-backend/shared'; import crypto from 'crypto'; import type { BaseDriver } from '@cubejs-backend/query-orchestrator'; @@ -115,7 +115,7 @@ export class DevServer { res.json({ cubejsToken, basePath: options.basePath, - anonymousId: this.cubejsServer.anonymousId, + anonymousId: getAnonymousId(), coreServerVersion: this.cubejsServer.coreServerVersion, dockerVersion: this.options.dockerVersion || null, projectFingerprint: this.cubejsServer.projectFingerprint, diff --git a/packages/cubejs-server-core/src/core/agentCollect.ts b/packages/cubejs-server-core/src/core/agentCollect.ts index be7b226b5ca18..2004d9d2f61f1 100644 --- a/packages/cubejs-server-core/src/core/agentCollect.ts +++ b/packages/cubejs-server-core/src/core/agentCollect.ts @@ -8,6 +8,7 @@ import crypto from 'crypto'; import WebSocket from 'ws'; import zlib from 'zlib'; import { promisify } from 'util'; +import { LoggerFnParams, LoggerFn } from './types'; const deflate = promisify(zlib.deflate); interface AgentTransport { @@ -175,11 +176,11 @@ const clearTransport = () => { agentInterval = null; }; -export default async (event: Record, endpointUrl: string, logger: any) => { +export const agentCollect = async (event: LoggerFnParams, endpointUrl: string, logger: LoggerFn) => { trackEvents.push({ + timestamp: new Date().toJSON(), ...event, id: crypto.randomBytes(16).toString('hex'), - timestamp: new Date().toJSON(), instanceId: getEnv('instanceId'), }); lastEvent = new Date(); @@ -227,3 +228,5 @@ export default async (event: Record, endpointUrl: string, logger: a }, getEnv('agentFlushInterval')); } }; + +export default agentCollect; diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 1c4951e60a647..1db5966d5af31 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -14,7 +14,6 @@ import { CancelableInterval, createCancelableInterval, formatDuration, - getAnonymousId, getEnv, assertDataSource, getRealType, @@ -31,7 +30,7 @@ import { RefreshScheduler, ScheduledRefreshOptions } from './RefreshScheduler'; import { OrchestratorApi, OrchestratorApiOptions } from './OrchestratorApi'; import { CompilerApi } from './CompilerApi'; import { DevServer } from './DevServer'; -import agentCollect from './agentCollect'; +import { agentCollect } from './agentCollect'; import { OrchestratorStorage } from './OrchestratorStorage'; import { prodLogger, devLogger } from './logger'; import { OptsHandler } from './OptsHandler'; @@ -60,6 +59,7 @@ import type { DriverConfig, ScheduledRefreshTimeZonesFn, ContextToCubeStoreRouterIdFn, + LoggerFnParams, } from './types'; import { ContextToOrchestratorIdFn, @@ -171,8 +171,6 @@ export class CubejsServerCore { public projectFingerprint: string | null = null; - public anonymousId: string | null = null; - public coreServerVersion: string | null = null; protected contextAcceptor: ContextAcceptor; @@ -233,7 +231,7 @@ export class CubejsServerCore { this.startScheduledRefreshTimer(); - this.event = async (name, props) => { + this.event = async (event, props: LoggerFnParams) => { if (!this.options.telemetry) { return; } @@ -248,15 +246,12 @@ export class CubejsServerCore { } } - if (!this.anonymousId) { - this.anonymousId = getAnonymousId(); - } - const internalExceptionsEnv = getEnv('internalExceptions'); try { await track({ - event: name, + timestamp: new Date().toJSON(), + event, projectFingerprint: this.projectFingerprint, coreServerVersion: this.coreServerVersion, dockerVersion: getEnv('dockerImageVersion'), @@ -410,7 +405,12 @@ export class CubejsServerCore { if (agentEndpointUrl) { const oldLogger = this.logger; this.preAgentLogger = oldLogger; + this.logger = (msg, params) => { + // Filling timestamp as much as earlier as we can, otherwise it can be incorrect. Because next code is async + // with await points which can be delayed with Node.js micro-tasking. + params.timestamp = params.timestamp || new Date().toJSON(); + oldLogger(msg, params); agentCollect( { diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts index 2f24d11671153..a4b5c749144f1 100644 --- a/packages/cubejs-server-core/src/core/types.ts +++ b/packages/cubejs-server-core/src/core/types.ts @@ -167,7 +167,12 @@ export type ExternalDbTypeFn = (context: RequestContext) => DatabaseType; export type ExternalDriverFactoryFn = (context: RequestContext) => Promise | BaseDriver; export type ExternalDialectFactoryFn = (context: RequestContext) => BaseQuery; -export type LoggerFn = (msg: string, params: Record) => void; +export type LoggerFnParams = { + // It's possible to fill timestamp at the place of logging, otherwise, it will be filled in automatically + timestamp?: string, + [key: string]: any, +}; +export type LoggerFn = (msg: string, params: LoggerFnParams) => void; export type BiToolSyncConfig = { type: string; diff --git a/rust/cubesql/cubesql/src/sql/compiler_cache.rs b/rust/cubesql/cubesql/src/sql/compiler_cache.rs index 56d5172240535..6444d2c89de24 100644 --- a/rust/cubesql/cubesql/src/sql/compiler_cache.rs +++ b/rust/cubesql/cubesql/src/sql/compiler_cache.rs @@ -191,7 +191,8 @@ impl CompilerCache for CompilerCacheImpl { .get(&(compiler_id, protocol.clone())) .cloned() }; - // Double checked locking + + // Double-checked locking let cache_entry = if let Some(cache_entry) = cache_entry { cache_entry } else {