From 6e6149cdf7cda2286a30ab7278213a71c89ff21a Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Tue, 24 Jun 2025 17:21:26 -0700 Subject: [PATCH 01/11] WIP - pipes through statsContext to evaluateLiquid directive function --- packages/core/src/destination-kit/action.ts | 2 +- packages/core/src/mapping-kit/index.ts | 25 +++++++++++++------ .../core/src/mapping-kit/liquid-directive.ts | 20 +++++++++++++-- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/packages/core/src/destination-kit/action.ts b/packages/core/src/destination-kit/action.ts index fc8872885ab..1a07f81dece 100644 --- a/packages/core/src/destination-kit/action.ts +++ b/packages/core/src/destination-kit/action.ts @@ -318,7 +318,7 @@ export class Action JSONLike +type Directive = (options: JSONValue, payload: JSONObject, statsContext?: StatsContext | undefined) => JSONLike type StringDirective = (value: string, payload: JSONObject) => JSONLike interface Directives { @@ -42,7 +43,7 @@ function registerStringDirective(name: string, fn: StringDirective): void { }) } -function runDirective(obj: JSONObject, payload: JSONObject): JSONLike { +function runDirective(obj: JSONObject, payload: JSONObject, statsContext?: StatsContext | undefined): JSONLike { const name = Object.keys(obj).find((key) => key.startsWith('@')) as string const directiveFn = directives[name] const value = obj[name] @@ -51,6 +52,10 @@ function runDirective(obj: JSONObject, payload: JSONObject): JSONLike { throw new Error(`${name} is not a valid directive, got ${realTypeOf(directiveFn)}`) } + if (name === '@liquid') { + return directiveFn(value, payload, statsContext) + } + return directiveFn(value, payload) } @@ -326,8 +331,8 @@ registerDirective('@excludeWhenNull', (value, payload) => { return cleanNulls(resolved) }) -registerDirective('@liquid', (opts, payload) => { - return evaluateLiquid(opts, payload) +registerDirective('@liquid', (opts, payload, statsContext) => { + return evaluateLiquid(opts, payload, statsContext) }) // Recursively remove all null values from an object @@ -381,13 +386,13 @@ function getMappingToProcess(mapping: JSONLikeObject): JSONLikeObject { * @param payload - the input data to apply to the mapping directives * @todo support arrays or array directives? */ -function resolve(mapping: JSONLike, payload: JSONObject): JSONLike { +function resolve(mapping: JSONLike, payload: JSONObject, statsContext?: StatsContext | undefined): JSONLike { if (!isObject(mapping) && !isArray(mapping)) { return mapping } if (isDirective(mapping)) { - return runDirective(mapping, payload) + return runDirective(mapping, payload, statsContext) } if (Array.isArray(mapping)) { @@ -409,7 +414,11 @@ function resolve(mapping: JSONLike, payload: JSONObject): JSONLike { * @param mapping - the directives and raw values * @param data - the input data to apply to directives */ -function transform(mapping: JSONLikeObject, data: InputData | undefined = {}): JSONObject { +function transform( + mapping: JSONLikeObject, + data: InputData | undefined = {}, + statsContext?: StatsContext | undefined +): JSONObject { const realType = realTypeOf(data) if (realType !== 'object') { throw new Error(`data must be an object, got ${realType}`) @@ -420,7 +429,7 @@ function transform(mapping: JSONLikeObject, data: InputData | undefined = {}): J // throws if the mapping config is invalid validate(mappingToProcess) - const resolved = resolve(mappingToProcess, data as JSONObject) + const resolved = resolve(mappingToProcess, data as JSONObject, statsContext) const cleaned = removeUndefined(resolved) // Cast because we know there are no `undefined` values anymore diff --git a/packages/core/src/mapping-kit/liquid-directive.ts b/packages/core/src/mapping-kit/liquid-directive.ts index 2ba6c4d55cc..2f480f2d36c 100644 --- a/packages/core/src/mapping-kit/liquid-directive.ts +++ b/packages/core/src/mapping-kit/liquid-directive.ts @@ -1,4 +1,5 @@ import { Liquid } from 'liquidjs' +import { StatsContext } from '../destination-kit' const liquidEngine = new Liquid({ renderLimit: 500, // 500 ms @@ -58,7 +59,8 @@ export function getLiquidKeys(liquidValue: string): string[] { return liquidEngine.fullVariablesSync(liquidValue) } -export function evaluateLiquid(liquidValue: any, event: any): string { +export function evaluateLiquid(liquidValue: any, event: any, statsContext?: StatsContext | undefined): string { + const fieldId = 'TODO' if (typeof liquidValue !== 'string') { // type checking of @liquid directive is done in validate.ts as well throw new Error('liquid template value must be a string') @@ -72,7 +74,21 @@ export function evaluateLiquid(liquidValue: any, event: any): string { throw new Error('liquid template values are limited to 1000 characters') } - const res = liquidEngine.parseAndRenderSync(liquidValue, event) + let status = 'success' + let res + const start = performance.now() + try { + res = liquidEngine.parseAndRenderSync(liquidValue, event) + } catch (_e) { + status = 'fail' + } + const duration = performance.now() - start + + statsContext?.statsClient?.histogram('liquid.template.evaluation_ms', duration, [ + ...statsContext.tags, + `field_id:${fieldId}`, + `result:${status}` // status = success/fail, not in the psudocode for simplicity + ]) if (typeof res !== 'string') { return 'error' From 9ce22c535c706312fc395a2002bf853dbccd0a3e Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Thu, 26 Jun 2025 11:14:39 -0700 Subject: [PATCH 02/11] WIP - pipes through statsContext to evaluateLiquid directive function Tracks liquid metrics when batching as well WIP passes tags hopefully --- packages/core/src/destination-kit/action.ts | 2 +- packages/core/src/mapping-kit/index.ts | 24 +++++++++++++++---- .../core/src/mapping-kit/liquid-directive.ts | 2 -- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/core/src/destination-kit/action.ts b/packages/core/src/destination-kit/action.ts index 1a07f81dece..975a3613a97 100644 --- a/packages/core/src/destination-kit/action.ts +++ b/packages/core/src/destination-kit/action.ts @@ -387,7 +387,7 @@ export class Action resolve(value, payload)) + return mapping.map((value) => resolve(value, payload, statsContext)) } const resolved: JSONLikeObject = {} for (const key of Object.keys(mapping)) { - resolved[key] = resolve(mapping[key], payload) + let originalTags: string[] = [] + const statsTagsExist = statsContext?.tags !== undefined + + if (statsTagsExist) { + originalTags = statsContext.tags + statsContext.tags = [...statsContext.tags, `fieldKey:${key}`] + } + + resolved[key] = resolve(mapping[key], payload, statsContext) + + if (statsTagsExist) { + statsContext.tags = originalTags + } } return resolved @@ -441,7 +453,11 @@ function transform( * @param mapping - the directives and raw values * @param data - the array input data to apply to directives */ -function transformBatch(mapping: JSONLikeObject, data: Array | undefined = []): JSONObject[] { +function transformBatch( + mapping: JSONLikeObject, + data: Array | undefined = [], + statsContext?: StatsContext | undefined +): JSONObject[] { const realType = realTypeOf(data) if (!isArray(data)) { throw new Error(`data must be an array, got ${realType}`) @@ -452,7 +468,7 @@ function transformBatch(mapping: JSONLikeObject, data: Array | undefi // throws if the mapping config is invalid validate(mappingToProcess) - const resolved = data.map((d) => resolve(mappingToProcess, d as JSONObject)) + const resolved = data.map((d) => resolve(mappingToProcess, d as JSONObject, statsContext)) // Cast because we know there are no `undefined` values after `removeUndefined` return removeUndefined(resolved) as JSONObject[] diff --git a/packages/core/src/mapping-kit/liquid-directive.ts b/packages/core/src/mapping-kit/liquid-directive.ts index 2f480f2d36c..bd3fbed5000 100644 --- a/packages/core/src/mapping-kit/liquid-directive.ts +++ b/packages/core/src/mapping-kit/liquid-directive.ts @@ -60,7 +60,6 @@ export function getLiquidKeys(liquidValue: string): string[] { } export function evaluateLiquid(liquidValue: any, event: any, statsContext?: StatsContext | undefined): string { - const fieldId = 'TODO' if (typeof liquidValue !== 'string') { // type checking of @liquid directive is done in validate.ts as well throw new Error('liquid template value must be a string') @@ -86,7 +85,6 @@ export function evaluateLiquid(liquidValue: any, event: any, statsContext?: Stat statsContext?.statsClient?.histogram('liquid.template.evaluation_ms', duration, [ ...statsContext.tags, - `field_id:${fieldId}`, `result:${status}` // status = success/fail, not in the psudocode for simplicity ]) From 856634acf710278105483494eefd1d5361945391 Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Thu, 26 Jun 2025 09:46:21 -0700 Subject: [PATCH 03/11] Adds action and destination metadata values to tags --- packages/core/src/destination-kit/index.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/core/src/destination-kit/index.ts b/packages/core/src/destination-kit/index.ts index d6ab886f4cd..d2d1feb9d8e 100644 --- a/packages/core/src/destination-kit/index.ts +++ b/packages/core/src/destination-kit/index.ts @@ -725,6 +725,17 @@ export class Destination { ): Promise { const isBatch = Array.isArray(events) + if (options?.statsContext?.tags !== undefined) { + options.statsContext.tags = [ + ...options.statsContext.tags, + `actionConfigId:${subscription.ConfigID}`, + `actionId:${subscription.ActionID}`, + `destinationConfigId:${subscription.ConfigID}`, + `sourceId:${subscription.ProjectID}`, + `partnerAction:${subscription.partnerAction}` + ] + } + const subscriptionStartedAt = time() const actionSlug = subscription.partnerAction const input = { From 5508da6cb29edba8563936f70230a76dffe99b98 Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Thu, 26 Jun 2025 16:10:24 -0700 Subject: [PATCH 04/11] Removes Subscription values which are undefined --- packages/core/src/destination-kit/index.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/core/src/destination-kit/index.ts b/packages/core/src/destination-kit/index.ts index d2d1feb9d8e..45927603b6f 100644 --- a/packages/core/src/destination-kit/index.ts +++ b/packages/core/src/destination-kit/index.ts @@ -726,14 +726,7 @@ export class Destination { const isBatch = Array.isArray(events) if (options?.statsContext?.tags !== undefined) { - options.statsContext.tags = [ - ...options.statsContext.tags, - `actionConfigId:${subscription.ConfigID}`, - `actionId:${subscription.ActionID}`, - `destinationConfigId:${subscription.ConfigID}`, - `sourceId:${subscription.ProjectID}`, - `partnerAction:${subscription.partnerAction}` - ] + options.statsContext.tags = [...options.statsContext.tags, `partnerAction:${subscription.partnerAction}`] } const subscriptionStartedAt = time() From 28b117cf61000749f80aef185a2aaa58b176bafc Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Thu, 26 Jun 2025 16:32:56 -0700 Subject: [PATCH 05/11] Imports performance --- packages/core/src/mapping-kit/liquid-directive.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/mapping-kit/liquid-directive.ts b/packages/core/src/mapping-kit/liquid-directive.ts index bd3fbed5000..4dca1885118 100644 --- a/packages/core/src/mapping-kit/liquid-directive.ts +++ b/packages/core/src/mapping-kit/liquid-directive.ts @@ -1,5 +1,6 @@ import { Liquid } from 'liquidjs' import { StatsContext } from '../destination-kit' +import { performance } from 'perf_hooks' const liquidEngine = new Liquid({ renderLimit: 500, // 500 ms From 084b860d3db54b9273a38442f9fb6c40fe2df49b Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Thu, 26 Jun 2025 16:44:14 -0700 Subject: [PATCH 06/11] Throws error only after logging out metrics --- .../core/src/mapping-kit/liquid-directive.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/core/src/mapping-kit/liquid-directive.ts b/packages/core/src/mapping-kit/liquid-directive.ts index 4dca1885118..ff90fb2accb 100644 --- a/packages/core/src/mapping-kit/liquid-directive.ts +++ b/packages/core/src/mapping-kit/liquid-directive.ts @@ -74,19 +74,25 @@ export function evaluateLiquid(liquidValue: any, event: any, statsContext?: Stat throw new Error('liquid template values are limited to 1000 characters') } - let status = 'success' let res const start = performance.now() + let duration try { res = liquidEngine.parseAndRenderSync(liquidValue, event) - } catch (_e) { - status = 'fail' + } catch (e) { + duration = performance.now() - start + + statsContext?.statsClient?.histogram('liquid.template.evaluation_ms', duration, [ + ...statsContext.tags, + `result:fail` // status = success/fail, not in the psudocode for simplicity + ]) + throw e } - const duration = performance.now() - start + duration = performance.now() - start statsContext?.statsClient?.histogram('liquid.template.evaluation_ms', duration, [ ...statsContext.tags, - `result:${status}` // status = success/fail, not in the psudocode for simplicity + `result:success` // status = success/fail, not in the psudocode for simplicity ]) if (typeof res !== 'string') { From 8850513eba3268c701611e2841e766434b060956 Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Mon, 30 Jun 2025 16:32:42 -0700 Subject: [PATCH 07/11] Does not attempt to resolve the perf_hooks package as it's not browser supported --- packages/browser-destinations/webpack.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/browser-destinations/webpack.config.js b/packages/browser-destinations/webpack.config.js index 94e6b324f7f..c01dd758c31 100644 --- a/packages/browser-destinations/webpack.config.js +++ b/packages/browser-destinations/webpack.config.js @@ -104,6 +104,7 @@ const unobfuscatedOutput = { mainFields: ['exports', 'module', 'browser', 'main'], extensions: ['.ts', '.js'], fallback: { + perf_hooks: false, // perf_hooks is not available in the browser. This fixes a build error. vm: require.resolve('vm-browserify'), crypto: false }, From 1964c15d81325161451e2d271f44a91a21abb4f2 Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Mon, 30 Jun 2025 16:45:22 -0700 Subject: [PATCH 08/11] Removes perf_hooks usage since it proved to be a never-ending rabbithole of import errors --- packages/core/src/mapping-kit/liquid-directive.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/core/src/mapping-kit/liquid-directive.ts b/packages/core/src/mapping-kit/liquid-directive.ts index ff90fb2accb..e1ec3c15658 100644 --- a/packages/core/src/mapping-kit/liquid-directive.ts +++ b/packages/core/src/mapping-kit/liquid-directive.ts @@ -1,6 +1,5 @@ import { Liquid } from 'liquidjs' import { StatsContext } from '../destination-kit' -import { performance } from 'perf_hooks' const liquidEngine = new Liquid({ renderLimit: 500, // 500 ms @@ -75,12 +74,12 @@ export function evaluateLiquid(liquidValue: any, event: any, statsContext?: Stat } let res - const start = performance.now() + const start = Date.now() let duration try { res = liquidEngine.parseAndRenderSync(liquidValue, event) } catch (e) { - duration = performance.now() - start + duration = Date.now() - start statsContext?.statsClient?.histogram('liquid.template.evaluation_ms', duration, [ ...statsContext.tags, @@ -88,7 +87,7 @@ export function evaluateLiquid(liquidValue: any, event: any, statsContext?: Stat ]) throw e } - duration = performance.now() - start + duration = Date.now() - start statsContext?.statsClient?.histogram('liquid.template.evaluation_ms', duration, [ ...statsContext.tags, From c2d87044e27612ce6254a167bca24e7f08553b28 Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Mon, 30 Jun 2025 16:55:09 -0700 Subject: [PATCH 09/11] Reverts packages/browser-destinations/webpack.config.js --- packages/browser-destinations/webpack.config.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/browser-destinations/webpack.config.js b/packages/browser-destinations/webpack.config.js index c01dd758c31..94e6b324f7f 100644 --- a/packages/browser-destinations/webpack.config.js +++ b/packages/browser-destinations/webpack.config.js @@ -104,7 +104,6 @@ const unobfuscatedOutput = { mainFields: ['exports', 'module', 'browser', 'main'], extensions: ['.ts', '.js'], fallback: { - perf_hooks: false, // perf_hooks is not available in the browser. This fixes a build error. vm: require.resolve('vm-browserify'), crypto: false }, From b29a7121b9e53ff863e246439187e306c8962727 Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Thu, 3 Jul 2025 09:14:02 -0700 Subject: [PATCH 10/11] Dry suggestion: uses finally block to call histogram --- .../core/src/mapping-kit/liquid-directive.ts | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/core/src/mapping-kit/liquid-directive.ts b/packages/core/src/mapping-kit/liquid-directive.ts index e1ec3c15658..75342f1779e 100644 --- a/packages/core/src/mapping-kit/liquid-directive.ts +++ b/packages/core/src/mapping-kit/liquid-directive.ts @@ -73,26 +73,22 @@ export function evaluateLiquid(liquidValue: any, event: any, statsContext?: Stat throw new Error('liquid template values are limited to 1000 characters') } - let res + let res: string const start = Date.now() - let duration + let status: 'success' | 'fail' = 'success' + try { res = liquidEngine.parseAndRenderSync(liquidValue, event) } catch (e) { - duration = Date.now() - start - + status = 'fail' + throw e + } finally { + const duration = Date.now() - start statsContext?.statsClient?.histogram('liquid.template.evaluation_ms', duration, [ ...statsContext.tags, - `result:fail` // status = success/fail, not in the psudocode for simplicity + `result:${status}` ]) - throw e } - duration = Date.now() - start - - statsContext?.statsClient?.histogram('liquid.template.evaluation_ms', duration, [ - ...statsContext.tags, - `result:success` // status = success/fail, not in the psudocode for simplicity - ]) if (typeof res !== 'string') { return 'error' From 349c8cbd67c2ec3b7c5e9ab4d74b5fa06175a141 Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Thu, 3 Jul 2025 09:26:43 -0700 Subject: [PATCH 11/11] Only append tags when isLiquidDirective is true --- packages/core/src/mapping-kit/index.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/core/src/mapping-kit/index.ts b/packages/core/src/mapping-kit/index.ts index e43c7b92921..1ecea5de2e5 100644 --- a/packages/core/src/mapping-kit/index.ts +++ b/packages/core/src/mapping-kit/index.ts @@ -9,6 +9,7 @@ import { arrify } from '../arrify' import { flattenObject } from './flatten' import { evaluateLiquid } from './liquid-directive' import { StatsContext } from '../destination-kit' +import { isLiquidDirective } from './value-keys' export type InputData = { [key: string]: unknown } export type Features = { [key: string]: boolean } @@ -392,7 +393,12 @@ function resolve(mapping: JSONLike, payload: JSONObject, statsContext?: StatsCon } if (isDirective(mapping)) { - return runDirective(mapping, payload, statsContext) + if (isLiquidDirective(mapping)) { + // Only include stats, and therefore extra fieldKey tags, if the mapping is a liquid directive to save on costs + return runDirective(mapping, payload, statsContext) + } + + return runDirective(mapping, payload) } if (Array.isArray(mapping)) {