Add anonymousId and timestamp to web signalData

.changeset/late-bikes-notice.md

@@ -0,0 +1,6 @@
+---
+'@segment/analytics-signals': minor
+'@segment/analytics-signals-runtime': minor
+---
+
+Add anonymousID and timestamp to signals

jest.config.js

@@ -18,5 +18,6 @@ module.exports = () =>
       '<rootDir>/packages/consent/consent-wrapper-onetrust',
       '<rootDir>/scripts',
       '<rootDir>/packages/signals/signals',
+      '<rootDir>/packages/test-helpers',
     ],
   })

packages/signals/signals-integration-tests/src/tests/signals-vanilla/basic.test.ts

@@ -35,7 +35,10 @@ test('network signals fetch', async () => {
     (el) => el.properties!.data.action === 'response'
   )
   expect(responses).toHaveLength(1)
-  expect(responses[0].properties!.data.data).toEqual({ someResponse: 'yep' })
+  const signalData = responses[0].properties!.data
+  expect(signalData.data).toEqual({ someResponse: 'yep' })
+  expect(signalData.anonymousId).toBe(responses[0].anonymousId)
+  expect(signalData.timestamp).toMatch(isoDateRegEx)
 })
 
 test('network signals xhr', async () => {
@@ -61,7 +64,8 @@ test('network signals xhr', async () => {
   expect(responses[0].properties!.data.page).toEqual(commonSignalData.page)
 })
 
-test('instrumentation signals', async () => {
+const isoDateRegEx = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
+test.only('instrumentation signals', async () => {
   /**
    * Make an analytics.page() call, see if it gets sent to the signals endpoint
    */
@@ -70,17 +74,20 @@ test('instrumentation signals', async () => {
     indexPage.waitForSignalsApiFlush(),
   ])
 
-  const isoDateRegEx = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
   const instrumentationEvents =
     indexPage.signalsAPI.getEvents('instrumentation')
   expect(instrumentationEvents).toHaveLength(1)
   const ev = instrumentationEvents[0]
   expect(ev.event).toBe('Segment Signal Generated')
   expect(ev.type).toBe('track')
   const rawEvent = ev.properties!.data.rawEvent
+  expect(ev.properties!.type).toBe('instrumentation')
+  expect(ev.properties!.timestamp).toMatch(isoDateRegEx)
+  expect(ev.properties!.anonymousId).toBe(ev.anonymousId)
+  expect(ev.properties)
   expect(rawEvent).toMatchObject({
     type: 'page',
-    anonymousId: expect.any(String),
+    anonymousId: ev.anonymousId,
     timestamp: expect.stringMatching(isoDateRegEx),
   })
 })
@@ -123,6 +130,8 @@ test('interaction signals', async () => {
     type: 'track',
     properties: {
       type: 'interaction',
+      anonymousId: interactionSignals[0].anonymousId,
+      timestamp: expect.stringMatching(isoDateRegEx),
       data,
     },
   })
@@ -154,6 +163,8 @@ test('navigation signals', async ({ page }) => {
     const ev = indexPage.signalsAPI.lastEvent('navigation')
     expect(ev.properties).toMatchObject({
       type: 'navigation',
+      anonymousId: ev.anonymousId,
+      timestamp: expect.stringMatching(isoDateRegEx),
       data: {
         action: 'pageLoad',
         url: indexPage.url,
@@ -176,6 +187,8 @@ test('navigation signals', async ({ page }) => {
     const ev = indexPage.signalsAPI.lastEvent('navigation')
     expect(ev.properties).toMatchObject({
       index: expect.any(Number),
+      anonymousId: ev.anonymousId,
+      timestamp: expect.stringMatching(isoDateRegEx),
       type: 'navigation',
       data: {
         action: 'urlChange',

packages/signals/signals-integration-tests/src/tests/signals-vanilla/top-level-metadata.test.ts

@@ -0,0 +1,52 @@
+import { test, expect } from '@playwright/test'
+import { IndexPage } from './index-page'
+
+const basicEdgeFn = `
+    // this is a process signal function
+    const processSignal = (signal) => {}
+`
+
+let indexPage: IndexPage
+
+const isoDateRegEx = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
+
+test.beforeEach(async ({ page }) => {
+  indexPage = await new IndexPage().loadAndWait(page, basicEdgeFn)
+})
+
+test('Signals should have anonymousId and timestamp at top level', async () => {
+  await indexPage.network.mockTestRoute()
+  await indexPage.network.makeFetchCall()
+  await Promise.all([
+    indexPage.clickButton(),
+    indexPage.makeAnalyticsPageCall(),
+    indexPage.waitForSignalsApiFlush(),
+    indexPage.waitForTrackingApiFlush(),
+  ])
+
+  const types = [
+    'network',
+    'interaction',
+    'instrumentation',
+    'navigation',
+  ] as const
+
+  const evs = types.map((type) => ({
+    type,
+    networkCalls: indexPage.signalsAPI.getEvents(type),
+  }))
+
+  evs.forEach((events) => {
+    if (!events.networkCalls.length) {
+      throw new Error(`No events found for type ${events.type}`)
+    }
+    events.networkCalls.forEach((event) => {
+      const expected = {
+        anonymousId: event.anonymousId,
+        timestamp: expect.stringMatching(isoDateRegEx),
+        type: event.properties!.type,
+      }
+      expect(event.properties).toMatchObject(expected)
+    })
+  })
+})

packages/signals/signals-runtime/package.json

@@ -26,7 +26,7 @@
     "build:cjs": "yarn tsc -p tsconfig.build.json --outDir ./dist/cjs --module commonjs",
     "build:global": "node build-signals-runtime-global.js",
     "assert-generated": "bash scripts/assert-generated.sh",
-    "watch": "rm -rf dist/esm && yarn build:esm --watch",
+    "watch": "rm -rf dist/esm && yarn build:esm && yarn build:esm --watch",
     "watch:test": "yarn test --watch",
     "tsc": "yarn run -T tsc",
     "eslint": "yarn run -T eslint",

packages/signals/signals-runtime/src/shared/shared-types.ts

@@ -1,5 +1,9 @@
+export type ID = string | null | undefined
+
 export interface BaseSignal {
   type: string
+  anonymousId: ID
+  timestamp: string
 }
 
 export type SignalOfType<

packages/signals/signals-runtime/src/test-helpers/mocks/mock-signal-types-web.ts

@@ -1,3 +1,4 @@
+import { BaseSignal } from '../../shared/shared-types'
 import {
   InteractionSignal,
   NavigationSignal,
@@ -8,6 +9,13 @@ import {
 } from '../../web/web-signals-types'
 // Mock data for testing
 
+type DefaultProps = Pick<BaseSignal, 'anonymousId' | 'timestamp'>
+
+const baseSignalProps: DefaultProps = {
+  anonymousId: '123',
+  timestamp: '2020-01-01T00:00:00.000Z',
+}
+
 export const mockPageData: PageData = {
   url: 'https://www.segment.com/docs/connections/sources/catalog/libraries/website/javascript/',
   path: '/docs/connections/sources/catalog/libraries/website/javascript/',
@@ -29,7 +37,7 @@ export const mockInteractionSignal: InteractionSignal = {
       attributes: { id: 'button1', class: 'btn-primary' },
     },
   },
-  metadata: { timestamp: Date.now() },
+  ...baseSignalProps,
 }
 
 export const mockNavigationSignal: NavigationSignal = {
@@ -41,7 +49,7 @@ export const mockNavigationSignal: NavigationSignal = {
     hash: '#section1',
     prevUrl: 'https://example.com/home',
   },
-  metadata: { timestamp: Date.now() },
+  ...baseSignalProps,
 }
 
 export const mockInstrumentationSignal: InstrumentationSignal = {
@@ -50,7 +58,7 @@ export const mockInstrumentationSignal: InstrumentationSignal = {
     page: mockPageData,
     rawEvent: { type: 'customEvent', detail: 'example' },
   },
-  metadata: { timestamp: Date.now() },
+  ...baseSignalProps,
 }
 
 export const mockNetworkSignal: NetworkSignal = {
@@ -63,7 +71,7 @@ export const mockNetworkSignal: NetworkSignal = {
     method: 'GET',
     data: { key: 'value' },
   },
-  metadata: { timestamp: Date.now() },
+  ...baseSignalProps,
 }
 
 export const mockUserDefinedSignal: UserDefinedSignal = {
@@ -72,5 +80,5 @@ export const mockUserDefinedSignal: UserDefinedSignal = {
     page: mockPageData,
     customField: 'customValue',
   },
-  metadata: { timestamp: Date.now() },
+  ...baseSignalProps,
 }

packages/signals/signals/jest.config.js

@@ -2,5 +2,5 @@ const { createJestTSConfig } = require('@internal/config')
 
 module.exports = createJestTSConfig(__dirname, {
   testEnvironment: 'jsdom',
-  setupFilesAfterEnv: ['./jest.setup.js'],
+  setupFilesAfterEnv: ['./jest.setup.ts'],
 })

packages/signals/signals/jest.setup.ts

@@ -0,0 +1,6 @@
+/* eslint-disable no-undef */
+import './src/test-helpers/jest-extended'
+import { JestSerializers } from '@internal/test-helpers'
+import 'fake-indexeddb/auto'
+globalThis.structuredClone = (v: any) => JSON.parse(JSON.stringify(v))
+expect.addSnapshotSerializer(JestSerializers.jestSnapshotSerializerTimestamp)

packages/signals/signals/package.json

@@ -25,6 +25,7 @@
   ],
   "scripts": {
     ".": "yarn run -T turbo run --filter=@segment/analytics-signals...",
+    "..": "yarn run -T turbo run --filter=...@segment/analytics-signals...",
     "test": "yarn jest",
     "lint": "yarn concurrently 'yarn:eslint .' 'yarn:tsc --noEmit'",
     "build": "rm -rf dist && yarn concurrently 'yarn:build:*'",

packages/signals/signals/src/core/analytics-service/__tests__/analytics-service.test.ts

@@ -42,6 +42,7 @@ describe(AnalyticsService, () => {
       delete call.data.page
       expect(call).toMatchInlineSnapshot(`
         {
+          "anonymousId": "",
           "data": {
             "rawEvent": {
               "context": {
@@ -50,6 +51,7 @@ describe(AnalyticsService, () => {
               "type": "track",
             },
           },
+          "timestamp": <ISO Timestamp>,
           "type": "instrumentation",
         }
       `)

packages/signals/signals/src/core/middleware/signals-ingest/__tests__/client.test.ts

@@ -1,3 +1,4 @@
+import { ISO_TIMESTAMP_REGEX } from '@internal/test-helpers'
 import { createSuccess } from '@segment/analytics-next/src/test-helpers/factories'
 import unfetch from 'unfetch'
 import { getPageData } from '../../../../lib/page-data'
@@ -29,7 +30,9 @@ describe(SignalsIngestClient, () => {
     const ctx = await client.send(signal)
 
     expect(ctx!.event.type).toEqual('track')
-    expect(ctx!.event.properties).toEqual({
+    expect(ctx!.event.properties).toMatchObject({
+      anonymousId: expect.any(String),
+      timestamp: expect.stringMatching(ISO_TIMESTAMP_REGEX),
       type: 'instrumentation',
       index: 0,
       data: {
@@ -53,26 +56,36 @@ describe(SignalsIngestClient, () => {
     })
 
     const ctx = await client.send(signal)
-    expect(ctx!.event.type).toEqual('track')
-    expect(ctx!.event.properties!.type).toBe('network')
-    expect(ctx!.event.properties!.data).toMatchInlineSnapshot(`
+    expect(ctx!.event.properties!).toMatchInlineSnapshot(`
       {
-        "action": "request",
-        "contentType": "application/json",
+        "anonymousId": "",
         "data": {
-          "hello": "XXX",
+          "action": "request",
+          "contentType": "application/json",
+          "data": {
+            "hello": "XXX",
+          },
+          "method": "POST",
+          "page": {
+            "hash": "",
+            "hostname": "localhost",
+            "path": "/",
+            "referrer": "",
+            "search": "",
+            "title": "",
+            "url": "http://localhost/",
+          },
+          "url": "http://foo.com",
         },
-        "method": "POST",
-        "page": {
-          "hash": "",
-          "hostname": "localhost",
-          "path": "/",
-          "referrer": "",
-          "search": "",
-          "title": "",
-          "url": "http://localhost/",
+        "index": 0,
+        "metadata": {
+          "filters": {
+            "allowed": [],
+            "disallowed": [],
+          },
         },
-        "url": "http://foo.com",
+        "timestamp": <ISO Timestamp>,
+        "type": "network",
       }
     `)
   })

packages/signals/signals/src/core/middleware/signals-ingest/__tests__/redact.test.ts

@@ -93,7 +93,7 @@ describe(redactSignalData, () => {
       listener: 'onchange',
       target: createMockTarget({ value: 'XXX', formData: { password: 'XXX' } }),
     })
-    expect(redactSignalData(signal)).toEqual(expected)
+    expect(redactSignalData(signal)).toMatchSignal(expected)
   })
 
   it('should redact attributes in change and in target if the listener is "mutation"', () => {
@@ -117,7 +117,7 @@ describe(redactSignalData, () => {
         innerText: 'XXX',
       }),
     })
-    expect(redactSignalData(signal)).toEqual(expected)
+    expect(redactSignalData(signal)).toMatchSignal(expected)
   })
 
   it('should redact the textContent and innerText in the "target" property if the listener is "contenteditable"', () => {
@@ -133,7 +133,7 @@ describe(redactSignalData, () => {
       change: { textContent: 'XXX' },
       target: createMockTarget({ textContent: 'XXX', innerText: 'XXX' }),
     })
-    expect(redactSignalData(signal)).toEqual(expected)
+    expect(redactSignalData(signal)).toMatchSignal(expected)
   })
 
   it('should redact the values in the "data" property if the type is "network"', () => {
@@ -157,7 +157,7 @@ describe(redactSignalData, () => {
       },
       metadataFixture
     )
-    expect(redactSignalData(signal)).toEqual(expected)
+    expect(redactSignalData(signal)).toMatchSignal(expected)
   })
 
   it('should not mutate the original signal object', () => {

packages/signals/signals/src/core/middleware/signals-ingest/signals-ingest-client.ts

@@ -85,8 +85,7 @@ export class SignalsIngestClient {
 
     return analytics.track(MAGIC_EVENT_NAME, {
       index: this.index++,
-      type: signal.type,
-      data: cleanSignal.data,
+      ...cleanSignal,
     })
   }
 

packages/signals/signals/src/core/middleware/user-info/index.ts

@@ -0,0 +1,17 @@
+import { Signal } from '@segment/analytics-signals-runtime'
+import { UserInfo } from '../../../types'
+import { SignalsMiddleware, SignalsMiddlewareContext } from '../../emitter'
+
+export class UserInfoMiddleware implements SignalsMiddleware {
+  user!: UserInfo
+
+  async load(ctx: SignalsMiddlewareContext) {
+    this.user = await ctx.analyticsInstance.user()
+  }
+
+  process(signal: Signal): Signal {
+    // anonymousId should always exist here unless the user is explicitly setting it to null
+    signal.anonymousId = this.user.anonymousId()
+    return signal
+  }
+}