Skip to content

Commit 64a8fb8

Browse files
joe-ayoub-segmentjoe-ayoub-segment
authored
STRATCONN-6584 - [AMC] - adding consent support (#3641)
* STRATCONN-6584 - [AMC] - adding consent support * updating as per Chinan guidance * unit tests * tidy up * reformatting * fixing type * adding flag * flag tests * changing flag name * correcting country codes * after talk with Chintan. Add EEA country list * unit tests * consent formatter correctly * remove console log * fixing type * removing bad test
1 parent cbbb5f6 commit 64a8fb8

File tree

8 files changed

+609
-54
lines changed

8 files changed

+609
-54
lines changed

‎packages/destination-actions/src/destinations/amazon-amc/function.ts‎

Lines changed: 73 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,56 @@
1-
import { InvalidAuthenticationError } from '@segment/actions-core'
1+
import { InvalidAuthenticationError, Features } from '@segment/actions-core'
22
import { JSONLikeObject, MultiStatusResponse, PayloadValidationError, RequestClient } from '@segment/actions-core'
33
import { AudienceSettings, Settings } from './generated-types'
44
import type { Payload } from './syncAudiencesToDSP/generated-types'
5-
import { AudienceRecord, HashedPIIObject } from './types'
6-
import { CONSTANTS, RecordsResponseType, REGEX_EXTERNALUSERID } from './utils'
5+
import { MaybeString, AudienceRecord, UserConsent, HashedPIIObject } from './types'
6+
import { FLAG_CONSENT_REQUIRED, CONSTANTS, RecordsResponseType, REGEX_EXTERNALUSERID, COUNTRY_CODES, UK_EEA_COUNTRY_CODES } from './utils'
77
import { processHashing } from '../../lib/hashing-utils'
88
import { AMAZON_AMC_API_VERSION } from './versioning-info'
99

10+
function hasStringValue(value: MaybeString): boolean {
11+
return typeof value === 'string' && value.trim().length > 0
12+
}
13+
14+
function getUserConsent(payloadConsent: Payload['consent'], countryCode: string): UserConsent {
15+
const { ipAddress, amznAdStorage, amznUserData, tcf, gpp } = payloadConsent || {}
16+
17+
if(!COUNTRY_CODES.includes(countryCode)){
18+
throw new PayloadValidationError(`Invalid country code: ${countryCode}. Country code must be a valid ISO 3166-1 alpha-2 code.`)
19+
}
20+
21+
const amzn: NonNullable<UserConsent['consent']>['amzn'] | undefined = hasStringValue(amznAdStorage as MaybeString) && hasStringValue(amznUserData as MaybeString) ? { amznAdStorage: amznAdStorage === 'GRANTED' ? 'GRANTED' : 'DENIED', amznUserData: amznUserData === 'GRANTED' ? 'GRANTED' : 'DENIED' } : undefined
22+
23+
if(UK_EEA_COUNTRY_CODES.includes(countryCode) && !amzn && !hasStringValue(tcf as MaybeString) && !hasStringValue(gpp as MaybeString)){
24+
throw new PayloadValidationError(`Consent required when sending data with UK and EEA country code ${countryCode}. Please provide valid consent for amznAdStorage and amznUserData or TCF or GPP.`)
25+
}
26+
27+
const geo: UserConsent['geo'] = {
28+
...(hasStringValue(ipAddress as MaybeString) && { ipAddress }),
29+
countryCode
30+
}
31+
32+
const consent: UserConsent['consent'] = {
33+
...(amzn && { amzn }),
34+
...(hasStringValue(tcf as MaybeString) && { tcf }),
35+
...(hasStringValue(gpp as MaybeString) && { gpp })
36+
}
37+
38+
const consentData: UserConsent = {
39+
geo,
40+
...(Object.keys(consent).length > 0 && { consent })
41+
}
42+
43+
return consentData
44+
}
45+
1046
export async function processPayload(
1147
request: RequestClient,
1248
settings: Settings,
1349
payload: Payload[],
14-
audienceSettings: AudienceSettings
50+
audienceSettings: AudienceSettings,
51+
features?: Features
1552
) {
16-
const payloadRecord = createPayloadToUploadRecords(payload, audienceSettings)
53+
const payloadRecord = createPayloadToUploadRecords(payload, audienceSettings, features)
1754
// Regular expression to find a audienceId numeric string and replace the quoted audienceId string with an unquoted number
1855
const payloadString = JSON.stringify(payloadRecord).replace(/"audienceId":"(\d+)"/, '"audienceId":$1')
1956

@@ -46,20 +83,26 @@ export async function processPayload(
4683
* @throws {PayloadValidationError} - Throws an error if any externalUserId does not
4784
* match the expected pattern.
4885
*/
49-
export function createPayloadToUploadRecords(payloads: Payload[], audienceSettings: AudienceSettings) {
86+
export function createPayloadToUploadRecords(
87+
payloads: Payload[],
88+
audienceSettings: AudienceSettings,
89+
features?: Features
90+
) {
5091
const records: AudienceRecord[] = []
5192
const { audienceId } = payloads[0]
5293
payloads.forEach((payload: Payload) => {
5394
// Check if the externalUserId matches the pattern
5495
if (!REGEX_EXTERNALUSERID.test(payload.externalUserId)) {
5596
return // Skip to the next iteration
5697
}
98+
const userConsent = features?.[FLAG_CONSENT_REQUIRED] ? getUserConsent(payload.consent, audienceSettings.countryCode) : undefined
5799
const hashedPII = hashedPayload(payload)
58100
const payloadRecord: AudienceRecord = {
59101
externalUserId: payload.externalUserId,
60102
countryCode: audienceSettings.countryCode,
61103
action: payload.event_name == 'Audience Entered' ? CONSTANTS.CREATE : CONSTANTS.DELETE,
62-
hashedPII: [hashedPII]
104+
hashedPII: [hashedPII],
105+
...(userConsent ? { userConsent } : {})
63106
}
64107
records.push(payloadRecord)
65108
})
@@ -79,7 +122,8 @@ export function createPayloadToUploadRecords(payloads: Payload[], audienceSettin
79122
function validateAndPreparePayload(
80123
payloads: Payload[],
81124
multiStatusResponse: MultiStatusResponse,
82-
audienceSettings: AudienceSettings
125+
audienceSettings: AudienceSettings,
126+
features?: Features
83127
) {
84128
const validPayloadIndicesBitmap: number[] = []
85129
const filteredPayloads: AudienceRecord[] = []
@@ -94,12 +138,28 @@ function validateAndPreparePayload(
94138
return
95139
}
96140

141+
let userConsent: UserConsent | undefined
142+
143+
try {
144+
userConsent = features?.[FLAG_CONSENT_REQUIRED] ? getUserConsent(payload.consent, audienceSettings.countryCode) : undefined
145+
}
146+
catch (error) {
147+
multiStatusResponse.setErrorResponseAtIndex(originalBatchIndex, {
148+
status: error.status || 400,
149+
errortype: 'PAYLOAD_VALIDATION_FAILED',
150+
errormessage: error.message,
151+
body: payload as object as JSONLikeObject
152+
})
153+
return
154+
}
155+
97156
const hashedPII = hashedPayload(payload)
98157
const payloadRecord: AudienceRecord = {
99158
externalUserId: payload.externalUserId,
100159
countryCode: audienceSettings.countryCode,
101160
action: payload.event_name == 'Audience Entered' ? CONSTANTS.CREATE : CONSTANTS.DELETE,
102-
hashedPII: [hashedPII]
161+
hashedPII: [hashedPII],
162+
...(userConsent ? { userConsent } : {})
103163
}
104164
filteredPayloads.push(payloadRecord)
105165
validPayloadIndicesBitmap.push(originalBatchIndex)
@@ -125,13 +185,15 @@ export async function processBatchPayload(
125185
request: RequestClient,
126186
settings: Settings,
127187
payloads: Payload[],
128-
audienceSettings: AudienceSettings
188+
audienceSettings: AudienceSettings,
189+
features?: Features
129190
) {
130191
const multiStatusResponse = new MultiStatusResponse()
131192
const { filteredPayloads, validPayloadIndicesBitmap } = validateAndPreparePayload(
132193
payloads,
133194
multiStatusResponse,
134-
audienceSettings
195+
audienceSettings,
196+
features
135197
)
136198

137199
if (!filteredPayloads.length) {

‎packages/destination-actions/src/destinations/amazon-amc/syncAudiencesToDSP/__tests__/__snapshots__/index.test.ts.snap‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Jest Snapshot v1, https://goo.gl/fbAQLP
22

3-
exports[`AmazonAds.syncAudiencesToDSP Normalise and Hash input with extra characters and spaces 1`] = `"{\\"records\\":[{\\"externalUserId\\":\\"test-kochar-01\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"CREATE\\",\\"hashedPII\\":[{\\"address\\":\\"ebb357a6f604e4d893f034561b06fff712d9dbb7082c4b1808418115c5628017\\",\\"postal\\":\\"516b1543763b8b04f15897aeac07eba66f4e36fdac6945bacb6bdac57e44598a\\",\\"phone\\":\\"e1bfd73a5dc6262163ec42add4ebe0229f929db9b23644c1485dbccd05a36363\\",\\"city\\":\\"61a01e4b10bf579b267bdc16858c932339e8388537363c9c0961bcf5520c8897\\",\\"state\\":\\"7e8eea5cc60980270c9ceb75ce8c087d48d726110fd3d17921f774eefd8e18d8\\",\\"email\\":\\"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151\\"}]}],\\"audienceId\\":379909525712777677}"`;
3+
exports[`AmazonAds.syncAudiencesToDSP Normalise and Hash input with extra characters and spaces 1`] = `"{\\"records\\":[{\\"externalUserId\\":\\"test-kochar-01\\",\\"countryCode\\":\\"US\\",\\"action\\":\\"CREATE\\",\\"hashedPII\\":[{\\"address\\":\\"ebb357a6f604e4d893f034561b06fff712d9dbb7082c4b1808418115c5628017\\",\\"postal\\":\\"516b1543763b8b04f15897aeac07eba66f4e36fdac6945bacb6bdac57e44598a\\",\\"phone\\":\\"e1bfd73a5dc6262163ec42add4ebe0229f929db9b23644c1485dbccd05a36363\\",\\"city\\":\\"61a01e4b10bf579b267bdc16858c932339e8388537363c9c0961bcf5520c8897\\",\\"state\\":\\"7e8eea5cc60980270c9ceb75ce8c087d48d726110fd3d17921f774eefd8e18d8\\",\\"email\\":\\"c551027f06bd3f307ccd6abb61edc500def2680944c010e932ab5b27a3a8f151\\"}],\\"userConsent\\":{\\"geo\\":{\\"countryCode\\":\\"US\\"}}}],\\"audienceId\\":379909525712777677}"`;
44

55
exports[`AmazonAds.syncAudiencesToDSP Normalise and Hash personally-identifiable input provided with SHA-256 1`] = `
66
Headers {

‎packages/destination-actions/src/destinations/amazon-amc/syncAudiencesToDSP/__tests__/flag-off.test.ts‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import nock from 'nock'
2+
import { createTestEvent, createTestIntegration } from '@segment/actions-core'
3+
import Destination from '../../index'
4+
5+
const testDestination = createTestIntegration(Destination)
6+
7+
const event = createTestEvent({
8+
context: {
9+
personas: {
10+
audience_settings: {
11+
advertiserId: '585806696618602999',
12+
countryCode: 'US',
13+
description: 'Test Event',
14+
externalAudienceId: '65241452'
15+
},
16+
computation_class: 'audience',
17+
computation_id: 'aud_2g5VilffxpBYqelWk4K0yBzAuFl',
18+
computation_key: 'amazon_ads_audience_6_may_24',
19+
external_audience_id: '379909525712777677',
20+
namespace: 'spa_rHVbZsJXToWAwcmbgpfo36',
21+
space_id: 'spa_rHVbZsJXToWAwcmbgpfo36'
22+
},
23+
traits: {
24+
email: 'test@twilio.com'
25+
}
26+
},
27+
event: 'Audience Entered',
28+
messageId: 'personas_2g5WGNhZtTET4DhSeILpE6muHnH',
29+
properties: {
30+
amazon_ads_audience_6_may_24: true,
31+
audience_key: 'amazon_ads_audience_6_may_24',
32+
externalId: '379909525712777677'
33+
},
34+
receivedAt: '2024-05-06T09:30:38.650Z',
35+
timestamp: '2024-05-06T09:30:22.829Z',
36+
type: 'track',
37+
userId: 'test-kochar-01',
38+
writeKey: 'REDACTED'
39+
})
40+
41+
const settings = {
42+
region: 'https://advertising-api.amazon.com'
43+
}
44+
45+
const mapping = {
46+
email: { '@path': '$.properties.email' },
47+
event_name: { '@path': '$.event' },
48+
externalUserId: { '@path': '$.userId' },
49+
audienceId: { '@path': '$.context.personas.external_audience_id' },
50+
enable_batching: true
51+
}
52+
53+
// These tests cover behavior when the FLAG_CONSENT_REQUIRED feature flag is OFF (no `features` passed).
54+
// Delete this file when the flag is removed from the production code.
55+
describe('AmazonAds.syncAudiencesToDSP (flag off)', () => {
56+
beforeEach(() => {
57+
nock.cleanAll()
58+
jest.resetAllMocks()
59+
})
60+
afterEach(() => {
61+
jest.resetAllMocks()
62+
})
63+
64+
it('should not include userConsent in the record when flag is off', async () => {
65+
nock('https://advertising-api.amazon.com')
66+
.post('/amc/audiences/records')
67+
.matchHeader('content-type', 'application/vnd.amcaudiences.v1+json')
68+
.reply(202, { jobRequestId: '1155d3e3-b18c-4b2b-a3b2-26173cdaf770' })
69+
70+
const response = await testDestination.executeBatch('syncAudiencesToDSP', {
71+
events: [{ ...event, userId: 'test_kochar-02' }],
72+
settings,
73+
mapping
74+
})
75+
76+
expect(response.length).toBe(1)
77+
expect(response[0].status).toBe(202)
78+
expect(response[0]).not.toHaveProperty('sent.userConsent')
79+
})
80+
81+
it('should not throw a consent error for EEA country code when flag is off (single event)', async () => {
82+
nock('https://advertising-api.amazon.com')
83+
.post('/amc/audiences/records')
84+
.matchHeader('content-type', 'application/vnd.amcaudiences.v1+json')
85+
.reply(202, { jobRequestId: '1155d3e3-b18c-4b2b-a3b2-26173cdaf770' })
86+
87+
const deEvent = {
88+
...event,
89+
context: {
90+
...event.context,
91+
personas: {
92+
...event.context!.personas,
93+
audience_settings: {
94+
...event.context!.personas!.audience_settings,
95+
countryCode: 'DE'
96+
}
97+
}
98+
}
99+
}
100+
101+
const response = await testDestination.testAction('syncAudiencesToDSP', {
102+
event: deEvent,
103+
settings,
104+
useDefaultMappings: true
105+
})
106+
107+
expect(response[0].status).toBe(202)
108+
})
109+
110+
it('should not produce a per-record consent error for EEA country code when flag is off (batch)', async () => {
111+
nock('https://advertising-api.amazon.com')
112+
.post('/amc/audiences/records')
113+
.matchHeader('content-type', 'application/vnd.amcaudiences.v1+json')
114+
.reply(202, { jobRequestId: '1155d3e3-b18c-4b2b-a3b2-26173cdaf770' })
115+
116+
const deEvent = {
117+
...event,
118+
userId: 'test_kochar-02',
119+
context: {
120+
...event.context,
121+
personas: {
122+
...event.context!.personas,
123+
audience_settings: {
124+
...event.context!.personas!.audience_settings,
125+
countryCode: 'DE'
126+
}
127+
}
128+
}
129+
}
130+
131+
const response = await testDestination.executeBatch('syncAudiencesToDSP', {
132+
events: [deEvent],
133+
settings,
134+
mapping
135+
})
136+
137+
expect(response.length).toBe(1)
138+
expect(response[0].status).toBe(202)
139+
expect(response[0]).not.toHaveProperty('errortype')
140+
})
141+
})

0 commit comments

Comments
 (0)