diff --git a/package.json b/package.json index f9b32ca1..dd90aca0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.1.29", + "version": "1.1.30", "main": "index.ts", "license": "UNLICENSED", "scripts": { @@ -21,6 +21,8 @@ "devDependencies": { "@shelf/jest-mongodb": "^1.2.2", "@types/jest": "^26.0.8", + "@types/lodash.clonedeep": "^4.5.9", + "@types/lodash.mergewith": "^4.6.9", "eslint": "^6.7.2", "eslint-config-codex": "1.2.4", "eslint-plugin-import": "^2.19.1", @@ -37,7 +39,8 @@ "@graphql-tools/schema": "^8.5.1", "@graphql-tools/utils": "^8.9.0", "@hawk.so/nodejs": "^3.1.1", - "@hawk.so/types": "^0.1.31", + "@hawk.so/types": "^0.1.33", + "@n1ru4l/json-patch-plus": "^0.2.0", "@types/amqp-connection-manager": "^2.0.4", "@types/bson": "^4.0.5", "@types/debug": "^4.1.5", @@ -70,6 +73,8 @@ "graphql-upload": "^13", "jsonwebtoken": "^8.5.1", "lodash": "^4.17.15", + "lodash.clonedeep": "^4.5.0", + "lodash.mergewith": "^4.6.2", "migrate-mongo": "^7.0.1", "mime-types": "^2.1.25", "mongodb": "^3.7.3", diff --git a/src/utils/merge.ts b/src/utils/merge.ts new file mode 100644 index 00000000..090ada79 --- /dev/null +++ b/src/utils/merge.ts @@ -0,0 +1,130 @@ +import mergeWith from 'lodash.mergewith'; +import cloneDeep from 'lodash.clonedeep'; +import { patch } from '@n1ru4l/json-patch-plus'; +import { GroupedEventDBScheme, RepetitionDBScheme } from '@hawk.so/types'; + +/** + * One of the features of the events is that their repetition is the difference + * between the original, which greatly optimizes storage. So we need to restore + * the original repetition payload using the very first event and its difference + * between its repetition + * + * @deprecated remove after 6 september 2025 + * @param originalEvent - the very first event we received + * @param repetition - the difference with its repetition, for the repetition we want to display + * @returns fully assembled payload of the current repetition + */ +export function repetitionAssembler(originalEvent: GroupedEventDBScheme['payload'], repetition: GroupedEventDBScheme['payload']): GroupedEventDBScheme['payload'] { + const customizer = (originalParam: any, repetitionParam: any): any => { + if (repetitionParam === null) { + return originalParam; + } + + if (typeof repetitionParam === 'object' && typeof originalParam === 'object') { + /** + * If original event has null but repetition has some value, we need to return repetition value + */ + if (originalParam === null) { + return repetitionParam; + /** + * Otherwise, we need to recursively merge original and repetition values + */ + } else { + return repetitionAssembler(originalParam, repetitionParam); + } + } + + return repetitionParam; + }; + + return mergeWith(cloneDeep(originalEvent), cloneDeep(repetition), customizer); +} + +/** + * Parse addons and context fields from string to object, in db it stores as string + * + * @param payload - the payload of the event + * @param field - the field to parse, can be 'addons' or 'context' + * @returns the payload with parsed field + */ +function parsePayloadField(payload: GroupedEventDBScheme['payload'], field: 'addons' | 'context') { + if (payload && payload[field] && typeof payload[field] === 'string') { + payload[field] = JSON.parse(payload[field] as string); + } + + return payload; +} + +/** + * Stringify addons and context fields from object to string, in db it stores as string + * + * @param payload - the payload of the event + * @param field - the field to stringify, can be 'addons' or 'context' + * @returns the payload with stringified field + */ +function stringifyPayloadField(payload: GroupedEventDBScheme['payload'], field: 'addons' | 'context') { + if (payload && payload[field]) { + payload[field] = JSON.stringify(payload[field]); + } + + return payload; +} + +/** + * Helps to merge original event and repetition due to delta format, + * in case of old delta format, we need to patch the payload + * in case of new delta format, we need to assemble the payload + * + * @param originalEvent {HawkEvent} - The original event + * @param repetition {HawkEventRepetition} - The repetition to process + * @returns {HawkEvent} Updated event with processed repetition payload + */ +export function composeFullRepetitionEvent(originalEvent: GroupedEventDBScheme, repetition: RepetitionDBScheme | undefined): GroupedEventDBScheme { + /** + * Make a deep copy of the original event, because we need to avoid mutating the original event + */ + const event = cloneDeep(originalEvent); + + if (!repetition) { + return event; + } + + /** + * New delta format (repetition.delta is not null) + */ + if (repetition.delta) { + /** + * Parse addons and context fields from string to object before patching + */ + event.payload = parsePayloadField(event.payload, 'addons'); + event.payload = parsePayloadField(event.payload, 'context'); + + event.payload = patch({ + left: event.payload, + delta: JSON.parse(repetition.delta), + }); + + /** + * Stringify addons and context fields from object to string after patching + */ + event.payload = stringifyPayloadField(event.payload, 'addons'); + event.payload = stringifyPayloadField(event.payload, 'context'); + + return event; + } + + /** + * New delta format (repetition.payload is null) and repetition.delta is null (there is no delta between original and repetition) + */ + if (!repetition.payload) { + return event; + } + + /** + * Old delta format (repetition.payload is not null) + * @todo remove after 6 september 2025 + */ + event.payload = repetitionAssembler(event.payload, repetition.payload); + + return event; +} diff --git a/test/utils/merge.test.ts b/test/utils/merge.test.ts new file mode 100644 index 00000000..c065d338 --- /dev/null +++ b/test/utils/merge.test.ts @@ -0,0 +1,366 @@ +import { composeFullRepetitionEvent } from '../../src/utils/merge'; +import { GroupedEventDBScheme, RepetitionDBScheme } from '@hawk.so/types'; + +import { diff } from '@n1ru4l/json-patch-plus'; + +describe('composeFullRepetitionEvent', () => { + const mockOriginalEvent: GroupedEventDBScheme = { + groupHash: 'original-event-1', + totalCount: 1, + catcherType: 'javascript', + payload: { + title: 'Original message', + type: 'error', + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }, + usersAffected: 1, + visitedBy: [], + timestamp: 1640995200, // 2023-01-01T00:00:00Z + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('when repetition is undefined', () => { + it('should return a deep copy of the original event', () => { + /** + * Arrange + */ + const repetition = undefined; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result).toEqual(mockOriginalEvent); + expect(result).toMatchObject(mockOriginalEvent); + expect(result.payload).toMatchObject(mockOriginalEvent.payload); + }); + }); + + describe('when repetition.delta is provided (new delta format)', () => { + it('should parse addons and context, apply patch, and stringify fields back', () => { + /** + * Arrange + */ + const delta = diff({ + left: mockOriginalEvent.payload, + right: { + ...mockOriginalEvent.payload, + title: 'Updated message', + type: 'warning', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Updated message', + type: 'warning', + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + + it('should handle delta with new fields', () => { + /** + * Arrange + */ + const delta = diff({ + left: mockOriginalEvent.payload, + right: { + ...mockOriginalEvent.payload, + release: 'v1.0.0', + catcherVersion: '2.0.0', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Original message', + type: 'error', + release: 'v1.0.0', + catcherVersion: '2.0.0', + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + }); + + describe('when repetition.delta is undefined and repetition.payload is undefined', () => { + it('should return the original event unchanged', () => { + /** + * Arrange + */ + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: undefined, + payload: undefined, + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result).toEqual(mockOriginalEvent); + expect(result).not.toBe(mockOriginalEvent); // Должна быть глубокая копия + }); + }); + + describe('when repetition.delta is undefined and repetition.payload is provided (old delta format)', () => { + it('should use repetitionAssembler to merge payloads', () => { + /** + * Arrange + */ + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: undefined, + payload: { + title: 'Updated message', + type: 'warning', + release: 'v1.0.0', + }, + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Updated message', + type: 'warning', + release: 'v1.0.0', + // Addons and context should be, because old format doesn't remove fields + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + + it('should handle null values in repetition payload', () => { + /** + * Arrange + */ + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: undefined, + payload: { + title: 'Updated title', + type: 'info', + }, + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Updated title', // repetition value replaces original + type: 'info', + // Addons and context should be, because old format doesn't remove fields + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + + it('should preserve original value when repetition payload has null', () => { + /** + * Arrange + */ + const repetition: RepetitionDBScheme = { + groupHash: 'original-event-1', + timestamp: 1640995200, + delta: undefined, + payload: { + title: null as any, + type: 'info', + }, + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(mockOriginalEvent, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'Original message', // null в repetition должно сохранить оригинальное значение + type: 'info', + addons: JSON.stringify({ userId: 123 }), + context: JSON.stringify({ sessionId: 'abc' }), + }); + }); + }); + + describe('edge cases', () => { + it('should handle empty payload in original event', () => { + /** + * Arrange + */ + const eventWithEmptyPayload: GroupedEventDBScheme = { + groupHash: 'event-4', + totalCount: 1, + catcherType: 'javascript', + payload: { + title: 'Empty event', + }, + usersAffected: 1, + visitedBy: [], + timestamp: 1640995200, + }; + + const delta = diff({ + left: eventWithEmptyPayload.payload, + right: { + ...eventWithEmptyPayload.payload, + title: 'New message', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'event-4', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(eventWithEmptyPayload, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'New message', + }); + }); + + it('should handle null payload in original event', () => { + /** + * Arrange + */ + const eventWithNullPayload: GroupedEventDBScheme = { + groupHash: 'event-5', + totalCount: 1, + catcherType: 'javascript', + payload: null as any, + usersAffected: 1, + visitedBy: [], + timestamp: 1640995200, + }; + + const delta = diff({ + left: eventWithNullPayload.payload, + right: { + title: 'New message', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'event-5', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act + */ + const result = composeFullRepetitionEvent(eventWithNullPayload, repetition); + + /** + * Assert + */ + expect(result.payload).toEqual({ + title: 'New message', + }); + }); + + it('should handle invalid JSON in addons or context', () => { + /** + * Arrange + */ + const eventWithInvalidJSON: GroupedEventDBScheme = { + groupHash: 'event-6', + totalCount: 1, + catcherType: 'javascript', + payload: { + title: 'Test', + addons: 'invalid json', + context: 'also invalid', + }, + usersAffected: 1, + visitedBy: [], + timestamp: 1640995200, + }; + + const delta = diff({ + left: eventWithInvalidJSON.payload, + right: { + ...eventWithInvalidJSON.payload, + title: 'Updated', + }, + }); + + const repetition: RepetitionDBScheme = { + groupHash: 'event-6', + timestamp: 1640995200, + delta: JSON.stringify(delta), + }; + + /** + * Act & Assert + */ + expect(() => { + composeFullRepetitionEvent(eventWithInvalidJSON, repetition); + }).toThrow(); // Должно выбросить ошибку при парсинге невалидного JSON + }); + }); +}); \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 9d2d9361..80a104ae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -458,10 +458,10 @@ dependencies: "@types/mongodb" "^3.5.34" -"@hawk.so/types@^0.1.31": - version "0.1.31" - resolved "https://registry.yarnpkg.com/@hawk.so/types/-/types-0.1.31.tgz#fba2c3451e927558bfcc3b1d942baaf8e72ad214" - integrity sha512-o1LeA3JVIUPRSIZegKwAdl4noQ1KYxwr80eisJMlghP9knu6PbYw20rIMyan5qQ3epOWs8gO1CU3iwHZprFiCg== +"@hawk.so/types@^0.1.33": + version "0.1.33" + resolved "https://registry.yarnpkg.com/@hawk.so/types/-/types-0.1.33.tgz#feb077b699b3e0001552588a372e1efe6cd58f40" + integrity sha512-q3AdVxzQ8Qk8qyYiAcAacxNZXWTG/oLmVpjQlcLm2Eh5OJgpaZvH8hQCeRQ/ml1cqbYW8gUrRbMMCS2QOcwxEw== dependencies: "@types/mongodb" "^3.5.34" @@ -720,6 +720,11 @@ semver "^7.3.5" tar "^6.1.11" +"@n1ru4l/json-patch-plus@^0.2.0": + version "0.2.0" + resolved "https://registry.yarnpkg.com/@n1ru4l/json-patch-plus/-/json-patch-plus-0.2.0.tgz#b8fa09fd980c3460dfdc109a7c4cc5590157aa6b" + integrity sha512-pLkJy83/rVfDTyQgDSC8GeXAHEdXNHGNJrB1b7wAyGQu0iv7tpMXntKVSqj0+XKNVQbco40SZffNfVALzIt0SQ== + "@phc/format@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@phc/format/-/format-1.0.0.tgz#b5627003b3216dc4362125b13f48a4daa76680e4" @@ -1077,6 +1082,25 @@ "@types/koa-compose" "*" "@types/node" "*" +"@types/lodash.clonedeep@^4.5.9": + version "4.5.9" + resolved "https://registry.yarnpkg.com/@types/lodash.clonedeep/-/lodash.clonedeep-4.5.9.tgz#ea48276c7cc18d080e00bb56cf965bcceb3f0fc1" + integrity sha512-19429mWC+FyaAhOLzsS8kZUsI+/GmBAQ0HFiCPsKGU+7pBXOQWhyrY6xNNDwUSX8SMZMJvuFVMF9O5dQOlQK9Q== + dependencies: + "@types/lodash" "*" + +"@types/lodash.mergewith@^4.6.9": + version "4.6.9" + resolved "https://registry.yarnpkg.com/@types/lodash.mergewith/-/lodash.mergewith-4.6.9.tgz#7093028a36de3cae4495d03b9d92c351cab1f8bf" + integrity sha512-fgkoCAOF47K7sxrQ7Mlud2TH023itugZs2bUg8h/KzT+BnZNrR2jAOmaokbLunHNnobXVWOezAeNn/lZqwxkcw== + dependencies: + "@types/lodash" "*" + +"@types/lodash@*": + version "4.17.20" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.20.tgz#1ca77361d7363432d29f5e55950d9ec1e1c6ea93" + integrity sha512-H3MHACvFUEiujabxhaI/ImO6gUrd8oOurg7LQtS7mbwIXA/cUqWrvBsaeJ23aZEPk1TAYkurjfMbSELfoCXlGA== + "@types/long@^4.0.0": version "4.0.2" resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" @@ -4616,6 +4640,11 @@ lockfile@^1.0.4: dependencies: signal-exit "^3.0.2" +lodash.clonedeep@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" + integrity sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ== + lodash.includes@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f" @@ -4646,6 +4675,11 @@ lodash.isstring@^4.0.1: resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451" integrity sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw== +lodash.mergewith@^4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/lodash.mergewith/-/lodash.mergewith-4.6.2.tgz#617121f89ac55f59047c7aec1ccd6654c6590f55" + integrity sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ== + lodash.once@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"