Skip to content

Commit 93b987a

Browse files
tahinitahini
committed
paradata: export valid/invalid fields separately
fixes #1232 This adds 2 columns to the paradata export file: `invalidFields` and `validFields`, which are '|'-separated strings of fields, respectively invalid and valid, triggered by this paradata event. It does not impact the export with values, which still exports all key/value pairs
1 parent 0a5b0ce commit 93b987a

File tree

2 files changed

+181
-44
lines changed

2 files changed

+181
-44
lines changed

‎packages/evolution-backend/src/services/adminExport/__tests__/exportInterviewLogs.test.ts‎

Lines changed: 139 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ describe('exportInterviewLogTask', () => {
8080
timestamp_sec: 5,
8181
event_date: new Date(5 * 1000),
8282
values_by_path: { 'response.household.carNumber': 1, 'response.household.bikeNumber': 10, 'validations.household.carNumber': false, 'validations.household.bikeNumber': true },
83-
unset_paths: [ 'validations.home.region', 'validation.home.country' ]
83+
unset_paths: [ 'validations.home.region', 'validations.home.country' ]
8484
}, {
8585
// No participant response in values_by_path and unset_paths
8686
...commonInterviewData,
@@ -142,13 +142,13 @@ describe('exportInterviewLogTask', () => {
142142
expect(logRows[i]).toEqual(expect.objectContaining({
143143
...commonInterviewDataInRows
144144
}));
145-
const modifiedKeys = Object.entries(logs[i].values_by_path).filter(([key, value]) => value !== null).map(([key, value]) => key).join('|');
146-
const initializedKeys = Object.entries(logs[i].values_by_path).filter(([key, value]) => value === null).map(([key, value]) => key).join('|');
145+
const modifiedKeys = Object.entries(logs[i].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
146+
const initializedKeys = Object.entries(logs[i].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
147147
expect(logRows[i].timestampMs).toEqual(String((i+1) * 1000));
148148
expect(logRows[i].event_date).toEqual(new Date((i+1) * 1000).toISOString());
149149
expect(logRows[i].modifiedFields).toEqual(modifiedKeys);
150150
expect(logRows[i].initializedFields).toEqual(initializedKeys);
151-
expect(logRows[i].unsetFields).toEqual(logs[i].unset_paths !== undefined ? logs[i].unset_paths.join('|') : '');
151+
expect(logRows[i].unsetFields).toEqual(logs[i].unset_paths !== undefined ? logs[i].unset_paths.filter((path: string) => !path.startsWith('validations.')).join('|') : '');
152152
expect(logRows[i].widgetType).toEqual('');
153153
expect(logRows[i].widgetPath).toEqual('');
154154
}
@@ -362,15 +362,17 @@ describe('exportInterviewLogTask', () => {
362362
...commonInterviewDataInRows,
363363
event_type: 'widget_interaction'
364364
}));
365-
const modifiedKeys = Object.entries(log.values_by_path).filter(([key, value]) => value !== null).map(([key, value]) => key).join('|');
366-
const initializedKeys = Object.entries(log.values_by_path).filter(([key, value]) => value === null).map(([key, value]) => key).join('|');
365+
const modifiedKeys = Object.entries(log.values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
366+
const initializedKeys = Object.entries(log.values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
367367
expect(currentLog.timestampMs).toEqual(String((1) * 1000));
368368
expect(currentLog.event_date).toEqual(new Date((1) * 1000).toISOString());
369369
expect(currentLog.modifiedFields).toEqual(modifiedKeys);
370370
expect(currentLog.initializedFields).toEqual(initializedKeys);
371-
expect(currentLog.unsetFields).toEqual(log.unset_paths !== undefined ? log.unset_paths.join('|') : '');
371+
expect(currentLog.unsetFields).toEqual(log.unset_paths !== undefined ? log.unset_paths.filter((path: string) => !path.startsWith('validations.')).join('|') : '');
372372
expect(currentLog.widgetType).toEqual(userAction.widgetType);
373373
expect(currentLog.widgetPath).toEqual(userAction.path);
374+
expect(currentLog.invalidFields).toEqual('');
375+
expect(currentLog.validFields).toEqual('home.geography');
374376
});
375377

376378
test('Test with an event of type widget_interaction with user action, with values', async () => {
@@ -490,34 +492,38 @@ describe('exportInterviewLogTask', () => {
490492
expect(logRows.length).toEqual(buttonLogs.length);
491493

492494
// Test the row values
493-
const modifiedKeysLog1 = Object.entries(buttonLogs[0].values_by_path).filter(([key, value]) => value !== null).map(([key, value]) => key).join('|');
494-
const initializedKeysLog1 = Object.entries(buttonLogs[0].values_by_path).filter(([key, value]) => value === null).map(([key, value]) => key).join('|');
495+
const modifiedKeysLog1 = Object.entries(buttonLogs[0].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
496+
const initializedKeysLog1 = Object.entries(buttonLogs[0].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
495497
expect(logRows[0]).toEqual({
496498
...commonInterviewDataInRows,
497499
event_type: 'button_click',
498500
timestampMs : String((1) * 1000),
499501
event_date: new Date((1) * 1000).toISOString(),
500502
modifiedFields: modifiedKeysLog1,
501503
initializedFields: initializedKeysLog1,
502-
unsetFields: buttonLogs[0].unset_paths !== undefined ? buttonLogs[0].unset_paths.join('|') : '',
504+
unsetFields: buttonLogs[0].unset_paths !== undefined ? buttonLogs[0].unset_paths.filter((path: string) => !path.startsWith('validations.')).join('|') : '',
503505
widgetType: '',
504506
widgetPath: userAction.buttonId,
505507
hiddenWidgets: '',
508+
invalidFields: '',
509+
validFields: 'home.geography'
506510
});
507511

508-
const modifiedKeys = Object.entries(buttonLogs[1].values_by_path).filter(([key, value]) => value !== null).map(([key, value]) => key).join('|');
509-
const initializedKeys = Object.entries(buttonLogs[1].values_by_path).filter(([key, value]) => value === null).map(([key, value]) => key).join('|');
512+
const modifiedKeys = Object.entries(buttonLogs[1].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
513+
const initializedKeys = Object.entries(buttonLogs[1].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
510514
expect(logRows[1]).toEqual({
511515
...commonInterviewDataInRows,
512516
event_type: 'button_click',
513517
timestampMs : String((2) * 1000),
514518
event_date: new Date((2) * 1000).toISOString(),
515519
modifiedFields: modifiedKeys,
516520
initializedFields: initializedKeys,
517-
unsetFields: buttonLogs[1].unset_paths !== undefined ? buttonLogs[1].unset_paths.join('|') : '',
521+
unsetFields: buttonLogs[1].unset_paths !== undefined ? buttonLogs[1].unset_paths.filter((path: string) => !path.startsWith('validations.')).join('|') : '',
518522
widgetType: '',
519523
widgetPath: userActionWithHidden.buttonId,
520524
hiddenWidgets: userActionWithHidden.hiddenWidgets.join('|'),
525+
invalidFields: '',
526+
validFields: ''
521527
});
522528
});
523529

@@ -563,34 +569,38 @@ describe('exportInterviewLogTask', () => {
563569
expect(logRows.length).toEqual(sectionChangeLogs.length);
564570

565571
// Test the row values
566-
const modifiedKeys = Object.entries(sectionChangeLogs[0].values_by_path).filter(([key, value]) => value !== null).map(([key, value]) => key).join('|');
567-
const initializedKeys = Object.entries(sectionChangeLogs[0].values_by_path).filter(([key, value]) => value === null).map(([key, value]) => key).join('|');
572+
const modifiedKeys = Object.entries(sectionChangeLogs[0].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
573+
const initializedKeys = Object.entries(sectionChangeLogs[0].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
568574
expect(logRows[0]).toEqual({
569575
...commonInterviewDataInRows,
570576
event_type: 'section_change',
571577
timestampMs : String((1) * 1000),
572578
event_date: new Date((1) * 1000).toISOString(),
573579
modifiedFields: modifiedKeys,
574580
initializedFields: initializedKeys,
575-
unsetFields: sectionChangeLogs[0].unset_paths !== undefined ? sectionChangeLogs[0].unset_paths.join('|') : '',
581+
unsetFields: sectionChangeLogs[0].unset_paths !== undefined ? sectionChangeLogs[0].unset_paths.filter((path: string) => !path.startsWith('validations.')).join('|') : '',
576582
widgetType: '',
577583
widgetPath: userAction.targetSection.sectionShortname,
578584
hiddenWidgets: '',
585+
invalidFields: '',
586+
validFields: 'home.geography'
579587
});
580588

581-
const modifiedKeys2 = Object.entries(sectionChangeLogs[1].values_by_path).filter(([key, value]) => value !== null).map(([key, value]) => key).join('|');
582-
const initializedKeys2 = Object.entries(sectionChangeLogs[1].values_by_path).filter(([key, value]) => value === null).map(([key, value]) => key).join('|');
589+
const modifiedKeys2 = Object.entries(sectionChangeLogs[1].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
590+
const initializedKeys2 = Object.entries(sectionChangeLogs[1].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
583591
expect(logRows[1]).toEqual({
584592
...commonInterviewDataInRows,
585593
event_type: 'section_change',
586594
timestampMs : String((2) * 1000),
587595
event_date: new Date((2) * 1000).toISOString(),
588596
modifiedFields: modifiedKeys2,
589597
initializedFields: initializedKeys2,
590-
unsetFields: sectionChangeLogs[1].unset_paths !== undefined ? sectionChangeLogs[1].unset_paths.join('|') : '',
598+
unsetFields: sectionChangeLogs[1].unset_paths !== undefined ? sectionChangeLogs[1].unset_paths.filter((path: string) => !path.startsWith('validations.')).join('|') : '',
591599
widgetType: '',
592600
widgetPath: userAction.targetSection.sectionShortname + '/' + (userActionWithHidden.targetSection.iterationContext || []).join('/'),
593601
hiddenWidgets: userActionWithHidden.hiddenWidgets.join('|'),
602+
invalidFields: '',
603+
validFields: ''
594604
});
595605

596606
});
@@ -629,19 +639,21 @@ describe('exportInterviewLogTask', () => {
629639
expect(logRows.length).toEqual(languageLogs.length);
630640

631641
// Test the row values
632-
const modifiedKeysLog1 = Object.entries(languageLogs[0].values_by_path).filter(([key, value]) => value !== null).map(([key, value]) => key).join('|');
633-
const initializedKeysLog1 = Object.entries(languageLogs[0].values_by_path).filter(([key, value]) => value === null).map(([key, value]) => key).join('|');
642+
const modifiedKeysLog1 = Object.entries(languageLogs[0].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
643+
const initializedKeysLog1 = Object.entries(languageLogs[0].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
634644
expect(logRows[0]).toEqual({
635645
...commonInterviewDataInRows,
636646
event_type: 'language_change',
637647
timestampMs : String((1) * 1000),
638648
event_date: new Date((1) * 1000).toISOString(),
639649
modifiedFields: modifiedKeysLog1,
640650
initializedFields: initializedKeysLog1,
641-
unsetFields: languageLogs[0].unset_paths !== undefined ? languageLogs[0].unset_paths.join('|') : '',
651+
unsetFields: languageLogs[0].unset_paths !== undefined ? languageLogs[0].unset_paths.filter((path: string) => !path.startsWith('validations.')).join('|') : '',
642652
widgetType: '',
643653
widgetPath: '',
644654
hiddenWidgets: '',
655+
invalidFields: '',
656+
validFields: 'home.geography'
645657
});
646658
});
647659

@@ -679,19 +691,121 @@ describe('exportInterviewLogTask', () => {
679691
expect(logRows.length).toEqual(interviewOpenLogs.length);
680692

681693
// Test the row values
682-
const modifiedKeysLog1 = Object.entries(interviewOpenLogs[0].values_by_path).filter(([key, value]) => value !== null).map(([key, value]) => key).join('|');
683-
const initializedKeysLog1 = Object.entries(interviewOpenLogs[0].values_by_path).filter(([key, value]) => value === null).map(([key, value]) => key).join('|');
694+
const modifiedKeysLog1 = Object.entries(interviewOpenLogs[0].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
695+
const initializedKeysLog1 = Object.entries(interviewOpenLogs[0].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
684696
expect(logRows[0]).toEqual({
685697
...commonInterviewDataInRows,
686698
event_type: 'interview_open',
687699
timestampMs : String((1) * 1000),
688700
event_date: new Date((1) * 1000).toISOString(),
689701
modifiedFields: modifiedKeysLog1,
690702
initializedFields: initializedKeysLog1,
691-
unsetFields: interviewOpenLogs[0].unset_paths !== undefined ? interviewOpenLogs[0].unset_paths.join('|') : '',
703+
unsetFields: interviewOpenLogs[0].unset_paths.filter((path: string) => !path.startsWith('validations.')).join('|'),
692704
widgetType: '',
693705
widgetPath: '',
694706
hiddenWidgets: '',
707+
invalidFields: '',
708+
validFields: 'home.geography'
709+
});
710+
});
711+
712+
test('Test with various events, with validations `true` and `false`', async () => {
713+
// Add some log statements with validations data: first statement has only true, second has one false, third has both true and false
714+
const interviewLogs: { [key: string]: any }[] = [{
715+
...commonInterviewData,
716+
event_type: 'button_click',
717+
timestamp_sec: 1,
718+
event_date: new Date(1 * 1000),
719+
values_by_path: { 'response.home.geography': { type: 'Point', coordinates: [ 1, 1 ] }, 'validations.home.geography': true, 'response.household.size': 3, 'response._activeTripId': null },
720+
unset_paths: [ 'response.home.someField', 'validations.home.someField' ],
721+
user_action: { type: 'buttonClick', buttonId: 'response.someField' }
722+
}, {
723+
...commonInterviewData,
724+
event_type: 'widget_interaction',
725+
timestamp_sec: 2,
726+
event_date: new Date(2 * 1000),
727+
values_by_path: { 'validations.home.someField': false },
728+
user_action: { type: 'widgetInteraction', path: 'response.home.someField', value: 'someValue', widgetType: 'radio' }
729+
},{
730+
...commonInterviewData,
731+
event_type: 'button_click',
732+
timestamp_sec: 3,
733+
event_date: new Date(3 * 1000),
734+
values_by_path: { 'validations.home.geography': true, 'validations.home.household.size': false, 'validations.home.someField': false },
735+
unset_paths: [ 'response.home.someField' ],
736+
user_action: { type: 'buttonClick', buttonId: 'response.someField' }
737+
}];
738+
// Add the logs to the stream
739+
mockGetInterviewLogsStream.mockReturnValue(new ObjectReadableMock(interviewLogs) as any);
740+
741+
const fileName = await exportInterviewLogTask({});
742+
743+
// Check the file content of the exported logs
744+
expect(mockCreateStream).toHaveBeenCalledTimes(1);
745+
expect(mockGetInterviewLogsStream).toHaveBeenCalledWith(undefined);
746+
747+
const csvFileName = Object.keys(fileStreams).find((filename) => filename.endsWith(fileName));
748+
expect(csvFileName).toBeDefined();
749+
750+
const csvStream = fileStreams[csvFileName as string];
751+
// There should be one row per log
752+
expect(csvStream.data.length).toEqual(interviewLogs.length);
753+
754+
// Get the actual rows in the file data
755+
const logRows = await getCsvFileRows(csvStream.data);
756+
// There should be one row per log
757+
expect(logRows.length).toEqual(interviewLogs.length);
758+
759+
// Test the row values
760+
const modifiedKeysLog1 = Object.entries(interviewLogs[0].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
761+
const initializedKeysLog1 = Object.entries(interviewLogs[0].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
762+
expect(logRows[0]).toEqual({
763+
...commonInterviewDataInRows,
764+
event_type: 'button_click',
765+
timestampMs : String((1) * 1000),
766+
event_date: new Date((1) * 1000).toISOString(),
767+
modifiedFields: modifiedKeysLog1,
768+
initializedFields: initializedKeysLog1,
769+
unsetFields: interviewLogs[0].unset_paths !== undefined ? interviewLogs[0].unset_paths.filter((path: string) => !path.startsWith('validations.')).join('|') : '',
770+
widgetType: '',
771+
widgetPath: interviewLogs[0].user_action.buttonId,
772+
hiddenWidgets: '',
773+
invalidFields: '',
774+
validFields: 'home.geography'
775+
});
776+
777+
const modifiedKeysLog2 = Object.entries(interviewLogs[1].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
778+
const initializedKeysLog2 = Object.entries(interviewLogs[1].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
779+
expect(logRows[1]).toEqual({
780+
...commonInterviewDataInRows,
781+
event_type: 'widget_interaction',
782+
timestampMs : String((2) * 1000),
783+
event_date: new Date((2) * 1000).toISOString(),
784+
modifiedFields: modifiedKeysLog2,
785+
initializedFields: initializedKeysLog2,
786+
unsetFields: interviewLogs[1].unset_paths !== undefined ? interviewLogs[1].unset_paths.join('|') : '',
787+
widgetType: interviewLogs[1].user_action.widgetType,
788+
widgetPath: interviewLogs[1].user_action.path,
789+
hiddenWidgets: '',
790+
invalidFields: 'home.someField',
791+
validFields: ''
792+
});
793+
794+
const modifiedKeysLog3 = Object.entries(interviewLogs[2].values_by_path).filter(([key, value]) => value !== null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
795+
const initializedKeysLog3 = Object.entries(interviewLogs[2].values_by_path).filter(([key, value]) => value === null && !key.startsWith('validations.')).map(([key, value]) => key).join('|');
796+
expect(logRows[2]).toEqual({
797+
...commonInterviewDataInRows,
798+
event_type: 'button_click',
799+
timestampMs : String((3) * 1000),
800+
event_date: new Date((3) * 1000).toISOString(),
801+
modifiedFields: modifiedKeysLog3,
802+
initializedFields: initializedKeysLog3,
803+
unsetFields: interviewLogs[2].unset_paths !== undefined ? interviewLogs[2].unset_paths.join('|') : '',
804+
widgetType: '',
805+
widgetPath: interviewLogs[2].user_action.buttonId,
806+
hiddenWidgets: '',
807+
invalidFields: ['home.household.size', 'home.someField'].join('|'),
808+
validFields: 'home.geography'
695809
});
696810
});
697811

0 commit comments

Comments
 (0)