PRMDR-876 Virus Scanner Timeouts (#373)

* [PRMDR-876] Implement retry logic for virus scan

* [PRMDR-876] remove unused import

* add temporary console.log to verify delay

* add logging for debug within sandbox

* remove console.log

* extract number of seconds as constant

* [PRMDR-876] fix cypress tests

* [PRMDR-876] unify duplicated imports

* minor refactoring

Author: joefong-nhs

app/cypress/e2e/0-ndr-core-tests/gp_user_workflows/upload_lloyd_george_is_bsol_gp_admin.cy.js

@@ -172,7 +172,7 @@ describe('GP Workflow: Upload Lloyd George record when user is GP admin BSOL and
                     'contain',
                     uploadedFileNames.LG[singleFileUsecaseIndex],
                 );
-                cy.wait(20);
+                cy.wait('@upload_confirm');
 
                 cy.getByTestId('upload-complete-page')
                     .should('include.text', 'Record uploaded for')
@@ -218,6 +218,8 @@ describe('GP Workflow: Upload Lloyd George record when user is GP admin BSOL and
                     .should('contain', uploadedFileNames.LG[multiFileUsecaseIndex][0])
                     .should('contain', uploadedFileNames.LG[multiFileUsecaseIndex][1]);
 
+                cy.wait('@upload_confirm');
+
                 cy.getByTestId('upload-complete-page')
                     .should('include.text', 'Record uploaded for')
                     .should('include.text', 'You have successfully uploaded 2 files')
@@ -260,6 +262,8 @@ describe('GP Workflow: Upload Lloyd George record when user is GP admin BSOL and
                     .should('contain', uploadedFileNames.LG[multiFileUsecaseIndex][0])
                     .should('contain', uploadedFileNames.LG[multiFileUsecaseIndex][1]);
 
+                cy.wait('@upload_confirm');
+
                 cy.getByTestId('upload-complete-page')
                     .should('include.text', 'Record uploaded for')
                     .should('include.text', 'You have successfully uploaded 2 files')
@@ -406,6 +410,7 @@ describe('GP Workflow: Upload Lloyd George record when user is GP admin BSOL and
                 cy.getByTestId('error-box-link').should('exist');
                 cy.getByTestId('error-box-link').click();
                 cy.wait('@s3_retry_upload');
+                cy.wait('@upload_confirm');
 
                 cy.getByTestId('upload-complete-page')
                     .should('include.text', 'Record uploaded for')

app/src/helpers/requests/uploadDocument.test.ts

@@ -1,11 +1,15 @@
 import axios, { AxiosError } from 'axios';
 import { buildDocument, buildTextFile } from '../test/testBuilders';
 import { DOCUMENT_UPLOAD_STATE as documentUploadStates } from '../../types/pages/UploadDocumentsPage/types';
-import { UpdateStateArgs, updateDocumentState } from './uploadDocuments';
+import { UpdateStateArgs, updateDocumentState, virusScanResult } from './uploadDocuments';
+import waitForSeconds from '../utils/waitForSeconds';
 
 // Mock out all top level functions, such as get, put, delete and post:
 jest.mock('axios');
+jest.mock('../utils/waitForSeconds');
+
 const mockedAxios = axios as jest.Mocked<typeof axios>;
+const mockedWaitForSeconds = waitForSeconds as jest.Mocked<typeof waitForSeconds>;
 
 describe('[POST] updateDocumentState', () => {
     test('updateDocumentState handles a 2XX response', async () => {
@@ -23,3 +27,60 @@ describe('[POST] updateDocumentState', () => {
         expect(() => updateDocumentState(args)).not.toThrow(error);
     });
 });
+
+describe('virusScanResult', () => {
+    const virusScanArgs = {
+        documentReference: 'mock_doc_id',
+        baseUrl: '/test',
+        baseHeaders: { 'Content-Type': 'application/json', test: 'test' },
+    };
+    const cleanResponse = { status: 200 };
+    const uncleanResponse = { response: { status: 400 } };
+    const gatewayTimeoutResponse = { response: { status: 504 } };
+
+    it('return CLEAN if virus scan api call result was clean', async () => {
+        mockedAxios.post.mockResolvedValueOnce(cleanResponse);
+
+        const result = await virusScanResult(virusScanArgs);
+
+        expect(result).toEqual(documentUploadStates.CLEAN);
+        expect(mockedWaitForSeconds).not.toBeCalled();
+    });
+
+    it('return INFECTED if virus scan api call result was unclean', async () => {
+        mockedAxios.post.mockRejectedValueOnce(uncleanResponse);
+
+        const result = await virusScanResult(virusScanArgs);
+
+        expect(result).toEqual(documentUploadStates.INFECTED);
+        expect(mockedWaitForSeconds).not.toBeCalled();
+    });
+
+    it('retry up to 3 times if virus scan api call timed out', async () => {
+        mockedAxios.post
+            .mockRejectedValueOnce(gatewayTimeoutResponse)
+            .mockRejectedValueOnce(gatewayTimeoutResponse)
+            .mockResolvedValueOnce(cleanResponse);
+
+        const delay_between_retry_in_seconds = 5;
+
+        const result = await virusScanResult(virusScanArgs);
+
+        expect(result).toEqual(documentUploadStates.CLEAN);
+
+        expect(mockedAxios.post).toBeCalledTimes(3);
+        expect(mockedWaitForSeconds).toBeCalledTimes(2);
+        expect(mockedWaitForSeconds).toHaveBeenCalledWith(delay_between_retry_in_seconds);
+    });
+
+    it('throw an error if timed out for 3 times', async () => {
+        mockedAxios.post.mockRejectedValue(gatewayTimeoutResponse);
+
+        await expect(virusScanResult(virusScanArgs)).rejects.toThrowError(
+            'Virus scan api calls timed-out for 3 attempts.',
+        );
+
+        expect(mockedAxios.post).toBeCalledTimes(3);
+        expect(mockedWaitForSeconds).toBeCalledTimes(3);
+    });
+});

app/src/helpers/requests/uploadDocuments.ts

@@ -10,6 +10,12 @@ import axios, { AxiosError } from 'axios';
 import { S3Upload, S3UploadFields, UploadSession } from '../../types/generic/uploadResult';
 import { Dispatch, SetStateAction } from 'react';
 import { setDocument } from '../../pages/lloydGeorgeUploadPage/LloydGeorgeUploadPage';
+import waitForSeconds from '../utils/waitForSeconds';
+
+const VIRUS_SCAN_RETRY_LIMIT = 3;
+const DELAY_BETWEEN_VIRUS_SCAN_RETRY_IN_SECONDS = 5;
+const TIMEOUT_ERROR_STATUS_CODE = 504;
+const TIMEOUT_ERROR = 'TIMEOUT_ERROR';
 
 type FileKeyBuilder = {
     [key in DOCUMENT_TYPE]: string[];
@@ -52,11 +58,20 @@ type UploadConfirmationArgs = {
     uploadSession: UploadSession;
 };
 
-export const virusScanResult = async ({
-    documentReference,
-    baseUrl,
-    baseHeaders,
-}: VirusScanArgs) => {
+export const virusScanResult = async (virusScanArgs: VirusScanArgs) => {
+    for (let i = 0; i < VIRUS_SCAN_RETRY_LIMIT; i++) {
+        const scanResult = await requestVirusScan(virusScanArgs);
+        if (scanResult === TIMEOUT_ERROR) {
+            await waitForSeconds(DELAY_BETWEEN_VIRUS_SCAN_RETRY_IN_SECONDS);
+            continue;
+        }
+        return scanResult;
+    }
+
+    throw new Error(`Virus scan api calls timed-out for ${VIRUS_SCAN_RETRY_LIMIT} attempts.`);
+};
+
+const requestVirusScan = async ({ documentReference, baseUrl, baseHeaders }: VirusScanArgs) => {
     const virusScanGatewayUrl = baseUrl + endpoints.VIRUS_SCAN;
     const body = { documentReference };
     try {
@@ -67,6 +82,10 @@ export const virusScanResult = async ({
         });
         return DOCUMENT_UPLOAD_STATE.CLEAN;
     } catch (e) {
+        const error = e as AxiosError;
+        if (error.response?.status === TIMEOUT_ERROR_STATUS_CODE) {
+            return TIMEOUT_ERROR;
+        }
         return DOCUMENT_UPLOAD_STATE.INFECTED;
     }
 };

app/src/helpers/utils/waitForSeconds.test.ts

@@ -0,0 +1,25 @@
+import waitForSeconds from './waitForSeconds';
+
+describe('waitForSeconds', () => {
+    it('postpone code execution by given number of seconds', async () => {
+        const mockFunction = jest.fn();
+        const secondsToWait = 2;
+        const testCode = async () => {
+            await waitForSeconds(secondsToWait);
+            mockFunction(Date.now());
+        };
+
+        const startTime = Date.now();
+        await testCode();
+
+        expect(mockFunction).toHaveBeenCalledTimes(1);
+
+        const timestampWhenMockFunctionCalled = mockFunction.mock.calls[0][0];
+        expect(timestampWhenMockFunctionCalled).toBeGreaterThanOrEqual(
+            startTime + 1000 * secondsToWait,
+        );
+        expect(timestampWhenMockFunctionCalled).toBeLessThanOrEqual(
+            startTime + 1000 * (secondsToWait + 1),
+        );
+    });
+});

app/src/helpers/utils/waitForSeconds.ts

@@ -0,0 +1,7 @@
+const waitForSeconds = (seconds: number): Promise<void> => {
+    return new Promise((resolve) => {
+        setTimeout(resolve, seconds * 1000);
+    });
+};
+
+export default waitForSeconds;

app/src/pages/lloydGeorgeUploadPage/LloydGeorgeUploadPage.test.tsx

@@ -34,6 +34,7 @@ jest.mock('moment', () => {
         return jest.requireActual('moment')(arg);
     };
 });
+jest.mock('../../helpers/utils/waitForSeconds');
 
 const mockedUsePatient = usePatient as jest.Mock;
 const mockUploadDocuments = uploadDocuments as jest.Mock;

app/src/pages/lloydGeorgeUploadPage/LloydGeorgeUploadPage.tsx

@@ -23,6 +23,7 @@ import { Outlet, Route, Routes, useNavigate } from 'react-router';
 import { errorToParams } from '../../helpers/utils/errorToParams';
 import LloydGeorgeRetryUploadStage from '../../components/blocks/_lloydGeorge/lloydGeorgeRetryUploadStage/LloydGeorgeRetryUploadStage';
 import { getLastURLPath } from '../../helpers/utils/urlManipulations';
+import waitForSeconds from '../../helpers/utils/waitForSeconds';
 export enum LG_UPLOAD_STAGE {
     SELECT = 0,
     UPLOAD = 1,
@@ -40,6 +41,8 @@ type UpdateDocumentArgs = {
     ref?: string;
 };
 
+const DELAY_BEFORE_VIRUS_SCAN_IN_SECONDS = 3;
+
 export const setDocument = (
     setDocuments: Dispatch<SetStateAction<UploadDocument[]>>,
     { id, state, progress, attempts, ref }: UpdateDocumentArgs,
@@ -68,7 +71,6 @@ function LloydGeorgeUploadPage() {
     const nhsNumber: string = patientDetails?.nhsNumber ?? '';
     const baseUrl = useBaseAPIUrl();
     const baseHeaders = useBaseAPIHeaders();
-    // const [stage, setStage] = useState<LG_UPLOAD_STAGE>(LG_UPLOAD_STAGE.SELECT);
     const [documents, setDocuments] = useState<Array<UploadDocument>>([]);
     const [uploadSession, setUploadSession] = useState<UploadSession | null>(null);
     const confirmedReference = useRef(false);
@@ -164,6 +166,7 @@ function LloydGeorgeUploadPage() {
                     state: DOCUMENT_UPLOAD_STATE.SCANNING,
                     progress: 'scan',
                 });
+                await waitForSeconds(DELAY_BEFORE_VIRUS_SCAN_IN_SECONDS);
                 const virusDocumentState = await virusScanResult({
                     documentReference: document.key ?? '',
                     baseUrl,