Skip to content

Commit cb58440

Browse files
kartikpersistentpraveshkumar1988
kartikpersistent
and
praveshkumar1988
committed
File expiration alert (#953)
* added the expiration alert * expiration days changes * disabled the continue button for expired files * Added gcs_file_cache in response of connect API and removed unused library import * Added GCS_FILE_CACHE on backend_config API * added the GCS_FILE_CACHE env variable check --------- Co-authored-by: Pravesh Kumar <[email protected]>
1 parent 3290d95 commit cb58440

File tree

21 files changed

+235
-293
lines changed

21 files changed

+235
-293
lines changed

‎backend/score.py‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,19 @@
1818
from src.communities import create_communities
1919
from src.neighbours import get_neighbour_nodes
2020
import json
21-
from typing import List, Mapping, Union
21+
from typing import List
2222
from starlette.middleware.sessions import SessionMiddleware
23-
import google_auth_oauthlib.flow
2423
from google.oauth2.credentials import Credentials
2524
import os
2625
from src.logger import CustomLogger
2726
from datetime import datetime, timezone
2827
import time
2928
import gc
30-
from Secweb import SecWeb
31-
from Secweb.StrictTransportSecurity import HSTS
32-
from Secweb.ContentSecurityPolicy import ContentSecurityPolicy
3329
from Secweb.XContentTypeOptions import XContentTypeOptions
3430
from Secweb.XFrameOptions import XFrame
3531
from fastapi.middleware.gzip import GZipMiddleware
3632
from src.ragas_eval import *
37-
from starlette.types import ASGIApp, Message, Receive, Scope, Send
38-
import gzip
33+
from starlette.types import ASGIApp, Receive, Scope, Send
3934
from langchain_neo4j import Neo4jGraph
4035

4136
logger = CustomLogger()
@@ -499,11 +494,13 @@ async def connect(uri=Form(), userName=Form(), password=Form(), database=Form())
499494
start = time.time()
500495
graph = create_graph_database_connection(uri, userName, password, database)
501496
result = await asyncio.to_thread(connection_check_and_get_vector_dimensions, graph, database)
497+
gcs_file_cache = os.environ.get('GCS_FILE_CACHE')
502498
end = time.time()
503499
elapsed_time = end - start
504500
json_obj = {'api_name':'connect','db_url':uri, 'userName':userName, 'database':database,'status':result, 'count':1, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}'}
505501
logger.log_struct(json_obj, "INFO")
506502
result['elapsed_api_time'] = f'{elapsed_time:.2f}'
503+
result['gcs_file_cache'] = gcs_file_cache
507504
return create_api_response('Success',data=result)
508505
except Exception as e:
509506
job_status = "Failed"
@@ -981,6 +978,7 @@ async def backend_connection_configuration():
981978
username= os.getenv('NEO4J_USERNAME')
982979
database= os.getenv('NEO4J_DATABASE')
983980
password= os.getenv('NEO4J_PASSWORD')
981+
gcs_file_cache = os.environ.get('GCS_FILE_CACHE')
984982
if all([uri, username, database, password]):
985983
print(f'uri:{uri}, usrName:{username}, database :{database}, password: {password}')
986984
graph = Neo4jGraph()
@@ -995,6 +993,7 @@ async def backend_connection_configuration():
995993
result["user_name"] = username
996994
result["database"] = database
997995
result["password"] = encoded_password
996+
result['gcs_file_cache'] = gcs_file_cache
998997
return create_api_response('Success',message=f"Backend connection successful",data=result)
999998
else:
1000999
graph_connection = False

‎frontend/src/components/ChatBot/ChatInfoModal.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ const ChatInfoModal: React.FC<chatInfoMessage> = ({
393393
<Typography variant='body-medium'>
394394
Currently ragas evaluation works on{' '}
395395
{supportedLLmsForRagas.map((s, idx) => (
396-
<span className='font-bold'>
396+
<span className='font-bold' key={s}>
397397
{capitalizeWithUnderscore(s) + (idx != supportedLLmsForRagas.length - 1 ? ',' : '')}
398398
</span>
399399
))}

‎frontend/src/components/ChatBot/EntitiesInfo.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,11 +93,11 @@ const EntitiesInfo: FC<EntitiesProps> = ({ loading, mode, graphonly_entities, in
9393
</ul>
9494
</li>
9595
))
96-
: sortedLabels.map((label, index) => {
96+
: sortedLabels.map((label) => {
9797
const entity = groupedEntities[label == 'undefined' ? 'Entity' : label];
9898
return (
9999
<li
100-
key={index}
100+
key={label}
101101
className='flex items-center mb-2 text-ellipsis whitespace-nowrap max-w-[100%)] overflow-hidden'
102102
>
103103
<GraphLabel type='node' className='legend' color={`${entity.color}`} isSelected={false}>

‎frontend/src/components/Content.tsx‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import { useMessageContext } from '../context/UserMessages';
3535
import PostProcessingToast from './Popups/GraphEnhancementDialog/PostProcessingCheckList/PostProcessingToast';
3636
import { getChunkText } from '../services/getChunkText';
3737
import ChunkPopUp from './Popups/ChunkPopUp';
38+
import { isExpired } from '../utils/Utils';
3839

3940
const ConfirmationDialog = lazy(() => import('./Popups/LargeFilePopUp/ConfirmationDialog'));
4041

@@ -55,8 +56,10 @@ const Content: React.FC<ContentProps> = ({
5556
const [openGraphView, setOpenGraphView] = useState<boolean>(false);
5657
const [inspectedName, setInspectedName] = useState<string>('');
5758
const [documentName, setDocumentName] = useState<string>('');
58-
const { setUserCredentials, userCredentials, setConnectionStatus, isGdsActive, isReadOnlyUser } = useCredentials();
59+
const { setUserCredentials, userCredentials, setConnectionStatus, isGdsActive, isReadOnlyUser, isGCSActive } =
60+
useCredentials();
5961
const [showConfirmationModal, setshowConfirmationModal] = useState<boolean>(false);
62+
const [showExpirationModal, setshowExpirationModal] = useState<boolean>(false);
6063
const [extractLoading, setextractLoading] = useState<boolean>(false);
6164
const [retryFile, setRetryFile] = useState<string>('');
6265
const [retryLoading, setRetryLoading] = useState<boolean>(false);
@@ -662,7 +665,8 @@ const Content: React.FC<ContentProps> = ({
662665
const onClickHandler = () => {
663666
const selectedRows = childRef.current?.getSelectedRows();
664667
if (selectedRows?.length) {
665-
let selectedLargeFiles: CustomFile[] = [];
668+
const selectedLargeFiles: CustomFile[] = [];
669+
const expiredFiles: CustomFile[] = [];
666670
for (let index = 0; index < selectedRows.length; index++) {
667671
const parsedData: CustomFile = selectedRows[index];
668672
if (
@@ -673,9 +677,18 @@ const Content: React.FC<ContentProps> = ({
673677
) {
674678
selectedLargeFiles.push(parsedData);
675679
}
680+
if (
681+
parsedData.fileSource === 'local file' &&
682+
(parsedData.status === 'New' || parsedData.status == 'Ready to Reprocess') &&
683+
isExpired(parsedData?.createdAt as Date)
684+
) {
685+
expiredFiles.push(parsedData);
686+
}
676687
}
677688
if (selectedLargeFiles.length) {
678689
setshowConfirmationModal(true);
690+
} else if (expiredFiles.length && isGCSActive) {
691+
setshowExpirationModal(true);
679692
} else {
680693
handleGenerateGraph(selectedRows.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess'));
681694
}
@@ -690,16 +703,34 @@ const Content: React.FC<ContentProps> = ({
690703
}
691704
return false;
692705
});
706+
const expiredFiles = filesData.filter((f) => {
707+
console.log(
708+
f.fileSource === 'local file'
709+
? { isExpired: isExpired(f?.createdAt as Date), name: f.name, createdAt: f.createdAt }
710+
: 'Not Local File'
711+
);
712+
if (
713+
f.fileSource === 'local file' &&
714+
(f.status === 'New' || f.status == 'Ready to Reprocess') &&
715+
isExpired(f?.createdAt as Date)
716+
) {
717+
return true;
718+
}
719+
return false;
720+
});
693721
const selectAllNewFiles = filesData.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess');
694722
const stringified = selectAllNewFiles.reduce((accu, f) => {
695723
const key = f.id;
696724
// @ts-ignore
697725
accu[key] = true;
698726
return accu;
699727
}, {});
728+
console.log(expiredFiles);
700729
setRowSelection(stringified);
701730
if (largefiles.length) {
702731
setshowConfirmationModal(true);
732+
} else if (expiredFiles.length && isGCSActive) {
733+
setshowExpirationModal(true);
703734
} else {
704735
handleGenerateGraph(filesData.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess'));
705736
}
@@ -729,6 +760,19 @@ const Content: React.FC<ContentProps> = ({
729760
></ConfirmationDialog>
730761
</Suspense>
731762
)}
763+
{showExpirationModal && filesForProcessing.length && (
764+
<Suspense fallback={<FallBackDialog />}>
765+
<ConfirmationDialog
766+
open={showExpirationModal}
767+
largeFiles={filesForProcessing}
768+
extractHandler={handleGenerateGraph}
769+
onClose={() => setshowExpirationModal(false)}
770+
loading={extractLoading}
771+
selectedRows={childRef.current?.getSelectedRows() as CustomFile[]}
772+
isLargeDocumentAlert={false}
773+
></ConfirmationDialog>
774+
</Suspense>
775+
)}
732776
{showDeletePopUp && (
733777
<DeletePopUp
734778
open={showDeletePopUp}

‎frontend/src/components/DataSources/Local/DropZone.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ const DropZone: FunctionComponent = () => {
4040
entityEntityRelCount: 0,
4141
communityNodeCount: 0,
4242
communityRelCount: 0,
43+
createdAt: new Date(),
4344
};
4445

4546
const copiedFilesData: CustomFile[] = [...filesData];
@@ -186,6 +187,7 @@ const DropZone: FunctionComponent = () => {
186187
...curfile,
187188
status: 'New',
188189
uploadProgress: 100,
190+
createdAt: new Date(),
189191
};
190192
}
191193
return curfile;
@@ -241,7 +243,7 @@ const DropZone: FunctionComponent = () => {
241243
'application/vnd.ms-powerpoint': ['.pptx'],
242244
'application/vnd.ms-excel': ['.xls'],
243245
'text/markdown': ['.md'],
244-
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':['.xlsx']
246+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'],
245247
},
246248
onDrop: (f: Partial<globalThis.File>[]) => {
247249
onDropHandler(f);

‎frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ export default function DropZoneForSmallLayouts() {
118118
...curfile,
119119
status: 'New',
120120
uploadprogess: 100,
121+
createdAt: new Date(),
121122
};
122123
}
123124
return curfile;
@@ -141,7 +142,7 @@ export default function DropZoneForSmallLayouts() {
141142
'application/vnd.ms-powerpoint': ['.pptx'],
142143
'application/vnd.ms-excel': ['.xls'],
143144
'text/markdown': ['.md'],
144-
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':['.xlsx']
145+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'],
145146
},
146147
onDrop: (f: Partial<globalThis.File>[]) => {
147148
onDropHandler(f);
@@ -176,6 +177,7 @@ export default function DropZoneForSmallLayouts() {
176177
entityEntityRelCount: 0,
177178
communityNodeCount: 0,
178179
communityRelCount: 0,
180+
createdAt: new Date(),
179181
};
180182

181183
const copiedFilesData: CustomFile[] = [...filesData];

‎frontend/src/components/FileTable.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import {
3434
getFileSourceStatus,
3535
isProcessingFileValid,
3636
capitalizeWithUnderscore,
37+
getParsedDate,
3738
} from '../utils/Utils';
3839
import { SourceNode, CustomFile, FileTableProps, UserCredentials, statusupdate, ChildRef } from '../types';
3940
import { useCredentials } from '../context/UserCredentials';
@@ -763,6 +764,7 @@ const FileTable = forwardRef<ChildRef, FileTableProps>((props, ref) => {
763764
entityEntityRelCount: item.entityEntityRelCount ?? 0,
764765
communityNodeCount: item.communityNodeCount ?? 0,
765766
communityRelCount: item.communityRelCount ?? 0,
767+
createdAt: item.createdAt != undefined ? getParsedDate(item?.createdAt) : undefined,
766768
});
767769
}
768770
});
@@ -783,6 +785,7 @@ const FileTable = forwardRef<ChildRef, FileTableProps>((props, ref) => {
783785
}
784786
setIsLoading(false);
785787
} catch (error: any) {
788+
console.log(error);
786789
if (error instanceof Error) {
787790
showErrorToast(error.message);
788791
}

‎frontend/src/components/Graph/ResultOverview.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,10 +143,10 @@ const ResultOverview: React.FunctionComponent<OverViewProps> = ({
143143
</Flex>
144144
<div className='flex gap-2 flex-wrap ml-2'>
145145
<ShowAll initiallyShown={RESULT_STEP_SIZE}>
146-
{nodeCheck.map((nodeLabel, index) => (
146+
{nodeCheck.map((nodeLabel) => (
147147
<LegendsChip
148148
type='node'
149-
key={index}
149+
key={nodeLabel}
150150
label={nodeLabel}
151151
scheme={newScheme}
152152
count={nodeCount(nodes, nodeLabel)}

‎frontend/src/components/Layout/PageLayout.tsx‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ const PageLayout: React.FC = () => {
6161
setErrorMessage,
6262
setShowDisconnectButton,
6363
showDisconnectButton,
64+
setIsGCSActive,
6465
} = useCredentials();
6566
const { cancel } = useSpeechSynthesis();
6667

@@ -97,6 +98,7 @@ const PageLayout: React.FC = () => {
9798
});
9899
setGdsActive(parsedConnection.isgdsActive);
99100
setIsReadOnlyUser(parsedConnection.isReadOnlyUser);
101+
setIsGCSActive(parsedConnection.isGCSActive);
100102
} else {
101103
console.error('Invalid parsed session data:', parsedConnection);
102104
}
@@ -105,7 +107,8 @@ const PageLayout: React.FC = () => {
105107
}
106108
};
107109
// To update credentials if environment values differ
108-
const updateSessionIfNeeded = (envCredentials: UserCredentials, storedSession: string) => {
110+
111+
const updateSessionIfNeeded = (envCredentials: any, storedSession: string) => {
109112
try {
110113
const storedCredentials = JSON.parse(storedSession);
111114
const isDiffCreds =
@@ -115,6 +118,7 @@ const PageLayout: React.FC = () => {
115118
envCredentials.database !== storedCredentials.database;
116119
if (isDiffCreds) {
117120
setUserCredentials(envCredentials);
121+
setIsGCSActive(envCredentials.isGCSActive ?? false);
118122
localStorage.setItem(
119123
'neo4j.connection',
120124
JSON.stringify({
@@ -147,7 +151,9 @@ const PageLayout: React.FC = () => {
147151
database: connectionData.data.database,
148152
isReadonlyUser: !connectionData.data.write_access,
149153
isgdsActive: connectionData.data.gds_status,
154+
isGCSActive: connectionData?.data?.gcs_file_cache === 'True',
150155
};
156+
setIsGCSActive(connectionData?.data?.gcs_file_cache === 'True');
151157
if (session) {
152158
const updated = updateSessionIfNeeded(envCredentials, session);
153159
if (!updated) {
@@ -168,6 +174,7 @@ const PageLayout: React.FC = () => {
168174
userDbVectorIndex: 384,
169175
isReadOnlyUser: envCredentials.isReadonlyUser,
170176
isgdsActive: envCredentials.isgdsActive,
177+
isGCSActive: envCredentials.isGCSActive,
171178
})
172179
);
173180
setConnectionStatus(true);

‎frontend/src/components/Popups/ChunkPopUp/index.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ const ChunkPopUp = ({
5858
<Loader title='loading...'></Loader>
5959
) : (
6060
<ol className='max-h-80 overflow-y-auto flex flex-col gap-4'>
61-
{sortedChunksData.map((c, idx) => (
62-
<li key={`${idx}${c.position}`} className='flex flex-row gap-1'>
61+
{sortedChunksData.map((c) => (
62+
<li key={`${c.position}`} className='flex flex-row gap-1'>
6363
<Flex flexDirection='column' gap='2'>
6464
<Flex flexDirection='row'>
6565
<Typography variant='label'>Position :</Typography>

0 commit comments

Comments
 (0)