data importer with format fixes (#1301)

* data importer with format fixes

* Update frontend/src/components/Layout/PageLayout.tsx

Co-authored-by: Copilot <[email protected]>

* fomrat issues

* heading name change

* Update frontend/src/components/Layout/PageLayout.tsx

Co-authored-by: Copilot <[email protected]>

* name correction

---------

Co-authored-by: Copilot <[email protected]>

Author: prakriti-solankey

frontend/src/components/Layout/PageLayout.tsx

@@ -16,13 +16,14 @@ import { envConnectionAPI } from '../../services/ConnectAPI';
 import { healthStatus } from '../../services/HealthStatus';
 import { useAuth0 } from '@auth0/auth0-react';
 import { showErrorToast } from '../../utils/Toasts';
-import { APP_SOURCES, LOCAL_KEYS } from '../../utils/Constants';
+import { APP_SOURCES } from '../../utils/Constants';
 import { createDefaultFormData } from '../../API/Index';
 import LoadDBSchemaDialog from '../Popups/GraphEnhancementDialog/EnitityExtraction/LoadExistingSchema';
 import PredefinedSchemaDialog from '../Popups/GraphEnhancementDialog/EnitityExtraction/PredefinedSchemaDialog';
 import { SKIP_AUTH } from '../../utils/Constants';
 import { useNavigate } from 'react-router';
 import { deduplicateByFullPattern, deduplicateNodeByValue } from '../../utils/Utils';
+import DataImporterSchemaDialog from '../Popups/GraphEnhancementDialog/EnitityExtraction/DataImporter';
 
 const GCSModal = lazy(() => import('../DataSources/GCS/GCSModal'));
 const S3Modal = lazy(() => import('../DataSources/AWS/S3Modal'));
@@ -186,13 +187,20 @@ const PageLayout: React.FC = () => {
     setSchemaValRels,
     setDbNodes,
     setDbRels,
-    setSchemaView,
     setPreDefinedNodes,
     setPreDefinedRels,
     setPreDefinedPattern,
     allPatterns,
     selectedNodes,
     selectedRels,
+    dataImporterSchemaDialog,
+    setDataImporterSchemaDialog,
+    setImporterPattern,
+    setImporterNodes,
+    setImporterRels,
+    setSourceOptions,
+    setTargetOptions,
+    setTypeOptions,
   } = useFileContext();
   const navigate = useNavigate();
   const { user, isAuthenticated } = useAuth0();
@@ -380,10 +388,9 @@ const PageLayout: React.FC = () => {
         const combined = [...rels, ...prevRels];
         return deduplicateByFullPattern(combined);
       });
-      setSchemaView('text');
-      localStorage.setItem(LOCAL_KEYS.source, JSON.stringify(updatedSource));
-      localStorage.setItem(LOCAL_KEYS.type, JSON.stringify(updatedType));
-      localStorage.setItem(LOCAL_KEYS.target, JSON.stringify(updatedTarget));
+      setSourceOptions((prev) => [...prev, ...updatedSource]);
+      setTargetOptions((prev) => [...prev, ...updatedTarget]);
+      setTypeOptions((prev) => [...prev, ...updatedType]);
     },
     []
   );
@@ -409,7 +416,6 @@ const PageLayout: React.FC = () => {
         triggeredFrom: 'loadExistingSchemaApply',
         show: true,
       });
-      setSchemaView('db');
       setDbNodes(nodes);
       setCombinedNodesVal((prevNodes: OptionType[]) => {
         const combined = [...nodes, ...prevNodes];
@@ -420,9 +426,9 @@ const PageLayout: React.FC = () => {
         const combined = [...rels, ...prevRels];
         return deduplicateByFullPattern(combined);
       });
-      localStorage.setItem(LOCAL_KEYS.source, JSON.stringify(updatedSource));
-      localStorage.setItem(LOCAL_KEYS.type, JSON.stringify(updatedType));
-      localStorage.setItem(LOCAL_KEYS.target, JSON.stringify(updatedTarget));
+      setSourceOptions((prev) => [...prev, ...updatedSource]);
+      setTargetOptions((prev) => [...prev, ...updatedTarget]);
+      setTypeOptions((prev) => [...prev, ...updatedType]);
     },
     []
   );
@@ -447,7 +453,6 @@ const PageLayout: React.FC = () => {
         triggeredFrom: 'predefinedSchemaApply',
         show: true,
       });
-      setSchemaView('preDefined');
       setPreDefinedNodes(nodes);
       setCombinedNodesVal((prevNodes: OptionType[]) => {
         const combined = [...nodes, ...prevNodes];
@@ -458,9 +463,47 @@ const PageLayout: React.FC = () => {
         const combined = [...rels, ...prevRels];
         return deduplicateByFullPattern(combined);
       });
-      localStorage.setItem(LOCAL_KEYS.source, JSON.stringify(updatedSource));
-      localStorage.setItem(LOCAL_KEYS.type, JSON.stringify(updatedType));
-      localStorage.setItem(LOCAL_KEYS.target, JSON.stringify(updatedTarget));
+      setSourceOptions((prev) => [...prev, ...updatedSource]);
+      setTargetOptions((prev) => [...prev, ...updatedTarget]);
+      setTypeOptions((prev) => [...prev, ...updatedType]);
+    },
+    []
+  );
+
+  const handleImporterApply = useCallback(
+    (
+      newPatterns: string[],
+      nodes: OptionType[],
+      rels: OptionType[],
+      updatedSource: OptionType[],
+      updatedTarget: OptionType[],
+      updatedType: OptionType[]
+    ) => {
+      setImporterPattern((prevPatterns: string[]) => {
+        const uniquePatterns = Array.from(new Set([...newPatterns, ...prevPatterns]));
+        return uniquePatterns;
+      });
+      setCombinedPatternsVal((prevPatterns: string[]) => {
+        const uniquePatterns = Array.from(new Set([...newPatterns, ...prevPatterns]));
+        return uniquePatterns;
+      });
+      setDataImporterSchemaDialog({
+        triggeredFrom: 'importerSchemaApply',
+        show: true,
+      });
+      setImporterNodes(nodes);
+      setCombinedNodesVal((prevNodes: OptionType[]) => {
+        const combined = [...nodes, ...prevNodes];
+        return deduplicateNodeByValue(combined);
+      });
+      setImporterRels(rels);
+      setCombinedRelsVal((prevRels: OptionType[]) => {
+        const combined = [...rels, ...prevRels];
+        return deduplicateByFullPattern(combined);
+      });
+      setSourceOptions((prev) => [...prev, ...updatedSource]);
+      setTargetOptions((prev) => [...prev, ...updatedTarget]);
+      setTypeOptions((prev) => [...prev, ...updatedType]);
     },
     []
   );
@@ -477,6 +520,10 @@ const PageLayout: React.FC = () => {
     setShowTextFromSchemaDialog({ triggeredFrom: 'schemadialog', show: true });
   }, []);
 
+  const openDataImporterSchema = useCallback(() => {
+    setDataImporterSchemaDialog({ triggeredFrom: 'schemadialog', show: true });
+  }, []);
+
   const openChatBot = useCallback(() => setShowChatBot(true), []);
 
   return (
@@ -565,6 +612,20 @@ const PageLayout: React.FC = () => {
         }}
         onApply={handlePredinedApply}
       ></PredefinedSchemaDialog>
+      <DataImporterSchemaDialog
+        open={dataImporterSchemaDialog.show}
+        onClose={() => {
+          setDataImporterSchemaDialog({ triggeredFrom: '', show: false });
+          switch (dataImporterSchemaDialog.triggeredFrom) {
+            case 'enhancementtab':
+              toggleEnhancementDialog();
+              break;
+            default:
+              break;
+          }
+        }}
+        onApply={handleImporterApply}
+      ></DataImporterSchemaDialog>
       {isLargeDesktop ? (
         <div
           className={`layout-wrapper ${!isLeftExpanded ? 'drawerdropzoneclosed' : ''} ${
@@ -596,6 +657,7 @@ const PageLayout: React.FC = () => {
             openTextSchema={openTextSchema}
             openLoadSchema={openLoadSchema}
             openPredefinedSchema={openPredefinedSchema}
+            openDataImporterSchema={openDataImporterSchema}
             showEnhancementDialog={showEnhancementDialog}
             toggleEnhancementDialog={toggleEnhancementDialog}
             setOpenConnection={setOpenConnection}
@@ -670,6 +732,7 @@ const PageLayout: React.FC = () => {
               openTextSchema={openTextSchema}
               openLoadSchema={openLoadSchema}
               openPredefinedSchema={openPredefinedSchema}
+              openDataImporterSchema={openDataImporterSchema}
               showEnhancementDialog={showEnhancementDialog}
               toggleEnhancementDialog={toggleEnhancementDialog}
               setOpenConnection={setOpenConnection}

frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/DataImporter.tsx

@@ -0,0 +1,179 @@
+import { Button, Dialog } from '@neo4j-ndl/react';
+import { useState } from 'react';
+import { OptionType, TupleType } from '../../../../types';
+import { extractOptions, updateSourceTargetTypeOptions } from '../../../../utils/Utils';
+import { useFileContext } from '../../../../context/UsersFiles';
+import ImporterInput from './ImporterInput';
+import SchemaViz from '../../../Graph/SchemaViz';
+import PatternContainer from './PatternContainer';
+import UploadJsonData from './UploadJsonData';
+
+interface DataImporterDialogProps {
+  open: boolean;
+  onClose: () => void;
+  onApply: (
+    patterns: string[],
+    nodeLabels: OptionType[],
+    relationshipLabels: OptionType[],
+    updatedSource: OptionType[],
+    updatedTarget: OptionType[],
+    updatedType: OptionType[]
+  ) => void;
+}
+
+const DataImporterSchemaDialog = ({ open, onClose, onApply }: DataImporterDialogProps) => {
+  const {
+    importerPattern,
+    setImporterPattern,
+    importerNodes,
+    setImporterNodes,
+    importerRels,
+    setImporterRels,
+    sourceOptions,
+    setSourceOptions,
+    targetOptions,
+    setTargetOptions,
+    typeOptions,
+    setTypeOptions,
+  } = useFileContext();
+
+  const [openGraphView, setOpenGraphView] = useState<boolean>(false);
+  const [viewPoint, setViewPoint] = useState<string>('');
+  const handleCancel = () => {
+    onClose();
+    setImporterPattern([]);
+    setImporterNodes([]);
+    setImporterRels([]);
+  };
+
+  const handleImporterCheck = async () => {
+    const [newSourceOptions, newTargetOptions, newTypeOptions] = await updateSourceTargetTypeOptions({
+      patterns: importerPattern.map((label) => ({ label, value: label })),
+      currentSourceOptions: sourceOptions,
+      currentTargetOptions: targetOptions,
+      currentTypeOptions: typeOptions,
+      setSourceOptions,
+      setTargetOptions,
+      setTypeOptions,
+    });
+    onApply(importerPattern, importerNodes, importerRels, newSourceOptions, newTargetOptions, newTypeOptions);
+    onClose();
+  };
+
+  const handleRemovePattern = (patternToRemove: string) => {
+    const updatedPatterns = importerPattern.filter((p) => p !== patternToRemove);
+    if (updatedPatterns.length === 0) {
+      setImporterPattern([]);
+      setImporterNodes([]);
+      setImporterRels([]);
+      return;
+    }
+    const updatedTuples: TupleType[] = updatedPatterns
+      .map((item: string) => {
+        const matchResult = item.match(/^(.+?)-\[:([A-Z_]+)\]->(.+)$/);
+        if (matchResult) {
+          const [source, rel, target] = matchResult.slice(1).map((s) => s.trim());
+          return {
+            value: `${source},${rel},${target}`,
+            label: `${source} -[:${rel}]-> ${target}`,
+            source,
+            target,
+            type: rel,
+          };
+        }
+        return null;
+      })
+      .filter(Boolean) as TupleType[];
+    const { nodeLabelOptions, relationshipTypeOptions } = extractOptions(updatedTuples);
+    setImporterPattern(updatedPatterns);
+    setImporterNodes(nodeLabelOptions);
+    setImporterRels(relationshipTypeOptions);
+  };
+
+  const handleSchemaView = () => {
+    setOpenGraphView(true);
+    setViewPoint('showSchemaView');
+  };
+
+  return (
+    <>
+      <Dialog isOpen={open} onClose={handleCancel}>
+        <Dialog.Header>JSON Data Graph Extraction Settings</Dialog.Header>
+        <Dialog.Content className='n-flex n-flex-col n-gap-token-6 p-6'>
+          <ImporterInput />
+          <UploadJsonData
+            onSchemaExtracted={({ nodeLabels, relationshipTypes, relationshipObjectTypes, nodeObjectTypes }) => {
+              const nodeLabelMap = Object.fromEntries(nodeLabels.map((n) => [n.$id, n.token]));
+              const relTypeMap = Object.fromEntries(relationshipTypes.map((r) => [r.$id, r.token]));
+              const nodeIdToLabel: Record<string, string> = {};
+              nodeObjectTypes.forEach((nodeObj: any) => {
+                const labelRef = nodeObj.labels?.[0]?.$ref;
+                if (labelRef && nodeLabelMap[labelRef.slice(1)]) {
+                  nodeIdToLabel[nodeObj.$id] = nodeLabelMap[labelRef.slice(1)];
+                }
+              });
+
+              const patterns = relationshipObjectTypes.map((relObj) => {
+                const fromId = relObj.from.$ref.slice(1);
+                const toId = relObj.to.$ref.slice(1);
+                const relId = relObj.type.$ref.slice(1);
+                const fromLabel = nodeIdToLabel[fromId] || 'source';
+                const toLabel = nodeIdToLabel[toId] || 'target';
+                const relLabel = relTypeMap[relId] || 'type';
+                const pattern = `${fromLabel} -[:${relLabel}]-> ${toLabel}`;
+                return pattern;
+              });
+
+              const importerTuples = patterns
+                .map((p) => {
+                  const match = p.match(/^(.+?) -\[:(.+?)\]-> (.+)$/);
+                  if (!match) {
+                    return null;
+                  }
+                  const [_, source, type, target] = match;
+                  return {
+                    label: `${source} -[:${type}]-> ${target}`,
+                    value: `${source},${type},${target}`,
+                    source,
+                    target,
+                    type,
+                  };
+                })
+                .filter(Boolean) as TupleType[];
+              const { nodeLabelOptions, relationshipTypeOptions } = extractOptions(importerTuples);
+              setImporterNodes(nodeLabelOptions);
+              setImporterRels(relationshipTypeOptions);
+              setImporterPattern(patterns);
+            }}
+          />
+          <PatternContainer
+            pattern={importerPattern}
+            handleRemove={handleRemovePattern}
+            handleSchemaView={handleSchemaView}
+            nodes={importerNodes}
+            rels={importerRels}
+          />
+          <Dialog.Actions className='n-flex n-justify-end n-gap-token-4 pt-4'>
+            <Button onClick={handleCancel} isDisabled={importerPattern.length === 0}>
+              Cancel
+            </Button>
+            <Button onClick={handleImporterCheck} isDisabled={importerPattern.length === 0}>
+              Apply
+            </Button>
+          </Dialog.Actions>
+        </Dialog.Content>
+      </Dialog>
+      {openGraphView && (
+        <SchemaViz
+          open={openGraphView}
+          setGraphViewOpen={setOpenGraphView}
+          viewPoint={viewPoint}
+          nodeValues={importerNodes ?? []}
+          relationshipValues={importerRels ?? []}
+        />
+      )}
+    </>
+  );
+};
+
+export default DataImporterSchemaDialog;