fix(ui/ingestion): bug fixes in the new ingestion flow (#15786)

Co-authored-by: Chris Collins <chriscollins3456@gmail.com>

Author: purnimagarg1

datahub-web-react/src/app/ingestV2/source/builder/RecipeForm/TestConnection/TestConnectionButton.tsx

@@ -18,16 +18,18 @@ import {
 } from '@graphql/ingestion.generated';
 import { ExecutionRequestResult, IngestionSource } from '@types';
 
-export function getRecipeJson(recipeYaml: string) {
+export function getRecipeJson(recipeYaml: string, hideWarnings?: boolean) {
     // Convert the recipe into it's json representation, and catch + report exceptions while we do it.
     let recipeJson;
     try {
         recipeJson = yamlToJson(recipeYaml);
     } catch (e) {
-        const messageText = (e as any).parsedLine
-            ? `Please fix line ${(e as any).parsedLine} in your recipe.`
-            : 'Please check your recipe configuration.';
-        message.warn(`Found invalid YAML. ${messageText}`);
+        if (!hideWarnings) {
+            const messageText = (e as any).parsedLine
+                ? `Please fix line ${(e as any).parsedLine} in your recipe.`
+                : 'Please check your recipe configuration.';
+            message.warn(`Found invalid YAML. ${messageText}`);
+        }
         return null;
     }
     return recipeJson;

datahub-web-react/src/app/ingestV2/source/builder/sources.json

@@ -315,7 +315,7 @@
         "displayName": "Vertica",
         "description": "Import Databases, Schemas, Tables, Views, Projections, statistics, and lineage from Vertica.",
         "docsUrl": "https://docs.datahub.com/docs/generated/ingestion/sources/vertica/",
-        "recipe": "source:\n    type: vertica\n    config:\n        # Coordinates\n        host_port: localhost:5433\n        # The name of the vertica database\n        database: Database_Name\n        # Credentials\n        username: Vertica_User\n        password: Vertica_Password\n\n        include_tables: true\n        include_views: true\n        include_projections: true\n        include_models: true\n        include_view_lineage: true\n        include_projection_lineage: true\n        profiling:\n            enabled: false\n        stateful_ingestion:\n            enabled: true  ",
+        "recipe": "source:\n    type: vertica\n    config:\n        # Coordinates\n        host_port: # localhost:5433\n        # The name of the vertica database\n        database: # Database_Name\n        # Credentials\n        username: # Vertica_User\n        password: # Vertica_Password\n\n        include_tables: true\n        include_views: true\n        include_projections: true\n        include_models: true\n        include_view_lineage: true\n        include_projection_lineage: true\n        profiling:\n            enabled: false\n        stateful_ingestion:\n            enabled: true  ",
         "category": "Data Warehouse",
         "isPopular": false
     },

datahub-web-react/src/app/ingestV2/source/multiStepBuilder/IngestionSourceUpdatePage.tsx

@@ -150,8 +150,10 @@ export function IngestionSourceUpdatePage() {
             sourceType: ingestionSourceData?.ingestionSource?.type,
             exitType: 'cancel',
         });
-        history.push(ingestionSourcesListBackUrl ?? PageRoutes.INGESTION);
-    }, [history, ingestionSourceData?.ingestionSource?.type, ingestionSourcesListBackUrl]);
+        history.push(ingestionSourcesListBackUrl ?? PageRoutes.INGESTION, {
+            createdOrUpdatedSourceUrn: urn,
+        });
+    }, [history, ingestionSourceData?.ingestionSource?.type, ingestionSourcesListBackUrl, urn]);
 
     const isDirtyChecker = useCallback(
         (

datahub-web-react/src/app/ingestV2/source/multiStepBuilder/steps/step2ConnectionDetails/ConnectionDetailsStep.tsx

@@ -9,6 +9,7 @@ import { LookerWarning } from '@app/ingestV2/source/builder/LookerWarning';
 import { getRecipeJson } from '@app/ingestV2/source/builder/RecipeForm/TestConnection/TestConnectionButton';
 import { CSV, LOOKER, LOOK_ML } from '@app/ingestV2/source/builder/constants';
 import { useIngestionSources } from '@app/ingestV2/source/builder/useIngestionSources';
+import { INGESTION_TYPE_ERROR } from '@app/ingestV2/source/multiStepBuilder/steps/step2ConnectionDetails/constants';
 import { AdvancedSection } from '@app/ingestV2/source/multiStepBuilder/steps/step2ConnectionDetails/sections/AdvansedSection';
 import { NameAndOwnersSection } from '@app/ingestV2/source/multiStepBuilder/steps/step2ConnectionDetails/sections/NameAndOwnersSection';
 import { RecipeSection } from '@app/ingestV2/source/multiStepBuilder/steps/step2ConnectionDetails/sections/recipeSection/RecipeSection';
@@ -41,12 +42,14 @@ export function ConnectionDetailsStep() {
     const [stagedRecipeYml, setStagedRecipeYml] = useState(initialRecipeYml || placeholderRecipe);
 
     const updateRecipe = useCallback(
-        (recipe: string, shouldSetIsRecipeValid?: boolean) => {
-            const recipeJson = getRecipeJson(recipe);
-            if (!recipeJson) return;
+        (recipe: string, shouldSetIsRecipeValid?: boolean, hideYamlWarnings = false) => {
+            const recipeJson = getRecipeJson(recipe, hideYamlWarnings);
+            if (!recipeJson) {
+                throw Error('Invalid YAML');
+            }
 
             if (!JSON.parse(recipeJson).source?.type) {
-                throw Error('Ingestion type is undefined');
+                throw Error(INGESTION_TYPE_ERROR);
             }
 
             const newState = {
@@ -67,7 +70,7 @@ export function ConnectionDetailsStep() {
         (recipe: string) => {
             setStagedRecipeYml(recipe);
             try {
-                updateRecipe(recipe);
+                updateRecipe(recipe, false, true);
             } catch (e: unknown) {
                 if (e instanceof Error) {
                     console.error(e.message);
@@ -120,11 +123,14 @@ export function ConnectionDetailsStep() {
             updateRecipe(stagedRecipeYml, true);
         } catch (e: unknown) {
             if (e instanceof Error) {
-                message.warning({
-                    content: `Please add valid ingestion type`,
-                    duration: 3,
-                });
+                if (e.message === INGESTION_TYPE_ERROR) {
+                    message.warning({
+                        content: `Please add valid ingestion type`,
+                        duration: 3,
+                    });
+                }
             }
+            throw e;
         }
     }, [stagedRecipeYml, updateRecipe]);
 

datahub-web-react/src/app/ingestV2/source/multiStepBuilder/steps/step2ConnectionDetails/constants.ts

@@ -1 +1,3 @@
 export const MAX_FORM_WIDTH = '524px';
+
+export const INGESTION_TYPE_ERROR = 'INGESTION_TYPE_ERROR';

datahub-web-react/src/app/ingestV2/source/multiStepBuilder/steps/step2ConnectionDetails/sections/recipeSection/recipeForm/fields/SelectField.tsx

@@ -9,11 +9,17 @@ export function SelectField({ field, updateFormValue }: CommonFieldProps) {
     const form = Form.useFormInstance();
     const value = Form.useWatch([field.name], form);
 
+    const handleUpdate = (values?: string[]) => {
+        const selectedValue = values?.[0];
+        form.setFieldsValue({ [field.name]: selectedValue });
+        updateFormValue(field.name, selectedValue);
+    };
+
     return (
         <RecipeFormItem recipeField={field} showHelperText>
             <SimpleSelect
                 values={value ? [value] : []}
-                onUpdate={(values) => updateFormValue(field.name, values?.[0])}
+                onUpdate={handleUpdate}
                 placeholder={field.placeholder}
                 options={field.options ?? []}
                 showClear={!field.required}