feat(ingest/vertica): performance improvement and bug fixes (#8328)

Co-authored-by: Harshal Sheth <[email protected]>

Author: vishalkSimplify

datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts

@@ -83,7 +83,7 @@ import {
     PROJECT_NAME,
 } from './lookml';
 import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto';
-import { BIGQUERY_BETA, DBT_CLOUD, MYSQL, POWER_BI, UNITY_CATALOG } from '../constants';
+import { BIGQUERY_BETA, DBT_CLOUD, MYSQL, POWER_BI, UNITY_CATALOG, VERTICA } from '../constants';
 import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta';
 import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql';
 import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql';
@@ -130,6 +130,17 @@ import {
     WORKSPACE_ID_DENY,
 } from './powerbi';
 
+import {
+    VERTICA_HOST_PORT,
+    VERTICA_DATABASE,
+    VERTICA_USERNAME,
+    VERTICA_PASSWORD,
+    INCLUDE_PROJECTIONS,
+    INCLUDE_MLMODELS,
+    INCLUDE_VIEW_LINEAGE,
+    INCLUDE_PROJECTIONS_LINEAGE,
+} from './vertica';
+
 export enum RecipeSections {
     Connection = 0,
     Filter = 1,
@@ -428,6 +439,20 @@ export const RECIPE_FIELDS: RecipeFields = {
         ],
         filterSectionTooltip: 'Include or exclude specific PowerBI Workspaces from ingestion.',
     },
+    [VERTICA]: {
+        fields: [VERTICA_HOST_PORT, VERTICA_DATABASE, VERTICA_USERNAME, VERTICA_PASSWORD],
+        filterFields: [SCHEMA_ALLOW, SCHEMA_DENY, TABLE_ALLOW, TABLE_DENY, VIEW_ALLOW, VIEW_DENY],
+        advancedFields: [
+            INCLUDE_TABLES,
+            INCLUDE_VIEWS,
+            INCLUDE_PROJECTIONS,
+            INCLUDE_MLMODELS,
+            INCLUDE_VIEW_LINEAGE,
+            INCLUDE_PROJECTIONS_LINEAGE,
+            TABLE_PROFILING_ENABLED,
+        ],
+        filterSectionTooltip: 'Include or exclude specific Schemas, Tables, Views and Projections from ingestion.',
+    },
 };
 
 export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));

datahub-web-react/src/app/ingest/source/builder/RecipeForm/vertica.ts

@@ -0,0 +1,119 @@
+import { get } from 'lodash';
+import { RecipeField, FieldType } from './common';
+
+export const VERTICA_HOST_PORT: RecipeField = {
+    name: 'host_port',
+    label: 'Host and Port',
+    tooltip:
+        "The host and port where Vertica is running. For example, 'localhost:5433'. Note: this host must be accessible on the network where DataHub is running (or allowed via an IP Allow List, AWS PrivateLink, etc).",
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.host_port',
+    placeholder: 'localhost:5433',
+    required: true,
+    rules: null,
+};
+
+export const VERTICA_DATABASE: RecipeField = {
+    name: 'database',
+    label: 'Database',
+    tooltip: 'Ingest metadata for a specific Database.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.database',
+    placeholder: 'Vertica_Database',
+    required: true,
+    rules: null,
+};
+
+export const VERTICA_USERNAME: RecipeField = {
+    name: 'username',
+    label: 'Username',
+    tooltip: 'The Vertica username used to extract metadata.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.username',
+    placeholder: 'Vertica_Username',
+    required: true,
+    rules: null,
+};
+
+export const VERTICA_PASSWORD: RecipeField = {
+    name: 'password',
+    label: 'Password',
+    tooltip: 'The Vertica password for the user.',
+    type: FieldType.SECRET,
+    fieldPath: 'source.config.password',
+    placeholder: 'Vertica_Password',
+    required: true,
+    rules: null,
+};
+
+const includeProjectionPath = 'source.config.include_projections';
+export const INCLUDE_PROJECTIONS: RecipeField = {
+    name: 'include_projections',
+    label: 'Include Projections',
+    tooltip: 'Extract Projections from source.',
+    type: FieldType.BOOLEAN,
+    fieldPath: includeProjectionPath,
+    // This is in accordance with what the ingestion sources do.
+    getValueFromRecipeOverride: (recipe: any) => {
+        const includeProjection = get(recipe, includeProjectionPath);
+        if (includeProjection !== undefined && includeProjection !== null) {
+            return includeProjection;
+        }
+        return true;
+    },
+    rules: null,
+};
+
+const includemodelsPath = 'source.config.include_models';
+export const INCLUDE_MLMODELS: RecipeField = {
+    name: 'include_models',
+    label: 'Include ML Models',
+    tooltip: 'Extract ML models from source.',
+    type: FieldType.BOOLEAN,
+    fieldPath: includemodelsPath,
+    // This is in accordance with what the ingestion sources do.
+    getValueFromRecipeOverride: (recipe: any) => {
+        const includeModel = get(recipe, includemodelsPath);
+        if (includeModel !== undefined && includeModel !== null) {
+            return includeModel;
+        }
+        return true;
+    },
+    rules: null,
+};
+
+const includeviewlineagePath = 'source.config.include_view_lineage';
+export const INCLUDE_VIEW_LINEAGE: RecipeField = {
+    name: 'include_view_lineage',
+    label: 'Include View Lineage',
+    tooltip: 'Extract View Lineage from source.',
+    type: FieldType.BOOLEAN,
+    fieldPath: includeviewlineagePath,
+    // This is in accordance with what the ingestion sources do.
+    getValueFromRecipeOverride: (recipe: any) => {
+        const includeviewlineage = get(recipe, includeviewlineagePath);
+        if (includeviewlineage !== undefined && includeviewlineage !== null) {
+            return includeviewlineage;
+        }
+        return true;
+    },
+    rules: null,
+};
+
+const includeprojectionlineagePath = 'source.config.include_projection_lineage';
+export const INCLUDE_PROJECTIONS_LINEAGE: RecipeField = {
+    name: 'include_projection_lineage',
+    label: 'Include Projection Lineage',
+    tooltip: 'Extract Projection Lineage from source.',
+    type: FieldType.BOOLEAN,
+    fieldPath: includeprojectionlineagePath,
+    // This is in accordance with what the ingestion sources do.
+    getValueFromRecipeOverride: (recipe: any) => {
+        const includeprojectionlineage = get(recipe, includeprojectionlineagePath);
+        if (includeprojectionlineage !== undefined && includeprojectionlineage !== null) {
+            return includeprojectionlineage;
+        }
+        return true;
+    },
+    rules: null,
+};

datahub-web-react/src/app/ingest/source/builder/sources.json

@@ -200,7 +200,7 @@
         "name": "vertica",
         "displayName": "Vertica",
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/vertica/",
-        "recipe": "source:\n    type: vertica\n    config:\n        # Coordinates\n        host_port: localhost:5433\n        # The name of the vertica database\n        database: Vmart\n        # Credentials\n        username: dbadmin\n    password:null\n    include_tables: true\n        include_views: true\n    include_projections: true\n    include_oauth: true\n   include_models: true\n        include_view_lineage: true\n      include_projection_lineage: true\n    profiling:\n            enabled: true\n        stateful_ingestion:\n            enabled: true  "
+        "recipe": "source:\n    type: vertica\n    config:\n        # Coordinates\n        host_port: localhost:5433\n        # The name of the vertica database\n        database: Database_Name\n        # Credentials\n        username: Vertica_User\n        password: Vertica_Password\n\n        include_tables: true\n        include_views: true\n        include_projections: true\n        include_models: true\n        include_view_lineage: true\n        include_projection_lineage: true\n        profiling:\n            enabled: false\n        stateful_ingestion:\n            enabled: true  "
     },
     {
         "urn": "urn:li:dataPlatform:custom",

metadata-ingestion/docs/sources/vertica/README.md

@@ -8,7 +8,6 @@ The DataHub Vertica Plugin extracts the following:
 * Metadata for databases, schemas, views, tables, and projections
 * Table level lineage
 * Metadata for ML Models
-* Metadata for Vertica OAuth
 
 
 ### Concept Mapping

metadata-ingestion/docs/sources/vertica/vertica_pre.md

@@ -2,5 +2,5 @@
 
 In order to ingest metadata from Vertica, you will need:
 
-- Vertica Server Version 10.1.1-0 and avobe. It may also work for older versions.
+- Vertica Server Version 10.1.1-0 and above. It may also work with, but is not been tested with, older versions .
 - Vertica Credentials (Username/Password)

metadata-ingestion/docs/sources/vertica/vertica_recipe.yml

@@ -12,7 +12,6 @@ source:
     include_tables: true
     include_views: true
     include_projections: true
-    include_oauth: true
     include_models: true
     include_view_lineage: true
     include_projection_lineage: true

metadata-ingestion/setup.py

@@ -374,7 +374,9 @@ def get_long_description():
     "nifi": {"requests", "packaging", "requests-gssapi"},
     "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"},
     "powerbi-report-server": powerbi_report_server,
-    "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.1"},
+
+    "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8"},
+
     "unity-catalog": databricks | sqllineage_lib,
 }
 
@@ -488,7 +490,8 @@ def get_long_description():
             "powerbi-report-server",
             "salesforce",
             "unity-catalog",
-            "nifi"
+            "nifi",
+            "vertica"
             # airflow is added below
         ]
         if plugin
@@ -522,7 +525,7 @@ def get_long_description():
             "mysql",
             "mariadb",
             "redash",
-            # "vertica",
+            "vertica",
         ]
         for dependency in plugins[plugin]
     ),