Rdf ingestion mvp

datahub-web-react/src/app/ingestV2/source/builder/RecipeForm/constants.ts

@@ -225,6 +225,14 @@ import {
     PRESTO_PASSWORD,
     PRESTO_USERNAME,
 } from '@app/ingestV2/source/builder/RecipeForm/presto';
+import {
+    RDF_DIALECT,
+    RDF_ENVIRONMENT,
+    RDF_EXTENSIONS,
+    RDF_FORMAT,
+    RDF_RECURSIVE,
+    RDF_SOURCE,
+} from '@app/ingestV2/source/builder/RecipeForm/rdf';
 import {
     REDSHIFT_DATABASE,
     REDSHIFT_HOST_PORT,
@@ -298,6 +306,7 @@ import {
     NOTION,
     OKTA,
     POWER_BI,
+    RDF,
     SAC,
     VERTICA,
 } from '@app/ingestV2/source/builder/constants';
@@ -751,6 +760,13 @@ export const RECIPE_FIELDS: RecipeFields = {
         ],
         hasDynamicFields: true,
     },
+    [RDF]: {
+        fields: [RDF_SOURCE],
+        filterFields: [],
+        advancedFields: [RDF_FORMAT, RDF_EXTENSIONS, RDF_RECURSIVE, RDF_ENVIRONMENT, RDF_DIALECT],
+        connectionSectionTooltip: 'Configure the RDF source location and basic settings.',
+        advancedSectionTooltip: 'Advanced options for RDF format, file processing, and dialect selection.',
+    },
 };
 
 const ALL_CONNECTORS_WITH_FORM = Object.keys(RECIPE_FIELDS);

datahub-web-react/src/app/ingestV2/source/builder/RecipeForm/rdf.ts

@@ -0,0 +1,89 @@
+import { FieldType, RecipeField } from '@app/ingestV2/source/builder/RecipeForm/common';
+
+export const RDF_SOURCE: RecipeField = {
+    name: 'source',
+    label: 'Source',
+    helper: 'RDF source: file, folder, URL, or comma-separated files',
+    tooltip:
+        'Source to process: file path, folder path, server URL, or comma-separated files. Examples: /path/to/file.ttl, /path/to/folder, https://example.com/data.ttl, file1.ttl,file2.ttl',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.source',
+    placeholder: '/path/to/file.ttl or /path/to/folder or https://example.com/data.ttl',
+    required: true,
+    rules: null,
+};
+
+export const RDF_FORMAT: RecipeField = {
+    name: 'format',
+    label: 'RDF Format',
+    helper: 'RDF format (auto-detected if not specified)',
+    tooltip: 'RDF format (auto-detected if not specified). Examples: turtle, xml, n3, nt, json-ld',
+    type: FieldType.SELECT,
+    fieldPath: 'source.config.format',
+    placeholder: 'Auto-detect',
+    options: [
+        { label: 'Auto-detect', value: '' },
+        { label: 'Turtle', value: 'turtle' },
+        { label: 'RDF/XML', value: 'xml' },
+        { label: 'N3', value: 'n3' },
+        { label: 'N-Triples', value: 'nt' },
+        { label: 'JSON-LD', value: 'json-ld' },
+    ],
+    rules: null,
+};
+
+export const RDF_EXTENSIONS: RecipeField = {
+    name: 'extensions',
+    label: 'File Extensions',
+    helper: 'File extensions to process when source is a folder',
+    tooltip: 'File extensions to process when source is a folder. Default: .ttl, .rdf, .owl, .n3, .nt',
+    type: FieldType.LIST,
+    fieldPath: 'source.config.extensions',
+    placeholder: '.ttl',
+    buttonLabel: 'Add extension',
+    rules: null,
+};
+
+export const RDF_RECURSIVE: RecipeField = {
+    name: 'recursive',
+    label: 'Recursive Folder Processing',
+    helper: 'Enable recursive folder processing when source is a folder',
+    tooltip: 'Enable recursive folder processing when source is a folder (default: true)',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.recursive',
+    rules: null,
+};
+
+export const RDF_ENVIRONMENT: RecipeField = {
+    name: 'environment',
+    label: 'DataHub Environment',
+    helper: 'DataHub environment (PROD, DEV, TEST, etc.)',
+    tooltip: 'DataHub environment (PROD, DEV, TEST, etc.)',
+    type: FieldType.SELECT,
+    fieldPath: 'source.config.environment',
+    placeholder: 'PROD',
+    options: [
+        { label: 'PROD', value: 'PROD' },
+        { label: 'DEV', value: 'DEV' },
+        { label: 'TEST', value: 'TEST' },
+        { label: 'UAT', value: 'UAT' },
+    ],
+    rules: null,
+};
+
+export const RDF_DIALECT: RecipeField = {
+    name: 'dialect',
+    label: 'RDF Dialect',
+    helper: 'Force a specific RDF dialect (default: auto-detect)',
+    tooltip: 'Force a specific RDF dialect (default: auto-detect). Options: default, fibo, generic',
+    type: FieldType.SELECT,
+    fieldPath: 'source.config.dialect',
+    placeholder: 'Auto-detect',
+    options: [
+        { label: 'Auto-detect', value: '' },
+        { label: 'Default', value: 'default' },
+        { label: 'FIBO', value: 'fibo' },
+        { label: 'Generic', value: 'generic' },
+    ],
+    rules: null,
+};

datahub-web-react/src/app/ingestV2/source/builder/constants.ts

@@ -187,6 +187,8 @@ export const SNAPLOGIC = 'snaplogic';
 export const SNAPLOGIC_URN = `urn:li:dataPlatform:${SNAPLOGIC}`;
 export const FABRIC_ONELAKE = 'fabric-onelake';
 export const FABRIC_ONELAKE_URN = `urn:li:dataPlatform:${FABRIC_ONELAKE}`;
+export const RDF = 'rdf';
+export const RDF_URN = `urn:li:dataPlatform:${RDF}`;
 
 export const PLATFORM_URN_TO_LOGO = {
     [AIRFLOW_URN]: airflowLogo,

datahub-web-react/src/app/ingestV2/source/builder/sources.json

@@ -552,5 +552,13 @@
         "recipe": "source:\n  type: feast\n  config:\n    # Coordinates\n    path: \"/path/to/repository/\"\n    # Options\n    environment: \"PROD\"",
         "category": "ML Platforms",
         "isPopular": false
+    },
+    {
+        "urn": "urn:li:dataPlatform:rdf",
+        "name": "rdf",
+        "displayName": "RDF",
+        "description": "Import glossary terms, term groups, and relationships from RDF/OWL ontologies (SKOS, Turtle, RDF/XML).",
+        "docsUrl": "https://docs.datahub.com/docs/generated/ingestion/sources/rdf",
+        "recipe": "source:\n  type: rdf\n  config:\n    source: path/to/glossary.ttl\n    environment: PROD\n    export_only:\n      - glossary"
     }
 ]

metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py

@@ -993,6 +993,46 @@ def test_airflow_plugin(
         # Verify that no MCPs were generated.
         assert not os.path.exists(airflow_instance.metadata_file)
     else:
+        # Check if metadata file exists before trying to read it
+        if not os.path.exists(airflow_instance.metadata_file):
+            # Try to get more diagnostic information
+            print(
+                f"ERROR: Metadata file does not exist: {airflow_instance.metadata_file}"
+            )
+            print("Checking if DAG completed successfully...")
+            try:
+                api_version = get_api_version()
+                res = _make_api_request(
+                    airflow_instance.session,
+                    f"{airflow_instance.airflow_url}/api/{api_version}/dags/{dag_id}/dagRuns",
+                )
+                dag_runs = res.json()["dag_runs"]
+                if dag_runs:
+                    dag_run = dag_runs[0]
+                    print(f"DAG run state: {dag_run['state']}")
+                    print(f"DAG run ID: {dag_run['dag_run_id']}")
+            except Exception as e:
+                print(f"Failed to get DAG run info: {e}")
+
+            # Check if the connection was configured correctly
+            print(f"Expected metadata file path: {airflow_instance.metadata_file}")
+            print(
+                f"Metadata file parent directory exists: {airflow_instance.metadata_file.parent.exists()}"
+            )
+            print(
+                f"Metadata file parent directory: {airflow_instance.metadata_file.parent}"
+            )
+            if airflow_instance.metadata_file.parent.exists():
+                print(
+                    f"Files in metadata directory: {list(airflow_instance.metadata_file.parent.iterdir())}"
+                )
+
+            raise FileNotFoundError(
+                f"Metadata file not found: {airflow_instance.metadata_file}. "
+                f"This usually means the DataHub plugin did not emit any metadata. "
+                f"Check Airflow logs for plugin errors."
+            )
+
         _sanitize_output_file(airflow_instance.metadata_file)
 
         check_golden_file(