From 7fef23fc10d2135dfa91af766ceb11fec105d140 Mon Sep 17 00:00:00 2001 From: jkukreja Date: Mon, 6 Jul 2026 00:41:55 -0400 Subject: [PATCH 1/3] [docs] Add documentation for Flink lineage support --- docs/docs/flink/lineage.md | 32 ++++++++++++++++++++++++++++++++ docs/sidebars.js | 3 ++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 docs/docs/flink/lineage.md diff --git a/docs/docs/flink/lineage.md b/docs/docs/flink/lineage.md new file mode 100644 index 000000000000..2e2c51de90d7 --- /dev/null +++ b/docs/docs/flink/lineage.md @@ -0,0 +1,32 @@ +--- +title: "Data Lineage" +sidebar_position: 13 +--- + +# Data Lineage + +Paimon implements [FLIP-314](https://cwiki.apache.org/confluence/spaces/FLINK/pages/240884116/FLIP-294+Support+Customized+Catalog+Modification+Listener) to expose lineage information through Flink's native `LineageVertexProvider` interface. This allows lineage consumers (such as [OpenLineage](https://openlineage.io/docs/integrations/flink/flink2)) to automatically discover Paimon datasets in the Flink job graph. + +## Table API / SQL + +When using Paimon tables via Flink SQL or Table API, lineage is automatically reported for both source and sink tables. + +Apart from the table lineage information provided by Flink natively (name, schema), Paimon enriches each dataset with a namespace and a **config facet** containing core table options and Iceberg metadata options. + +**Note:** For Flink's planner to extract Paimon's lineage datasets from `DataStreamScanProvider` and `DataStreamSinkProvider`, [this upstream Flink change](https://github.com/apache/flink/pull/27727) is pending merge. Without it, Table API lineage events may have missing Paimon table datasets. + +## DataStream API + +Paimon also supports lineage for the DataStream API. Lineage is automatically reported for sources built via `FlinkSourceBuilder` and for all sinks that extend `FlinkSink`. + +Similar to the Table API, it reports all lineage information including name, namespace, schema facet, and config facet. + +## Configuration + +To consume lineage events, a [JobStatusChangedListener](https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/advanced/job_status_listener/#configuration) must be added in your Flink configuration. + +For example, to configure with OpenLineage: + +``` +execution.job-status-changed-listeners = io.openlineage.flink.listener.OpenLineageJobStatusChangedListenerFactory +``` diff --git a/docs/sidebars.js b/docs/sidebars.js index 62a0f3ec039c..4223c1347faf 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -151,7 +151,8 @@ const sidebars = { "flink/default-value", "flink/procedures", "flink/action-jars", - "flink/savepoint" + "flink/savepoint", + "flink/lineage" ] }, { From 769ad8c100bdd1eb949ddcfa8a3b9a3802c63ad2 Mon Sep 17 00:00:00 2001 From: jkukreja Date: Mon, 6 Jul 2026 10:01:53 -0400 Subject: [PATCH 2/3] docs: fix broken link + move the flink pending merge note to warning --- docs/docs/flink/lineage.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/docs/flink/lineage.md b/docs/docs/flink/lineage.md index 2e2c51de90d7..0893cc441c61 100644 --- a/docs/docs/flink/lineage.md +++ b/docs/docs/flink/lineage.md @@ -5,7 +5,7 @@ sidebar_position: 13 # Data Lineage -Paimon implements [FLIP-314](https://cwiki.apache.org/confluence/spaces/FLINK/pages/240884116/FLIP-294+Support+Customized+Catalog+Modification+Listener) to expose lineage information through Flink's native `LineageVertexProvider` interface. This allows lineage consumers (such as [OpenLineage](https://openlineage.io/docs/integrations/flink/flink2)) to automatically discover Paimon datasets in the Flink job graph. +Paimon implements [FLIP-314](https://cwiki.apache.org/confluence/spaces/FLINK/pages/255070913/FLIP-314+Support+Customized+Job+Lineage+Listener) to expose lineage information through Flink's native `LineageVertexProvider` interface. This allows lineage consumers (such as [OpenLineage](https://openlineage.io/docs/integrations/flink/flink2)) to automatically discover Paimon datasets in the Flink job graph. ## Table API / SQL @@ -13,7 +13,9 @@ When using Paimon tables via Flink SQL or Table API, lineage is automatically re Apart from the table lineage information provided by Flink natively (name, schema), Paimon enriches each dataset with a namespace and a **config facet** containing core table options and Iceberg metadata options. -**Note:** For Flink's planner to extract Paimon's lineage datasets from `DataStreamScanProvider` and `DataStreamSinkProvider`, [this upstream Flink change](https://github.com/apache/flink/pull/27727) is pending merge. Without it, Table API lineage events may have missing Paimon table datasets. +:::warning +Full Table API lineage support requires this upstream Flink change which is pending merge. Without it, Flink will not call `getLineageVertex()` for `DataStreamScanProvider` and `DataStreamSinkProvider`, which can lead to potentially missing Paimon input or output datasets. This applies to all Flink 2.x versions that do not yet include this change. +::: ## DataStream API From db07b48060d50f9e1cff378ac9c989f547f610c3 Mon Sep 17 00:00:00 2001 From: jkukreja Date: Tue, 7 Jul 2026 19:46:14 -0400 Subject: [PATCH 3/3] add text that lineage only works for flink 2.x --- docs/docs/flink/lineage.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/docs/flink/lineage.md b/docs/docs/flink/lineage.md index 0893cc441c61..8c4c99008f85 100644 --- a/docs/docs/flink/lineage.md +++ b/docs/docs/flink/lineage.md @@ -5,7 +5,11 @@ sidebar_position: 13 # Data Lineage -Paimon implements [FLIP-314](https://cwiki.apache.org/confluence/spaces/FLINK/pages/255070913/FLIP-314+Support+Customized+Job+Lineage+Listener) to expose lineage information through Flink's native `LineageVertexProvider` interface. This allows lineage consumers (such as [OpenLineage](https://openlineage.io/docs/integrations/flink/flink2)) to automatically discover Paimon datasets in the Flink job graph. +Paimon implements [FLIP-314](https://cwiki.apache.org/confluence/spaces/FLINK/pages/255070913/FLIP-314+Support+Customized+Job+Lineage+Listener) to expose lineage information through Flink's native [LineageVertexProvider](https://nightlies.apache.org/flink/flink-docs-stable/docs/internals/data_lineage/) interface. This allows lineage consumers (such as [OpenLineage](https://openlineage.io/docs/integrations/flink/flink2)) to automatically discover Paimon datasets in the Flink job graph. + +:::info +Data Lineage support is available only for Flink 2.0 and above. +::: ## Table API / SQL