Skip to content

Commit b3e6f5d

Browse files
committed
Fix lineage in BQ plugins
1 parent 5e99a46 commit b3e6f5d

File tree

2 files changed

+6
-2
lines changed

2 files changed

+6
-2
lines changed

src/main/java/co/cask/gcp/bigquery/BigQuerySink.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,10 @@ public void prepareRun(BatchSinkContext context) throws Exception {
121121
BigQueryFileFormat.NEWLINE_DELIMITED_JSON,
122122
TextOutputFormat.class);
123123

124-
setOutputFormat(context);
124+
// Both emitLineage and setOutputFormat internally try to create an external dataset if it does not already exists.
125+
// We call emitLineage before since it creates the dataset with schema which .
125126
emitLineage(context, fields);
127+
setOutputFormat(context);
126128
}
127129

128130
@Override

src/main/java/co/cask/gcp/bigquery/BigQuerySource.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,10 @@ public void prepareRun(BatchSourceContext context) throws Exception {
112112
job.setOutputKeyClass(LongWritable.class);
113113
job.setOutputKeyClass(Text.class);
114114

115-
setInputFormat(context);
115+
// Both emitLineage and setOutputFormat internally try to create an external dataset if it does not already exists.
116+
// We call emitLineage before since it creates the dataset with schema which .
116117
emitLineage(context);
118+
setInputFormat(context);
117119
}
118120

119121
@Override

0 commit comments

Comments
 (0)