Skip to content

Commit 606bcd0

Browse files
Fix backfill table name for backfill overwrite (#327)
1 parent 76f727c commit 606bcd0

File tree

2 files changed

+6
-5
lines changed

2 files changed

+6
-5
lines changed

src/main/scala/services/bootstrap/DefaultStreamBootstrapper.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class DefaultStreamBootstrapper(
3939
_ <- zlog("Creating backfill table %s", stagingSettings.table.backfillTableName)
4040
_ <- stagingEntityManager.createTable(
4141
CreateTableRequest(
42-
name = stagingSettings.table.backfillTableName,
42+
name = stagingSettings.table.backfillTableName.parts.name,
4343
schema = schema,
4444
replace = false
4545
)

src/main/scala/services/streaming/processors/batch_processors/streaming/MergeBatchProcessor.scala

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import models.app.PluginStreamContext
66
import models.batches.{MergeableBatch, StagedVersionedBatch}
77
import models.settings.*
88
import models.settings.TableNaming.*
9+
import models.settings.backfill.BackfillBehavior.Overwrite
910
import models.settings.sink.SinkSettings
1011
import services.base.MergeServiceClient
1112
import services.iceberg.base.*
@@ -30,7 +31,7 @@ class MergeBatchProcessor(
3031
targetTableSettings: SinkSettings,
3132
declaredMetrics: DeclaredMetrics,
3233
schemaMigrationEnabled: Boolean,
33-
isBackfilling: Boolean
34+
isTargetInStaging: Boolean
3435
) extends StagedBatchProcessor:
3536

3637
private def alignSchemas(
@@ -59,9 +60,9 @@ class MergeBatchProcessor(
5960
ZIO.when(!batch.isEmpty && schemaMigrationEnabled) {
6061
for
6162
// for streams, we migrate sink table
62-
_ <- ZIO.unless(isBackfilling)(alignSchemas(batch, sinkPropertyManager, sinkEntityManager))
63+
_ <- ZIO.unless(isTargetInStaging)(alignSchemas(batch, sinkPropertyManager, sinkEntityManager))
6364
// for backfills, we migrate staging table
64-
_ <- ZIO.when(isBackfilling)(alignSchemas(batch, stagingPropertyManager, stagingEntityManager))
65+
_ <- ZIO.when(isTargetInStaging)(alignSchemas(batch, stagingPropertyManager, stagingEntityManager))
6566
yield ()
6667
}
6768
)
@@ -157,6 +158,6 @@ object MergeBatchProcessor:
157158
context.sink,
158159
declaredMetrics,
159160
!context.staging.table.isUnifiedSchema,
160-
context.isBackfilling
161+
context.isBackfilling && context.streamMode.backfill.backfillBehavior == Overwrite
161162
)
162163
}

0 commit comments

Comments
 (0)