Bump framework to reduce Azure Storage list call count (#129)

Author: george-zubrienko

.github/workflows/release.yaml

@@ -16,4 +16,4 @@ jobs:
         uses: SneaksAndData/github-actions/semver_release@v0.1.9
         with:
           major_v: 1
-          minor_v: 0
+          minor_v: 1

build.sbt

@@ -25,7 +25,7 @@ lazy val plugin = (project in file("."))
     name := "arcane-stream-microsoft-synapse-link",
     idePackagePrefix := Some("com.sneaksanddata.arcane.microsoft_synapse_link"),
 
-    libraryDependencies += "com.sneaksanddata" % "arcane-framework_3" % "0.7.5",
+    libraryDependencies += "com.sneaksanddata" % "arcane-framework_3" % "0.8.0",
     libraryDependencies += "io.netty" % "netty-tcnative-boringssl-static" % "2.0.65.Final",
 
     // bugfix for upgrade header

integration-tests.env

@@ -1,5 +1,5 @@
 STREAMCONTEXT__BACKFILL=false
-STREAMCONTEXT__SPEC='{ "backfillJobTemplateRef": { "apiGroup": "streaming.sneaksanddata.com", "kind": "StreamingJobTemplate", "name": "arcane-stream-microsoft-synapse-link-large-job" }, "groupingIntervalSeconds": 1, "groupsPerFile": 1, "httpClientMaxRetries": 3, "httpClientRetryDelaySeconds": 1, "jobTemplateRef": { "apiGroup": "streaming.sneaksanddata.com", "kind": "StreamingJobTemplate", "name": "arcane-stream-microsoft-synapse-link-standard-job" }, "lookBackInterval": 21000, "tableProperties": { "partitionExpressions": ["bucket(recid, 3)"], "format": "PARQUET", "sortedBy": [], "parquetBloomFilterColumns": [] }, "rowsPerGroup": 10000, "sinkSettings": { "archiveTableName": "iceberg.test.archive_test", "optimizeSettings": { "batchThreshold": 60, "fileSizeThreshold": "512MB" }, "orphanFilesExpirationSettings": { "batchThreshold": 60, "retentionThreshold": "6h" }, "snapshotExpirationSettings": { "batchThreshold": 60, "retentionThreshold": "6h" }, "targetTableName": "iceberg.test.test" }, "sourceSettings": { "baseLocation": "abfss://cdm-e2e@devstoreaccount1.dfs.core.windows.net/", "changeCaptureIntervalSeconds": 5, "changeCapturePeriodSeconds": 60, "name": "synapsetable" }, "stagingDataSettings": { "catalog": { "catalogName": "iceberg", "catalogUri": "http://localhost:20001/catalog", "namespace": "test", "schemaName": "test", "warehouse": "demo" }, "dataLocation": "s3://tmp/initial-warehouse/test", "tableNamePrefix": "staging_inventtrans" }, "fieldSelectionRule": {"ruleType": "all", "fields": []}, "backfillBehavior": "overwrite", "backfillStartDate": "2025-03-04T07.00.00Z" }'
+STREAMCONTEXT__SPEC='{ "backfillJobTemplateRef": { "apiGroup": "streaming.sneaksanddata.com", "kind": "StreamingJobTemplate", "name": "arcane-stream-microsoft-synapse-link-large-job" }, "groupingIntervalSeconds": 1, "groupsPerFile": 1, "httpClientMaxRetries": 3, "httpClientRetryDelaySeconds": 1, "jobTemplateRef": { "apiGroup": "streaming.sneaksanddata.com", "kind": "StreamingJobTemplate", "name": "arcane-stream-microsoft-synapse-link-standard-job" }, "lookBackInterval": 21000, "tableProperties": { "partitionExpressions": ["bucket(recid, 3)"], "format": "PARQUET", "sortedBy": [], "parquetBloomFilterColumns": [] }, "rowsPerGroup": 10000, "sinkSettings": { "archiveTableName": "iceberg.test.archive_test", "optimizeSettings": { "batchThreshold": 60, "fileSizeThreshold": "512MB" }, "orphanFilesExpirationSettings": { "batchThreshold": 60, "retentionThreshold": "6h" }, "snapshotExpirationSettings": { "batchThreshold": 60, "retentionThreshold": "6h" }, "targetTableName": "iceberg.test.test" }, "sourceSettings": { "baseLocation": "abfss://cdm-e2e@devstoreaccount1.dfs.core.windows.net/", "changeCaptureIntervalSeconds": 5, "changeCapturePeriodSeconds": 60, "name": "synapsetable" }, "stagingDataSettings": { "catalog": { "catalogName": "iceberg", "catalogUri": "http://localhost:20001/catalog", "namespace": "test", "schemaName": "test", "warehouse": "demo" }, "tableNamePrefix": "staging_inventtrans", "maxRowsPerFile": 10000 }, "fieldSelectionRule": {"ruleType": "all", "fields": []}, "backfillBehavior": "overwrite", "backfillStartDate": "2025-03-04T07.00.00Z" }'
 STREAMCONTEXT__STREAM_ID=test
 STREAMCONTEXT__STREAM_KIND=CdmChangeFeed
 APPLICATION_VERSION=0.0.1

src/main/scala/models/app/MicrosoftSynapseLinkStreamContext.scala

@@ -1,13 +1,11 @@
 package com.sneaksanddata.arcane.microsoft_synapse_link
 package models.app
 
-import models.app.contracts.{OptimizeSettingsSpec, SnapshotExpirationSettingsSpec, StreamSpec, given_Conversion_TablePropertiesSettings_TableProperties}
+import models.app.contracts.StreamSpec
 
 import com.sneaksanddata.arcane.framework.models.app.StreamContext
 import com.sneaksanddata.arcane.framework.models.settings
-import com.sneaksanddata.arcane.framework.models.settings.TableFormat.PARQUET
 import com.sneaksanddata.arcane.framework.models.settings.{BackfillBehavior, BackfillSettings, FieldSelectionRule, FieldSelectionRuleSettings, GroupingSettings, OptimizeSettings, OrphanFilesExpirationSettings, SnapshotExpirationSettings, StagingDataSettings, SynapseSourceSettings, TableFormat, TableMaintenanceSettings, TablePropertiesSettings, TargetTableSettings, VersionedDataGraphBuilderSettings}
-import com.sneaksanddata.arcane.framework.services.base.MergeServiceClient
 import com.sneaksanddata.arcane.framework.services.lakehouse.IcebergCatalogCredential
 import com.sneaksanddata.arcane.framework.services.lakehouse.base.{IcebergCatalogSettings, S3CatalogFileIO}
 import com.sneaksanddata.arcane.framework.services.merging.JdbcMergeServiceClientOptions