Skip to content

Commit 53524f8

Browse files
committed
Adding SparkSQLProperty for split-size
1 parent 1753a6d commit 53524f8

File tree

4 files changed

+6
-0
lines changed

4 files changed

+6
-0
lines changed

spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ public long splitSize() {
191191
return confParser
192192
.longConf()
193193
.option(SparkReadOptions.SPLIT_SIZE)
194+
.sessionConf(SparkSQLProperties.SPLIT_SIZE)
194195
.tableProperty(TableProperties.SPLIT_SIZE)
195196
.defaultValue(TableProperties.SPLIT_SIZE_DEFAULT)
196197
.parse();

spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,7 @@ private SparkSQLProperties() {}
6666

6767
// Controls whether to report locality information to Spark while allocating input partitions
6868
public static final String LOCALITY = "spark.sql.iceberg.locality.enabled";
69+
70+
// Controls the spark input split size.
71+
public static final String SPLIT_SIZE = "spark.sql.iceberg.split-size";
6972
}

spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@ public long splitSize() {
201201
return confParser
202202
.longConf()
203203
.option(SparkReadOptions.SPLIT_SIZE)
204+
.sessionConf(SparkSQLProperties.SPLIT_SIZE)
204205
.tableProperty(TableProperties.SPLIT_SIZE)
205206
.defaultValue(TableProperties.SPLIT_SIZE_DEFAULT)
206207
.parse();

spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ public long splitSize() {
199199
return confParser
200200
.longConf()
201201
.option(SparkReadOptions.SPLIT_SIZE)
202+
.sessionConf(SparkSQLProperties.SPLIT_SIZE)
202203
.tableProperty(TableProperties.SPLIT_SIZE)
203204
.defaultValue(TableProperties.SPLIT_SIZE_DEFAULT)
204205
.parse();

0 commit comments

Comments
 (0)