Skip to content

Commit b0983a1

Browse files
[GLUTEN-11341][VL] Add iceberg shim to fix deprecated method
1 parent b6edc89 commit b0983a1

File tree

5 files changed

+80
-3
lines changed

5 files changed

+80
-3
lines changed
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.gluten;
18+
19+
import org.apache.iceberg.ContentFile;
20+
21+
public class ContentFileUtil {
22+
public static String getFilePath(ContentFile<?> file) {
23+
return file.location();
24+
}
25+
}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.gluten;
18+
19+
import org.apache.iceberg.ContentFile;
20+
21+
public class ContentFileUtil {
22+
public static String getFilePath(ContentFile<?> file) {
23+
return file.path().toString;
24+
}
25+
}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.gluten;
18+
19+
import org.apache.iceberg.ContentFile;
20+
21+
public class ContentFileUtil {
22+
public static String getFilePath(ContentFile<?> file) {
23+
return file.path().toString;
24+
}
25+
}

gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717
package org.apache.iceberg.spark.source
1818

19+
import org.apache.gluten.ContentFileUtil
1920
import org.apache.gluten.backendsapi.BackendsApiManager
2021
import org.apache.gluten.exception.GlutenNotSupportException
2122
import org.apache.gluten.execution.SparkDataSourceRDDPartition
@@ -33,7 +34,6 @@ import org.apache.iceberg.spark.SparkSchemaUtil
3334
import java.lang.{Class, Long => JLong}
3435
import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap}
3536

36-
import scala.annotation.nowarn
3737
import scala.collection.JavaConverters._
3838

3939
object GlutenIcebergSourceUtil {
@@ -51,7 +51,6 @@ object GlutenIcebergSourceUtil {
5151
}
5252
}
5353

54-
@nowarn
5554
def genSplitInfo(
5655
partition: SparkDataSourceRDDPartition,
5756
readPartitionSchema: StructType): SplitInfo = {
@@ -69,7 +68,7 @@ object GlutenIcebergSourceUtil {
6968
task =>
7069
paths.add(
7170
BackendsApiManager.getTransformerApiInstance
72-
.encodeFilePathIfNeed(task.file().path().toString))
71+
.encodeFilePathIfNeed(ContentFileUtil.getFilePath(task.file())))
7372
starts.add(task.start())
7473
lengths.add(task.length())
7574
partitionColumns.add(getPartitionColumns(task, readPartitionSchema))

pom.xml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
<spark.version>3.5.5</spark.version>
7474
<sparkshim.artifactId>spark-sql-columnar-shims-spark35</sparkshim.artifactId>
7575
<iceberg.version>1.10.0</iceberg.version>
76+
<iceberg.binary.version>10</iceberg.binary.version>
7677
<paimon.version>1.2.0</paimon.version>
7778
<delta.package.name>delta-spark</delta.package.name>
7879
<delta.version>3.3.2</delta.version>
@@ -1077,6 +1078,7 @@
10771078
<sparkshim.artifactId>spark-sql-columnar-shims-spark32</sparkshim.artifactId>
10781079
<spark.version>3.2.2</spark.version>
10791080
<iceberg.version>1.3.1</iceberg.version>
1081+
<iceberg.binary.version>3</iceberg.binary.version>
10801082
<delta.package.name>delta-core</delta.package.name>
10811083
<delta.version>2.0.1</delta.version>
10821084
<delta.binary.version>20</delta.binary.version>
@@ -1109,6 +1111,7 @@
11091111
<sparkshim.artifactId>spark-sql-columnar-shims-spark33</sparkshim.artifactId>
11101112
<spark.version>3.3.1</spark.version>
11111113
<iceberg.version>1.5.0</iceberg.version>
1114+
<iceberg.binary.version>5</iceberg.binary.version>
11121115
<delta.package.name>delta-core</delta.package.name>
11131116
<delta.version>2.3.0</delta.version>
11141117
<delta.binary.version>23</delta.binary.version>

0 commit comments

Comments
 (0)