chore: [DNM] Release 1.1.1 rc2 upstream

.github/workflows/bot.yml

@@ -1199,6 +1199,15 @@ jobs:
           architecture: x64
       - name: Check disk space
         run: df -h
+      - name: 'Free space'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          docker system prune --all --force --volumes
+      - name: Check disk space after cleanup
+        run: df -h
       - name: Build Project
         env:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
@@ -1225,7 +1234,6 @@ jobs:
           SPARK_ARCHIVE_BASENAME=$(basename $SPARK_ARCHIVE)
           export SPARK_HOME=$GITHUB_WORKSPACE/${SPARK_ARCHIVE_BASENAME%.*}
           rm -f $GITHUB_WORKSPACE/$SPARK_ARCHIVE
-          docker system prune --all --force
           mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl !hudi-flink-datasource/hudi-flink $MVN_ARGS
 
   build-spark-java17:

.github/workflows/release_candidate_validation.yml

@@ -17,15 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     if: false
     env:
-      HUDI_VERSION: 1.0.0
-      STAGING_REPO_NUM: 1123
+      HUDI_VERSION: 1.1.0
+      STAGING_REPO_NUM: 1164
     strategy:
       matrix:
         include:
-          - scalaProfile: 'scala-2.13'
-            flinkProfile: 'flink2.0'
-            sparkProfile: 'spark3.5'
-            sparkRuntime: 'spark3.5.1'
           - scalaProfile: 'scala-2.13'
             flinkProfile: 'flink1.20'
             sparkProfile: 'spark3.5'
@@ -75,3 +71,73 @@ jobs:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
         run: |
           ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 $STAGING_REPO_NUM
+
+  validate-release-candidate-bundles-spark4:
+    runs-on: ubuntu-latest
+    if: false
+    env:
+      HUDI_VERSION: 1.1.0
+      STAGING_REPO_NUM: 1164
+    strategy:
+      matrix:
+        include:
+          - scalaProfile: 'scala-2.13'
+            flinkProfile: 'flink1.20'
+            sparkProfile: 'spark4.0'
+            sparkRuntime: 'spark4.0.0'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          architecture: x64
+          cache: maven
+      - name: IT - Bundle Validation - OpenJDK 17
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        run: |
+          ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 $STAGING_REPO_NUM
+
+  validate-release-candidate-bundles-flink2:
+    runs-on: ubuntu-latest
+    if: false
+    env:
+      HUDI_VERSION: 1.1.0
+      STAGING_REPO_NUM: 1164
+    strategy:
+      matrix:
+        include:
+          - scalaProfile: 'scala-2.12'
+            flinkProfile: 'flink2.0'
+            sparkProfile: 'spark3.5'
+            sparkRuntime: 'spark3.5.1'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          architecture: x64
+          cache: maven
+      - name: IT - Bundle Validation - OpenJDK 11
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        run: |
+          ./packaging/bundle-validation/ci_run.sh hudi_docker_java11 $HUDI_VERSION openjdk11 $STAGING_REPO_NUM
+      - name: IT - Bundle Validation - OpenJDK 17
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        run: |
+          ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 $STAGING_REPO_NUM

docker/hoodie/hadoop/base/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/base_java11/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/datanode/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/historyserver/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/hive_base/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/namenode/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>

docker/hoodie/hadoop/prestobase/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/spark_base/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/sparkadhoc/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/sparkmaster/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/sparkworker/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <packaging>pom</packaging>

docker/hoodie/hadoop/trinobase/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>hudi-hadoop-docker</artifactId>
         <groupId>org.apache.hudi</groupId>
-        <version>1.1.0</version>
+        <version>1.1.1-rc2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>

docker/hoodie/hadoop/trinocoordinator/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>hudi-hadoop-docker</artifactId>
         <groupId>org.apache.hudi</groupId>
-        <version>1.1.0</version>
+        <version>1.1.1-rc2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>

docker/hoodie/hadoop/trinoworker/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>hudi-hadoop-docker</artifactId>
         <groupId>org.apache.hudi</groupId>
-        <version>1.1.0</version>
+        <version>1.1.1-rc2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>

hudi-aws/pom.xml

@@ -19,12 +19,12 @@
     <parent>
         <artifactId>hudi</artifactId>
         <groupId>org.apache.hudi</groupId>
-        <version>1.1.0</version>
+        <version>1.1.1-rc2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>hudi-aws</artifactId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
 
     <name>hudi-aws</name>
     <packaging>jar</packaging>

hudi-cli/pom.xml

@@ -19,7 +19,7 @@
   <parent>
     <artifactId>hudi</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

hudi-client/hudi-client-common/pom.xml

@@ -20,12 +20,12 @@
   <parent>
     <artifactId>hudi-client</artifactId>
     <groupId>org.apache.hudi</groupId>
-    <version>1.1.0</version>
+    <version>1.1.1-rc2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
   <artifactId>hudi-client-common</artifactId>
-  <version>1.1.0</version>
+  <version>1.1.1-rc2</version>
 
   <name>hudi-client-common</name>
   <packaging>jar</packaging>

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java

@@ -54,6 +54,7 @@
 import org.apache.hudi.common.table.marker.MarkerType;
 import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
 import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
+import org.apache.hudi.common.table.view.FileSystemViewStorageType;
 import org.apache.hudi.common.util.ConfigUtils;
 import org.apache.hudi.common.util.FileIOUtils;
 import org.apache.hudi.common.util.HoodieRecordUtils;
@@ -1628,6 +1629,12 @@ public boolean isEmbeddedTimelineServerReuseEnabled() {
     return getBoolean(EMBEDDED_TIMELINE_SERVER_REUSE_ENABLED);
   }
 
+  public boolean isRemoteViewStorageType() {
+    FileSystemViewStorageType storageType = getViewStorageConfig().getStorageType();
+    return storageType == FileSystemViewStorageType.REMOTE_ONLY
+        || storageType == FileSystemViewStorageType.REMOTE_FIRST;
+  }
+
   public int getEmbeddedTimelineServerPort() {
     return Integer.parseInt(getStringOrDefault(EMBEDDED_TIMELINE_SERVER_PORT_NUM));
   }
@@ -2696,7 +2703,7 @@ public float getRecordIndexGrowthFactor() {
     return metadataConfig.getRecordIndexGrowthFactor();
   }
 
-  public int getRecordIndexMaxFileGroupSizeBytes() {
+  public long getRecordIndexMaxFileGroupSizeBytes() {
     return metadataConfig.getRecordIndexMaxFileGroupSizeBytes();
   }
 

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteMergeHandle.java

@@ -242,10 +242,6 @@ protected void populateIncomingRecordsMap(Iterator<HoodieRecord<T>> newRecordsIt
     }
   }
 
-  public boolean isEmptyNewRecords() {
-    return keyToNewRecords.isEmpty();
-  }
-
   protected boolean writeUpdateRecord(HoodieRecord<T> newRecord, HoodieRecord<T> oldRecord, HoodieRecord combineRecord, Schema writerSchema) throws IOException {
     boolean isDelete = false;
     if (oldRecord.getData() != combineRecord.getData()) {

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java

@@ -53,7 +53,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -133,25 +132,23 @@ public HoodieCompactionPlan generateCompactionPlan(String compactionInstant) thr
         .getLatestFileSlicesStateless(partitionPath)
         .filter(slice -> filterFileSlice(slice, lastCompletedInstantTime, fgIdsInPendingCompactionAndClustering, instantRange))
         .map(s -> {
-          List<HoodieLogFile> logFiles = s.getLogFiles()
-              // ==============================================================
-              // IMPORTANT
-              // ==============================================================
-              // Currently, our filesystem view could return a file slice with pending log files there,
-              // these files should be excluded from the plan, let's say we have such a sequence of actions
-
-              // t10: a delta commit starts,
-              // t20: the compaction is scheduled and the t10 delta commit is still pending, and the "fg_10.log" is included in the plan
-              // t25: the delta commit 10 finishes,
-              // t30: the compaction execution starts, now the reader considers the log file "fg_10.log" as valid.
-
-              // for both OCC and NB-CC, this is in-correct.
-              .filter(logFile -> completionTimeQueryView.isCompletedBefore(compactionInstant, logFile.getDeltaCommitTime()))
-              .sorted(HoodieLogFile.getLogFileComparator()).collect(toList());
-          if (logFiles.isEmpty()) {
-            // compaction is not needed if there is no log file.
-            return null;
-          }
+          // ==============================================================
+          // IMPORTANT
+          // ==============================================================
+          // Currently, our filesystem view could return a file slice with pending log files there,
+          // these files should be excluded from the plan, let's say we have such a sequence of actions
+
+          // t10: a delta commit starts,
+          // t20: the compaction is scheduled and the t10 delta commit is still pending, and the "fg_10.log" is included in the plan
+          // t25: the delta commit 10 finishes,
+          // t30: the compaction execution starts, now the reader considers the log file "fg_10.log" as valid.
+
+          // for both OCC and NB-CC, this is in-correct.
+          return s.filterLogFiles(logFile -> completionTimeQueryView.isCompletedBefore(compactionInstant, logFile.getDeltaCommitTime()));
+        })
+        .filter(FileSlice::hasLogFiles) // compaction is not needed if there is no log file.
+        .map(s -> {
+          List<HoodieLogFile> logFiles = s.getLogFiles().sorted(HoodieLogFile.getLogFileComparator()).collect(toList());
           totalLogFiles.add(logFiles.size());
           totalFileSlices.add(1L);
           // Avro generated classes are not inheriting Serializable. Using CompactionOperation POJO
@@ -160,7 +157,7 @@ public HoodieCompactionPlan generateCompactionPlan(String compactionInstant) thr
           Option<HoodieBaseFile> dataFile = s.getBaseFile();
           return new CompactionOperation(dataFile, partitionPath, logFiles,
               writeConfig.getCompactionStrategy().captureMetrics(writeConfig, s));
-        }).filter(Objects::nonNull), partitionPaths.size()).stream()
+        }), partitionPaths.size()).stream()
         .map(CompactionUtils::buildHoodieCompactionOperation).collect(toList());
 
     LOG.info("Total of {} compaction operations are retrieved for table {}", operations.size(), hoodieTable.getConfig().getBasePath());
@@ -194,7 +191,7 @@ public HoodieCompactionPlan generateCompactionPlan(String compactionInstant) thr
 
   protected abstract List<String> getPartitions();
 
-  protected abstract HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations, Pair<List<String>,List<String>> partitionPair);
+  protected abstract HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, List<HoodieCompactionOperation> operations, Pair<List<String>, List<String>> partitionPair);
 
   protected abstract boolean filterLogCompactionOperations();