Merge remote-tracking branch 'delta-io/master' into spark-4.0-upgrade-merge

Author: allisonport-db

kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/client/TestDefaultFileSystemClient.java

@@ -18,7 +18,6 @@ package io.delta.kernel.defaults
 import io.delta.golden.GoldenTableUtils.goldenTablePath
 
 import io.delta.kernel.defaults.utils.{TestRow, TestUtils}
-import io.delta.kernel.defaults.utils.DefaultKernelTestUtils.getTestResourceFilePath
 import org.apache.hadoop.conf.Configuration
 import org.scalatest.funsuite.AnyFunSuite
 

kernel/kernel-defaults/src/test/java/io/delta/kernel/defaults/client/TestDefaultJsonHandler.java

@@ -16,13 +16,14 @@
 package io.delta.kernel.defaults
 
 import java.math.{BigDecimal => BigDecimalJ}
+
 import scala.collection.JavaConverters._
+
 import io.delta.golden.GoldenTableUtils.goldenTablePath
 import io.delta.kernel.defaults.utils.{TestRow, TestUtils}
 import io.delta.kernel.expressions.{Column, Expression, Predicate}
 import io.delta.kernel.expressions.Literal._
 import io.delta.kernel.types._
-import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.scalatest.funsuite.AnyFunSuite
 
 class PartitionPruningSuite extends AnyFunSuite with TestUtils {

kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeletionVectorSuite.scala

@@ -15,8 +15,6 @@
  */
 package io.delta.kernel.defaults.internal.expressions
 
-import java.util
-
 import io.delta.kernel.data.{ColumnarBatch, ColumnVector}
 import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch
 import io.delta.kernel.defaults.utils.{TestUtils, VectorTestUtils}

kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/PartitionPruningSuite.scala

@@ -35,7 +35,7 @@ import io.delta.kernel.internal.data.ScanStateRow
 import io.delta.kernel.internal.util.ColumnMapping
 import io.delta.kernel.internal.util.Utils.singletonCloseableIterator
 import io.delta.kernel.types._
-import io.delta.kernel.utils.{CloseableIterator, DataFileStatus}
+import io.delta.kernel.utils.CloseableIterator
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.shaded.org.apache.commons.io.FileUtils
 import org.apache.spark.sql.SparkSession

kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/expressions/ExpressionSuiteBase.scala

@@ -143,7 +143,7 @@ are responsible to define the commit atomicity and backfill protocols which the
 
 At a high level, the `commit-owner` needs to provide:
 - API to atomically commit a version `x` with given set of `actions`. This is explained in detail in the [commit protocol](#commit-protocol) section.
-- API to retrieve information about the recent commits on the table. This is explained in detail in the [getting un-backfilled commits from commit-owner](#getting-un-backfilled-commits-from-commit-owner) section.
+- API to retrieve information about the recent commits and the latest ratified version on the table. This is explained in detail in the [getting un-backfilled commits from commit-owner](#getting-un-backfilled-commits-from-commit-owner) section.
 
 ### Commit Protocol
 
@@ -161,7 +161,14 @@ Even after a commit succeeds, Delta clients can only discover the commit through
 have no way to determine which file in `_delta_log/_commits` directory corresponds to the actual commit `v`.
 
 The commit-owner is responsible to implement an API (defined by the Delta client) that Delta clients can use to retrieve information about un-backfilled commits maintained
-by the commit-owner. Delta clients who are unaware of the commit-owner (or unwilling to talk to it), may not see recent un-backfilled commits and thus may encounter stale reads.
+by the commit-owner. The API must also return the latest version of the table ratified by the commit-owner (if any).
+Providing the latest ratified table version helps address potential race conditions between listing commits and contacting the commit-owner.
+For example, if a client performs a listing before a recently ratified commit is backfilled, and then contacts the commit-owner after the backfill completes,
+the commit-owner may return an empty list of un-backfilled commits. Without knowing the latest ratified version, the client might incorrectly assume their listing was complete
+and read a stale snapshot.
+
+Delta clients who are unaware of the commit-owner (or unwilling to talk to it), may not see recent un-backfilled commits and thus may encounter stale reads.
+
 
 ## Sample Commit Owner API
 
@@ -176,7 +183,7 @@ interface CommitStore {
      * @param version The version we want to commit.
      * @param actions Actions that need to be committed.
      *
-     * returns CommitResponse which has details around the new committed delta file.
+     * @return CommitResponse which has details around the new committed delta file.
      */
     def commit(
         version: Long,
@@ -191,13 +198,16 @@ interface CommitStore {
      * Note that the first version returned by this API may not be equal to the `startVersion`. This
      * happens when few versions starting from `startVersion` are already backfilled and so
      * CommitStore may have stopped tracking them.
+     * The returned latestTableVersion is the maximum commit version ratified by the Commit-Owner.
+     * Note that returning latestTableVersion as -1 is acceptable only if the commit-owner never
+     * ratified any version i.e. it never accepted any un-backfilled commit.
      *
-     * @return a list of `Commit` which are tracked by commit-owner.
-     *
+     * @return GetCommitsResponse which contains a list of `Commit`s and the latestTableVersion
+     *         tracked by the commit-owner.
      */
     def getCommits(
         startVersion: Long,
-        endVersion: Long): Seq[Commit]
+        endVersion: Long): GetCommitsResponse
 
     /**
      * API to ask the commit-owner to backfill all commits <= given `version`.