From 5160124731b33f02e3295f4a511c7a49ef950947 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 9 Jul 2025 20:28:34 -0700 Subject: [PATCH] [SPARK-52744] Add `MacOS` integration test with Apache Spark 4.1.0-preview1 RC1 --- .github/workflows/build_and_test.yml | 18 ++++++++++++++++++ Sources/SparkConnect/DataFrame.swift | 1 - Tests/SparkConnectTests/DataFrameTests.swift | 9 ++++++--- .../SparkConnectTests/SparkSessionTests.swift | 2 +- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d9a1240..50d92e9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -93,6 +93,24 @@ jobs: docker run swiftlang/swift:nightly-6.2-noble uname -a docker run -v $PWD:/spark -w /spark swiftlang/swift:nightly-6.2-noble swift build + integration-test-mac-spark41: + runs-on: macos-15 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: swift-actions/setup-swift@d10500c1ac8822132eebbd74c48c3372c71d7ff5 + with: + swift-version: "6.1" + - name: Test + run: | + curl -LO https://dist.apache.org/repos/dist/dev/spark/v4.1.0-preview1-rc1-bin/spark-4.1.0-preview1-bin-hadoop3.tgz + tar xvfz spark-4.1.0-preview1-bin-hadoop3.tgz + mv spark-4.1.0-preview1-bin-hadoop3 /tmp/spark + cd /tmp/spark/sbin + ./start-connect-server.sh + cd - + swift test --no-parallel + integration-test-mac: runs-on: macos-15 timeout-minutes: 20 diff --git a/Sources/SparkConnect/DataFrame.swift b/Sources/SparkConnect/DataFrame.swift index bbd98bd..6ef2740 100644 --- a/Sources/SparkConnect/DataFrame.swift +++ b/Sources/SparkConnect/DataFrame.swift @@ -431,7 +431,6 @@ public actor DataFrame: Sendable { values.append(array.asAny(i) as! Date) case .timeInfo(.timestamp): let timestampType = column.data.type as! ArrowTypeTimestamp - assert(timestampType.timezone == "Etc/UTC") let timestamp = array.asAny(i) as! Int64 let timeInterval = switch timestampType.unit { diff --git a/Tests/SparkConnectTests/DataFrameTests.swift b/Tests/SparkConnectTests/DataFrameTests.swift index 62b117a..8db57d3 100644 --- a/Tests/SparkConnectTests/DataFrameTests.swift +++ b/Tests/SparkConnectTests/DataFrameTests.swift @@ -332,8 +332,10 @@ struct DataFrameTests { @Test func isLocal() async throws { let spark = try await SparkSession.builder.getOrCreate() - #expect(try await spark.sql("SHOW DATABASES").isLocal()) - #expect(try await spark.sql("SHOW TABLES").isLocal()) + if !(await spark.version.starts(with: "4.1")) { // TODO(SPARK-52746) + #expect(try await spark.sql("SHOW DATABASES").isLocal()) + #expect(try await spark.sql("SHOW TABLES").isLocal()) + } #expect(try await spark.range(1).isLocal() == false) await spark.stop() } @@ -936,8 +938,9 @@ struct DataFrameTests { @Test func timestamp() async throws { let spark = try await SparkSession.builder.getOrCreate() + // TODO(SPARK-52747) let df = try await spark.sql( - "SELECT TIMESTAMP '2025-05-01 16:23:40', TIMESTAMP '2025-05-01 16:23:40.123456'") + "SELECT TIMESTAMP '2025-05-01 16:23:40Z', TIMESTAMP '2025-05-01 16:23:40.123456Z'") let expected = [ Row( Date(timeIntervalSince1970: 1746116620.0), Date(timeIntervalSince1970: 1746116620.123456)) diff --git a/Tests/SparkConnectTests/SparkSessionTests.swift b/Tests/SparkConnectTests/SparkSessionTests.swift index 326f37d..5c3e634 100644 --- a/Tests/SparkConnectTests/SparkSessionTests.swift +++ b/Tests/SparkConnectTests/SparkSessionTests.swift @@ -96,7 +96,7 @@ struct SparkSessionTests { await SparkSession.builder.clear() let spark = try await SparkSession.builder.getOrCreate() let version = await spark.version - #expect(version.starts(with: "4.0.0") || version.starts(with: "3.5.")) + #expect(version.starts(with: "4.") || version.starts(with: "3.5.")) await spark.stop() }