Skip to content

Commit 99afb3c

Browse files
dongjoon-hyundongjoon-hyun
committed
[SPARK-51461] Setup SparkConnect package structure and CI to test build 
1 parent 4e28527 commit 99afb3c

File tree

8 files changed

+263
-66
lines changed

8 files changed

+263
-66
lines changed

‎.github/.licenserc.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,6 @@ header:
1414
- 'NOTICE'
1515
- '.asf.yaml'
1616
- '.nojekyll'
17+
- 'Package.swift'
1718

1819
comment: on-failure

‎.github/workflows/build_and_test.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,19 @@ jobs:
3838
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3939
with:
4040
config: .github/.licenserc.yaml
41+
42+
build:
43+
runs-on: ${{ matrix.os }}
44+
strategy:
45+
fail-fast: false
46+
matrix:
47+
os:
48+
- ubuntu-latest
49+
- macos-15
50+
steps:
51+
- uses: actions/checkout@v4
52+
- uses: swift-actions/[email protected]
53+
with:
54+
swift-version: "6"
55+
- name: Build
56+
run: swift build -v

‎.gitignore‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Xcode
2+
#
3+
# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore
4+
5+
## Swift Package Manager
6+
Package.resolved
7+
.swiftpm/
8+
9+
## User settings
10+
xcuserdata/
11+
12+
## Obj-C/Swift specific
13+
*.hmap
14+
15+
## App packaging
16+
*.ipa
17+
*.dSYM.zip
18+
*.dSYM
19+
20+
## Playgrounds
21+
timeline.xctimeline
22+
playground.xcworkspace
23+
24+
# Swift Package Manager
25+
#
26+
# Add this line if you want to avoid checking in source code from Swift Package Manager dependencies.
27+
# Packages/
28+
# Package.pins
29+
# Package.resolved
30+
# *.xcodeproj
31+
#
32+
# Xcode automatically generates this directory with a .xcworkspacedata file and xcuserdata
33+
# hence it is not needed unless you have added a package configuration file to your project
34+
# .swiftpm
35+
36+
.build/
37+
38+
# CocoaPods
39+
#
40+
# We recommend against adding the Pods directory to your .gitignore. However
41+
# you should judge for yourself, the pros and cons are mentioned at:
42+
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
43+
#
44+
# Pods/
45+
#
46+
# Add this line if you want to avoid checking in source code from the Xcode workspace
47+
# *.xcworkspace
48+
49+
# Carthage
50+
#
51+
# Add this line if you want to avoid checking in source code from Carthage dependencies.
52+
# Carthage/Checkouts
53+
54+
Carthage/Build/
55+
56+
# fastlane
57+
#
58+
# It is recommended to not store the screenshots in the git repo.
59+
# Instead, use fastlane to re-generate the screenshots whenever they are needed.
60+
# For more information about the recommended setup visit:
61+
# https://docs.fastlane.tools/best-practices/source-control/#source-control
62+
63+
fastlane/report.xml
64+
fastlane/Preview.html
65+
fastlane/screenshots/**/*.png
66+
fastlane/test_output

‎LICENSE‎

Lines changed: 0 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -199,68 +199,3 @@
199199
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200200
See the License for the specific language governing permissions and
201201
limitations under the License.
202-
203-
204-
------------------------------------------------------------------------------------
205-
This product bundles various third-party components under other open source licenses.
206-
This section summarizes those components and their licenses. See licenses/
207-
for text of these licenses.
208-
209-
210-
Apache Software Foundation License 2.0
211-
--------------------------------------
212-
213-
common/network-common/src/main/java/org/apache/spark/network/util/LimitedInputStream.java
214-
core/src/main/java/org/apache/spark/util/collection/TimSort.java
215-
core/src/main/resources/org/apache/spark/ui/static/bootstrap*
216-
core/src/main/resources/org/apache/spark/ui/static/vis*
217-
connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java
218-
core/src/main/resources/org/apache/spark/ui/static/d3-flamegraph.min.js
219-
core/src/main/resources/org/apache/spark/ui/static/d3-flamegraph.css
220-
221-
Python Software Foundation License
222-
----------------------------------
223-
224-
python/pyspark/loose_version.py
225-
226-
BSD 3-Clause
227-
------------
228-
229-
python/lib/py4j-*-src.zip
230-
python/pyspark/cloudpickle/*.py
231-
python/pyspark/join.py
232-
233-
The CSS style for the navigation sidebar of the documentation was originally
234-
submitted by Óscar Nájera for the scikit-learn project. The scikit-learn project
235-
is distributed under the 3-Clause BSD license.
236-
237-
238-
MIT License
239-
-----------
240-
241-
core/src/main/resources/org/apache/spark/ui/static/dagre-d3.min.js
242-
core/src/main/resources/org/apache/spark/ui/static/*dataTables*
243-
core/src/main/resources/org/apache/spark/ui/static/graphlib-dot.min.js
244-
core/src/main/resources/org/apache/spark/ui/static/jquery*
245-
core/src/main/resources/org/apache/spark/ui/static/sorttable.js
246-
docs/js/vendor/bootstrap*
247-
docs/js/vendor/anchor.min.js
248-
docs/js/vendor/jquery*
249-
docs/js/vendor/modernizer*
250-
docs/js/vendor/docsearch.min.js
251-
252-
ISC License
253-
-----------
254-
255-
core/src/main/resources/org/apache/spark/ui/static/d3.min.js
256-
257-
258-
Creative Commons CC0 1.0 Universal Public Domain Dedication
259-
-----------------------------------------------------------
260-
(see LICENSE-CC0.txt)
261-
262-
data/mllib/images/kittens/29.5.a_b_EGDP022204.jpg
263-
data/mllib/images/kittens/54893.jpg
264-
data/mllib/images/kittens/DP153539.jpg
265-
data/mllib/images/kittens/DP802813.jpg
266-
data/mllib/images/multi-channel/chr30.4.184.jpg

‎Package.swift‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// swift-tools-version: 6.0
2+
// The swift-tools-version declares the minimum version of Swift required to build this package.
3+
//
4+
// Licensed to the Apache Software Foundation (ASF) under one
5+
// or more contributor license agreements. See the NOTICE file
6+
// distributed with this work for additional information
7+
// regarding copyright ownership. The ASF licenses this file
8+
// to you under the Apache License, Version 2.0 (the
9+
// "License"); you may not use this file except in compliance
10+
// with the License. You may obtain a copy of the License at
11+
//
12+
// http://www.apache.org/licenses/LICENSE-2.0
13+
//
14+
// Unless required by applicable law or agreed to in writing,
15+
// software distributed under the License is distributed on an
16+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
// KIND, either express or implied. See the License for the
18+
// specific language governing permissions and limitations
19+
// under the License.
20+
//
21+
import PackageDescription
22+
23+
let package = Package(
24+
name: "SparkConnect",
25+
platforms: [
26+
.macOS(.v15),
27+
.iOS(.v18),
28+
.watchOS(.v11),
29+
.tvOS(.v18),
30+
],
31+
products: [
32+
.library(
33+
name: "SparkConnect",
34+
targets: ["SparkConnect"])
35+
],
36+
dependencies: [
37+
.package(url: "https://github.com/grpc/grpc-swift.git", from: "2.1.0"),
38+
.package(url: "https://github.com/grpc/grpc-swift-protobuf.git", from: "1.1.0"),
39+
.package(url: "https://github.com/grpc/grpc-swift-nio-transport.git", from: "1.0.1"),
40+
.package(url: "https://github.com/scottrhoyt/SwiftyTextTable.git", from: "0.9.0"),
41+
.package(url: "https://github.com/google/flatbuffers.git", branch: "v24.3.7"),
42+
],
43+
targets: [
44+
.target(
45+
name: "SparkConnect",
46+
dependencies: [
47+
.product(name: "GRPCCore", package: "grpc-swift"),
48+
.product(name: "GRPCProtobuf", package: "grpc-swift-protobuf"),
49+
.product(name: "GRPCNIOTransportHTTP2", package: "grpc-swift-nio-transport"),
50+
.product(name: "SwiftyTextTable", package: "SwiftyTextTable"),
51+
.product(name: "FlatBuffers", package: "flatbuffers"),
52+
]
53+
),
54+
.testTarget(
55+
name: "SparkConnectTests",
56+
dependencies: ["SparkConnect"]
57+
),
58+
]
59+
)

‎README.md‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,60 @@
1-
# Apache Spark Connect Client for Swift language
1+
# Apache Spark Connect Client for Swift
2+
3+
[![GitHub Actions Build](https://github.com/apache/spark-connect-swift/actions/workflows/build_and_test.yml/badge.svg)](https://github.com/apache/spark-connect-swift/blob/main/.github/workflows/build_and_test.yml)
4+
5+
This is an experimental Swift library to show how to connect to a remote Apache Spark Connect Server and run SQL statements to manipulate remote data.
6+
7+
So far, this library project is tracking the upstream changes like the [Apache Spark](https://spark.apache.org) 4.0.0 RC2 release and [Apache Arrow](https://arrow.apache.org) project's Swift-support.
8+
9+
## Requirement
10+
- [Apache Spark 4.0.0 RC2 (March 2025)](https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc2-bin/)
11+
- [Swift 6.0 (2024)](https://swift.org)
12+
- [gRPC Swift 2.1 (March 2025)](https://github.com/grpc/grpc-swift/releases/tag/2.1.0)
13+
- [gRPC Swift Protobuf 1.0 (March 2025)](https://github.com/grpc/grpc-swift-protobuf/releases/tag/1.1.0)
14+
- [gRPC Swift NIO Transport 1.0 (March 2025)](https://github.com/grpc/grpc-swift-nio-transport/releases/tag/1.0.1)
15+
- [Apache Arrow Swift](https://github.com/apache/arrow/tree/main/swift)
16+
17+
## Run `Apache Spark 4.0.0 RC2 Connect Server`
18+
19+
$ curl -LO https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc2-bin/spark-4.0.0-bin-hadoop3.tgz
20+
$ tar xvfz spark-4.0.0-bin-hadoop3.tgz
21+
$ cd spark-4.0.0-bin-hadoop3
22+
$ sbin/start-connect-server.sh
23+
24+
## Run tests
25+
26+
```
27+
$ cd spark-connect-swift
28+
$ swift test
29+
```
30+
31+
## How to use in your apps
32+
33+
Create a Swift project.
34+
```
35+
$ mkdir SparkConnectSwiftApp
36+
$ cd SparkConnectSwiftApp
37+
$ swift package init --name SparkConnectSwiftApp --type executable
38+
```
39+
40+
Add `SparkConnect` package to the dependency like the following
41+
```
42+
$ cat Package.swift
43+
import PackageDescription
44+
45+
let package = Package(
46+
name: "SparkConnectSwiftApp",
47+
platforms: [
48+
.macOS(.v15)
49+
],
50+
dependencies: [
51+
.package(url: "https://github.com/apache/spark-connect-swift.git", branch: "main")
52+
],
53+
targets: [
54+
.executableTarget(
55+
name: "SparkConnectSwiftApp",
56+
dependencies: [.product(name: "SparkConnect", package: "spark-connect-swift")]
57+
)
58+
]
59+
)
60+
```

‎Sources/SparkConnect/SparkConnectError.swift‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
20+
/// A enum for ``SparkConnect`` package errors
21+
enum SparkConnectError: Error {
22+
case UnsupportedOperationException
23+
}

‎Tests/SparkConnectTests/BuilderTests.swift‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
20+
import Testing
21+
22+
@testable import SparkConnect
23+
24+
/// A test suite for `SparkSession.Builder`
25+
@Suite(.serialized)
26+
struct BuilderTests {
27+
@Test
28+
func builderDefault() async throws {
29+
}
30+
31+
@Test
32+
func remote() async throws {
33+
}
34+
35+
@Test
36+
func appName() async throws {
37+
}
38+
}

0 commit comments

Comments
 (0)