diff --git a/Examples/app/Dockerfile b/Examples/app/Dockerfile new file mode 100644 index 0000000..1bbacd2 --- /dev/null +++ b/Examples/app/Dockerfile @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM swift:6.1 AS builder + +WORKDIR /app + +COPY . . + +RUN swift build -c release + +FROM swift:6.1-slim + +ARG SPARK_UID=185 + +LABEL org.opencontainers.image.authors="Apache Spark project " +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.ref.name="Apache Spark Connect for Swift" + +ENV SPARK_SWIFT_HOME=/opt/spark-swift +ENV SPARK_SWIFT_APP=SparkConnectSwiftApp + +WORKDIR $SPARK_SWIFT_HOME + +RUN groupadd --system --gid=$SPARK_UID spark && \ + useradd --system --home-dir $SPARK_SWIFT_HOME --uid=$SPARK_UID --gid=spark spark && \ + chown -R spark:spark $SPARK_SWIFT_HOME + +COPY --from=builder --chown=spark:spark /app/.build/*-unknown-linux-gnu/release/$SPARK_SWIFT_APP . + +USER spark + +ENTRYPOINT ["/bin/sh", "-c", "$SPARK_SWIFT_HOME/$SPARK_SWIFT_APP"] diff --git a/Examples/app/README.md b/Examples/app/README.md index 285a0d3..bede7c2 100644 --- a/Examples/app/README.md +++ b/Examples/app/README.md @@ -4,12 +4,55 @@ This is an example Swift application to show how to use Apache Spark Connect Swi ## How to run -Run this Swift application. +Prepare `Spark Connect Server` via running Docker image. + +``` +docker run --rm -p 15002:15002 apache/spark:4.0.0-preview2 bash -c "/opt/spark/sbin/start-connect-server.sh --wait" +``` + +Build an application Docker image. + +``` +$ docker build -t apache/spark-connect-swift:app . +$ docker images apache/spark-connect-swift:app +REPOSITORY TAG IMAGE ID CREATED SIZE +apache/spark-connect-swift app e132e1b38348 5 seconds ago 368MB +``` + +Run `app` docker image. + +``` +$ docker run --rm -e SPARK_REMOTE=sc://host.docker.internal:15002 apache/spark-connect-swift:app +Connected to Apache Spark 4.0.0-preview2 Server +EXECUTE: DROP TABLE IF EXISTS t +EXECUTE: CREATE TABLE IF NOT EXISTS t(a INT) USING ORC +EXECUTE: INSERT INTO t VALUES (1), (2), (3) +SELECT * FROM t ++---+ +| a| ++---+ +| 2| +| 1| +| 3| ++---+ + ++---+ +| id| ++---+ +| 0| +| 8| +| 6| +| 2| +| 4| ++---+ +``` + +Run from source code. ``` $ swift run ... -Connected to Apache Spark 4.0.0 Server +Connected to Apache Spark 4.0.0-preview2 Server EXECUTE: DROP TABLE IF EXISTS t EXECUTE: CREATE TABLE IF NOT EXISTS t(a INT) USING ORC EXECUTE: INSERT INTO t VALUES (1), (2), (3) diff --git a/Examples/pi/Dockerfile b/Examples/pi/Dockerfile new file mode 100644 index 0000000..49ee0de --- /dev/null +++ b/Examples/pi/Dockerfile @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM swift:6.1 AS builder + +WORKDIR /app + +COPY . . + +RUN swift build -c release + +FROM swift:6.1-slim + +ARG SPARK_UID=185 + +LABEL org.opencontainers.image.authors="Apache Spark project " +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.ref.name="Apache Spark Connect for Swift" + +ENV SPARK_SWIFT_HOME=/opt/spark-swift +ENV SPARK_SWIFT_APP=SparkConnectSwiftPi + +WORKDIR $SPARK_SWIFT_HOME + +RUN groupadd --system --gid=$SPARK_UID spark && \ + useradd --system --home-dir $SPARK_SWIFT_HOME --uid=$SPARK_UID --gid=spark spark && \ + chown -R spark:spark $SPARK_SWIFT_HOME + +COPY --from=builder --chown=spark:spark /app/.build/*-unknown-linux-gnu/release/$SPARK_SWIFT_APP . + +USER spark + +ENTRYPOINT ["/bin/sh", "-c", "$SPARK_SWIFT_HOME/$SPARK_SWIFT_APP"] diff --git a/Examples/pi/README.md b/Examples/pi/README.md index 3ae9d6b..715c381 100644 --- a/Examples/pi/README.md +++ b/Examples/pi/README.md @@ -4,11 +4,31 @@ This is an example Swift application to show how to use Apache Spark Connect Swi ## How to run -Run this Swift application. +Prepare `Spark Connect Server` via running Docker image. +``` +docker run --rm -p 15002:15002 apache/spark:4.0.0-preview2 bash -c "/opt/spark/sbin/start-connect-server.sh --wait" +``` + +Build an application Docker image. + +``` +$ docker build -t apache/spark-connect-swift:pi . +$ docker images apache/spark-connect-swift:pi +REPOSITORY TAG IMAGE ID CREATED SIZE +apache/spark-connect-swift pi d03952577564 4 seconds ago 369MB +``` + +Run `pi` docker image. + +``` +$ docker run --rm -e SPARK_REMOTE=sc://host.docker.internal:15002 apache/spark-connect-swift:pi +Pi is roughly 3.1412831412831412 +``` + +Run from source code. ``` -$ swift run SparkConnectSwiftPi 1000000 +$ swift run ... -Connected to Apache Spark 4.0.0 Server Pi is roughly 3.1423711423711422 ``` diff --git a/Examples/stream/Dockerfile b/Examples/stream/Dockerfile new file mode 100644 index 0000000..16583cd --- /dev/null +++ b/Examples/stream/Dockerfile @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM swift:6.1 AS builder + +WORKDIR /app + +COPY . . + +RUN swift build -c release + +FROM swift:6.1-slim + +ARG SPARK_UID=185 + +LABEL org.opencontainers.image.authors="Apache Spark project " +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.ref.name="Apache Spark Connect for Swift" + +ENV SPARK_SWIFT_HOME=/opt/spark-swift +ENV SPARK_SWIFT_APP=SparkConnectSwiftNetworkWordCount + +WORKDIR $SPARK_SWIFT_HOME + +RUN groupadd --system --gid=$SPARK_UID spark && \ + useradd --system --home-dir $SPARK_SWIFT_HOME --uid=$SPARK_UID --gid=spark spark && \ + chown -R spark:spark $SPARK_SWIFT_HOME + +COPY --from=builder --chown=spark:spark /app/.build/*-unknown-linux-gnu/release/$SPARK_SWIFT_APP . + +USER spark + +ENTRYPOINT ["/bin/sh", "-c", "$SPARK_SWIFT_HOME/$SPARK_SWIFT_APP"] diff --git a/Examples/stream/README.md b/Examples/stream/README.md index ee8553e..3d0221a 100644 --- a/Examples/stream/README.md +++ b/Examples/stream/README.md @@ -5,7 +5,7 @@ This is an example Swift stream processing application to show how to count word ## Run `Spark Connect Server` ```bash -./sbin/start-connect-server.sh --wait -c spark.log.level=ERROR +docker run --rm -p 15002:15002 apache/spark:4.0.0-preview2 bash -c "/opt/spark/sbin/start-connect-server.sh --wait -c spark.log.level=ERROR" ``` ## Run `Netcat` as a streaming input server @@ -16,12 +16,21 @@ You will first need to run Netcat (a small utility found in most Unix-like syste nc -lk 9999 ``` -## Start streaming processing application +## Build and run from docker image + +Build an application Docker image. -```bash -$ swift run -... -Connected to Apache Spark 4.0.0 Server +``` +$ docker build -t apache/spark-connect-swift:stream . +$ docker images apache/spark-connect-swift:stream +REPOSITORY TAG IMAGE ID CREATED SIZE +apache/spark-connect-swift stream a4daa10ad9c5 7 seconds ago 369MB +``` + +Run `stream` docker image. + +``` +$ docker run --rm -e SPARK_REMOTE=sc://host.docker.internal:15002 -e TARGET_HOST=host.docker.internal apache/spark-connect-swift:stream ``` ## Send input and check output @@ -34,7 +43,7 @@ apache spark apache hadoop ``` - `Spark Connect Server` output will look something like the following. +`Spark Connect Server` output will look something like the following. ```bash ------------------------------------------- @@ -66,3 +75,11 @@ Batch: 2 |hadoop| 1| +------+--------+ ``` + +## Run from source code + +```bash +$ swift run +... +Connected to Apache Spark 4.0.0 Server +``` diff --git a/Examples/stream/Sources/main.swift b/Examples/stream/Sources/main.swift index 2ce9a40..5c73356 100644 --- a/Examples/stream/Sources/main.swift +++ b/Examples/stream/Sources/main.swift @@ -17,17 +17,20 @@ // under the License. // +import Foundation import SparkConnect let spark = try await SparkSession.builder.getOrCreate() print("Connected to Apache Spark \(await spark.version) Server") +let host = ProcessInfo.processInfo.environment["TARGET_HOST"] ?? "localhost" + let lines = await spark .readStream .format("socket") - .option("host", "localhost") - .option("port", "9999") + .option("host", host) + .option("port", 9999) .load() let word = diff --git a/Examples/web/Dockerfile b/Examples/web/Dockerfile new file mode 100644 index 0000000..681f81f --- /dev/null +++ b/Examples/web/Dockerfile @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM swift:6.1 AS builder + +WORKDIR /app + +COPY . . + +RUN swift build -c release + +FROM swift:6.1-slim + +ARG SPARK_UID=185 + +LABEL org.opencontainers.image.authors="Apache Spark project " +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.ref.name="Apache Spark Connect for Swift" + +ENV SPARK_SWIFT_HOME=/opt/spark-swift +ENV SPARK_SWIFT_APP=SparkConnectSwiftWeb + +WORKDIR $SPARK_SWIFT_HOME + +RUN groupadd --system --gid=$SPARK_UID spark && \ + useradd --system --home-dir $SPARK_SWIFT_HOME --uid=$SPARK_UID --gid=spark spark && \ + chown -R spark:spark $SPARK_SWIFT_HOME + +COPY --from=builder --chown=spark:spark /app/.build/*-unknown-linux-gnu/release/$SPARK_SWIFT_APP . + +USER spark + +EXPOSE 8080 + +ENTRYPOINT ["/bin/sh", "-c", "$SPARK_SWIFT_HOME/$SPARK_SWIFT_APP"] diff --git a/Examples/web/Package.swift b/Examples/web/Package.swift index 7d793ab..f17d697 100644 --- a/Examples/web/Package.swift +++ b/Examples/web/Package.swift @@ -20,7 +20,7 @@ import PackageDescription let package = Package( - name: "SparkConnectSwiftWebapp", + name: "SparkConnectSwiftWeb", platforms: [ .macOS(.v15) ], @@ -33,7 +33,7 @@ let package = Package( ], targets: [ .executableTarget( - name: "SparkConnectSwiftWebapp", + name: "SparkConnectSwiftWeb", dependencies: [ .product(name: "Vapor", package: "vapor"), .product(name: "NIOCore", package: "swift-nio"), diff --git a/Examples/web/README.md b/Examples/web/README.md index 3ae473e..c744ef2 100644 --- a/Examples/web/README.md +++ b/Examples/web/README.md @@ -2,7 +2,6 @@ This project is designed to illustrate a Swift-based HTTP WebServer with Apache Spark Connect. -- https://swiftpackageindex.com/apache/spark-connect-swift - https://swiftpackageindex.com/vapor/vapor ## Create a Swift project @@ -73,17 +72,42 @@ index 2edcc8f..22313c8 100644 } ``` -## Run this Swift application. +## How to run + +Prepare `Spark Connect Server` via running Docker image. ``` -$ swift run +docker run --rm -p 15002:15002 apache/spark:4.0.0-preview2 bash -c "/opt/spark/sbin/start-connect-server.sh --wait" +``` + +Build an application Docker image. + +``` +$ docker build -t apache/spark-connect-swift:web . +$ docker images apache/spark-connect-swift:web +REPOSITORY TAG IMAGE ID CREATED SIZE +apache/spark-connect-swift web 3fd2422fdbee 27 seconds ago 417MB +``` + +Run `web` docker image + +``` +$ docker run -it --rm -p 8080:8080 -e SPARK_REMOTE=sc://host.docker.internal:15002 apache/spark-connect-swift:web +[ NOTICE ] Server started on http://127.0.0.1:8080 ``` -## Connect to the Swift Web Server to talk with `Apache Spark`. +Connect to the Swift Web Server to talk with `Apache Spark`. ``` $ curl http://127.0.0.1:8080/ Welcome to the Swift world. Say hello!% + $ curl http://127.0.0.1:8080/hello Hi, this is powered by the Apache Spark 4.0.0.% ``` + +Run from source code. + +``` +$ swift run +``` diff --git a/Examples/web/Sources/SparkConnectSwiftWeb/configure.swift b/Examples/web/Sources/SparkConnectSwiftWeb/configure.swift index cbc3edb..f70192a 100644 --- a/Examples/web/Sources/SparkConnectSwiftWeb/configure.swift +++ b/Examples/web/Sources/SparkConnectSwiftWeb/configure.swift @@ -21,6 +21,7 @@ import Vapor // configures your application public func configure(_ app: Application) async throws { + app.http.server.configuration.hostname = "0.0.0.0" // register routes try routes(app) }