Skip to content

Commit 31587f9

Browse files
committed
feat: spark docker compose instructions added
1 parent 38078fe commit 31587f9

File tree

1 file changed

+39
-0
lines changed

1 file changed

+39
-0
lines changed
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# ⭐ Apache Spark
2+
3+
## Deployment
4+
5+
### Docker compose
6+
7+
```yaml
8+
services:
9+
spark_master:
10+
restart: always
11+
image: bitnami/spark:3.5
12+
ports:
13+
- 8080:8080
14+
- 7077:7077
15+
hostname: spark-master
16+
environment:
17+
- SPARK_MODE=master
18+
- SPARK_RPC_AUTHENTICATION_ENABLED=no
19+
- SPARK_RPC_ENCRYPTION_ENABLED=no
20+
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
21+
- SPARK_SSL_ENABLED=no
22+
- SPARK_WORKER_WEBUI_PORT=8081
23+
24+
spark_worker:
25+
restart: always
26+
image: bitnami/spark:3.5
27+
ports:
28+
- 8081:8081
29+
environment:
30+
- SPARK_MODE=worker
31+
- SPARK_MASTER_URL=spark://spark-master:7077
32+
- SPARK_WORKER_MEMORY=8G
33+
- SPARK_WORKER_CORES=4
34+
- AWS_ACCESS_KEY_ID=user
35+
- AWS_SECRET_ACCESS_KEY=password
36+
- AWS_REGION=us-east-1
37+
depends_on:
38+
- spark_master
39+
```

0 commit comments

Comments
 (0)