File tree Expand file tree Collapse file tree 1 file changed +39
-0
lines changed
docs/data_engineering/data_lakehouse Expand file tree Collapse file tree 1 file changed +39
-0
lines changed Original file line number Diff line number Diff line change 1+ # ⭐ Apache Spark
2+
3+ ## Deployment
4+
5+ ### Docker compose
6+
7+ ``` yaml
8+ services :
9+ spark_master :
10+ restart : always
11+ image : bitnami/spark:3.5
12+ ports :
13+ - 8080:8080
14+ - 7077:7077
15+ hostname : spark-master
16+ environment :
17+ - SPARK_MODE=master
18+ - SPARK_RPC_AUTHENTICATION_ENABLED=no
19+ - SPARK_RPC_ENCRYPTION_ENABLED=no
20+ - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
21+ - SPARK_SSL_ENABLED=no
22+ - SPARK_WORKER_WEBUI_PORT=8081
23+
24+ spark_worker :
25+ restart : always
26+ image : bitnami/spark:3.5
27+ ports :
28+ - 8081:8081
29+ environment :
30+ - SPARK_MODE=worker
31+ - SPARK_MASTER_URL=spark://spark-master:7077
32+ - SPARK_WORKER_MEMORY=8G
33+ - SPARK_WORKER_CORES=4
34+ - AWS_ACCESS_KEY_ID=user
35+ - AWS_SECRET_ACCESS_KEY=password
36+ - AWS_REGION=us-east-1
37+ depends_on :
38+ - spark_master
39+ ` ` `
You can’t perform that action at this time.
0 commit comments