spark-defaults.conf

# See: https://spark.apache.org/docs/latest/configuration.html
# Template for this configuration file is located at: $SPARK_HOME/conf/spark-defaults.conf.template

# Default system properties included when running spark-submit.
# This is useful for setting default environmental settings.

# Driver UI port
spark.ui.port                            4040

# History server settings
spark.eventLog.enabled                   true
spark.eventLog.dir                       file:///var/spark-events/eventlog
spark.eventLog.compress                  true
spark.eventLog.logBlockUpdates.enabled   true
spark.history.fs.logDirectory            file:///var/spark-events/eventlog

# Enable client deploy-mode driver logging persistence in History Server
spark.driver.log.persistToDfs.enabled    true
spark.driver.log.dfsDir                  file:///var/spark-events/driverLog

# Execution configurations
spark.serializer                         org.apache.spark.serializer.KryoSerializer

# Execution resources
spark.driver.cores                       2
spark.driver.memory                      2g
spark.executor.cores                     3
spark.executor.memory                    4g