Merge pull request #141 from arangodb-helper/custom-log-dir

Added `--log.dir` for custom log file folder

Author: ewoutp

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Changes from version 0.10.4 to master
 
+- Added `--log.dir` option to configure a custom directory to which all log files will be written.
+- It is no longer allowed to use `log.file` as a passthrough option.
 - Added `--starter.host` option, to bind the HTTP server to a specific network interface instead of the default `0.0.0.0`.
 - Added `POST /data-auto-upgrade` support to perform a rolling upgrade of all servers (with single `--database.auto-upgrade` restart)
 - Renamed mode option `resilientsingle` to `activefailover`. (`resilientsingle` is being supported as alias for a while)

docs/Manual/Programs/Starter/Options.md

@@ -277,6 +277,14 @@ debugging only.
 
 show more information (default false).
 
+- `--log.dir=path`
+
+set a custom directory to which all log files will be written to.
+When using the Starter in docker, make sure that this directory is
+mounted as a volume for the Starter.
+
+Note: When using a custom log directory, all files will be named as `arangod-<role>-<port>.log`.
+
 - `--log.rotate-files-to-keep=int`
 
 set the number of old log files to keep when rotating log files of server components (default 5).

main.go

@@ -96,6 +96,7 @@ var (
 	startLocalSlaves         bool
 	mode                     string
 	dataDir                  string
+	logDir                   string // Custom log directory (default "")
 	ownAddress               string
 	bindAddress              string
 	masterAddresses          []string
@@ -159,6 +160,7 @@ func init() {
 	f.BoolVar(&enableSync, "starter.sync", false, "If set, the starter will also start arangosync instances")
 
 	pf.BoolVar(&verbose, "log.verbose", false, "Turn on debug logging")
+	pf.StringVar(&logDir, "log.dir", getEnvVar("LOG_DIR", ""), "Custom log file directory.")
 	f.IntVar(&logRotateFilesToKeep, "log.rotate-files-to-keep", defaultLogRotateFilesToKeep, "Number of files to keep when rotating log files")
 	f.DurationVar(&logRotateInterval, "log.rotate-interval", defaultLogRotateInterval, "Time between log rotations (0 disables log rotation)")
 
@@ -545,6 +547,14 @@ func mustPrepareService(generateAutoKeyFile bool) (*service.Service, service.Boo
 		log.Fatalf("Cannot create data directory %s because %v, giving up.", dataDir, err)
 	}
 
+	// Make custom log directory absolute
+	if logDir != "" {
+		logDir, _ = filepath.Abs(logDir)
+		if err := os.MkdirAll(logDir, 0755); err != nil {
+			log.Fatalf("Cannot create custom log directory %s because %v, giving up.", logDir, err)
+		}
+	}
+
 	// Read jwtSecret (if any)
 	var jwtSecret string
 	if jwtSecretFile != "" {
@@ -651,6 +661,7 @@ func mustPrepareService(generateAutoKeyFile bool) (*service.Service, service.Boo
 		MasterPort:              masterPort,
 		RrPath:                  rrPath,
 		DataDir:                 dataDir,
+		LogDir:                  logDir,
 		OwnAddress:              ownAddress,
 		BindAddress:             bindAddress,
 		MasterAddresses:         masterAddresses,

service/arangod_config_builder.go

@@ -129,7 +129,7 @@ func createArangodConf(log *logging.Logger, bsCfg BootstrapConfig, myHostDir, my
 }
 
 // createArangodArgs returns the command line arguments needed to run an arangod server of given type.
-func createArangodArgs(log *logging.Logger, config Config, clusterConfig ClusterConfig, myContainerDir string,
+func createArangodArgs(log *logging.Logger, config Config, clusterConfig ClusterConfig, myContainerDir, myContainerLogFile string,
 	myPeerID, myAddress, myPort string, serverType ServerType, arangodConfig configFile, agentRecoveryID string, databaseAutoUpgrade bool) []string {
 	containerConfFileName := filepath.Join(myContainerDir, arangodConfFileName)
 
@@ -149,7 +149,7 @@ func createArangodArgs(log *logging.Logger, config Config, clusterConfig Cluster
 		optionPair{"--database.directory", slasher(filepath.Join(myContainerDir, "data"))},
 		optionPair{"--javascript.startup-directory", slasher(jsStartup)},
 		optionPair{"--javascript.app-path", slasher(filepath.Join(myContainerDir, "apps"))},
-		optionPair{"--log.file", slasher(filepath.Join(myContainerDir, arangodLogFileName))},
+		optionPair{"--log.file", slasher(myContainerLogFile)},
 		optionPair{"--log.force-direct", "false"},
 	)
 	if databaseAutoUpgrade {

service/arangosync_config_builder.go

@@ -71,7 +71,7 @@ func createArangoSyncClusterSecretFile(log *logging.Logger, bsCfg BootstrapConfi
 }
 
 // createArangoSyncArgs returns the command line arguments needed to run an arangosync server of given type.
-func createArangoSyncArgs(log *logging.Logger, config Config, clusterConfig ClusterConfig, myContainerDir string,
+func createArangoSyncArgs(log *logging.Logger, config Config, clusterConfig ClusterConfig, myContainerDir, myContainerLogFile string,
 	myPeerID, myAddress, myPort string, serverType ServerType, clusterJWTSecretFile string) ([]string, error) {
 
 	options := make([]optionPair, 0, 32)
@@ -81,7 +81,7 @@ func createArangoSyncArgs(log *logging.Logger, config Config, clusterConfig Clus
 	}
 
 	options = append(options,
-		optionPair{"--log.file", filepath.Join(myContainerDir, arangoSyncLogFileName)},
+		optionPair{"--log.file", myContainerLogFile},
 		optionPair{"--server.endpoint", "https://" + net.JoinHostPort(myAddress, myPort)},
 		optionPair{"--server.port", myPort},
 		optionPair{"--monitoring.token", config.SyncMonitoringToken},

service/passthrough.go

@@ -52,6 +52,7 @@ var (
 		"database.directory",
 		"javascript.startup-directory",
 		"javascript.app-path",
+		"log.file",
 		"rocksdb.encryption-keyfile",
 		"server.endpoint",
 		"server.authentication",

service/process_type.go

@@ -22,6 +22,8 @@
 
 package service
 
+import "fmt"
+
 // ProcessType specifies the types of process (the starter can work with).
 type ProcessType string
 
@@ -49,12 +51,12 @@ func (s ProcessType) CommandFileName() string {
 }
 
 // LogFileName returns the name of the log file used by this process
-func (s ProcessType) LogFileName() string {
+func (s ProcessType) LogFileName(suffix string) string {
 	switch s {
 	case ProcessTypeArangod:
-		return arangodLogFileName
+		return fmt.Sprintf("arangod%s.log", suffix)
 	case ProcessTypeArangoSync:
-		return arangoSyncLogFileName
+		return fmt.Sprintf("arangosync%s.log", suffix)
 	default:
 		return ""
 	}

service/runtime_server_manager.go

@@ -59,6 +59,13 @@ type runtimeServerManagerContext interface {
 
 	// serverHostDir returns the path of the folder (in host namespace) containing data for the given server.
 	serverHostDir(serverType ServerType) (string, error)
+	// serverContainerDir returns the path of the folder (in container namespace) containing data for the given server.
+	serverContainerDir(serverType ServerType) (string, error)
+
+	// serverHostLogFile returns the path of the logfile (in host namespace) to which the given server will write its logs.
+	serverHostLogFile(serverType ServerType) (string, error)
+	// serverContainerLogFile returns the path of the logfile (in container namespace) to which the given server will write its logs.
+	serverContainerLogFile(serverType ServerType) (string, error)
 
 	// removeRecoveryFile removes any recorded RECOVERY file.
 	removeRecoveryFile()
@@ -88,6 +95,15 @@ func startServer(ctx context.Context, log *logging.Logger, runtimeContext runtim
 	if err != nil {
 		return nil, false, maskAny(err)
 	}
+	myContainerDir, err := runtimeContext.serverContainerDir(serverType)
+	if err != nil {
+		return nil, false, maskAny(err)
+	}
+	myContainerLogFile, err := runtimeContext.serverContainerLogFile(serverType)
+	if err != nil {
+		return nil, false, maskAny(err)
+	}
+
 	os.MkdirAll(filepath.Join(myHostDir, "data"), 0755)
 	os.MkdirAll(filepath.Join(myHostDir, "apps"), 0755)
 
@@ -120,7 +136,6 @@ func startServer(ctx context.Context, log *logging.Logger, runtimeContext runtim
 	}
 
 	log.Infof("Starting %s on port %d", serverType, myPort)
-	myContainerDir := runner.GetContainerDir(myHostDir, dockerDataDir)
 	processType := serverType.ProcessType()
 	// Create/read arangod.conf
 	var confVolumes []Volume
@@ -147,7 +162,7 @@ func startServer(ctx context.Context, log *logging.Logger, runtimeContext runtim
 	clusterConfig, myPeer, _ := runtimeContext.ClusterConfig()
 	upgradeManager := runtimeContext.UpgradeManager()
 	databaseAutoUpgrade := upgradeManager.ServerDatabaseAutoUpgrade(serverType)
-	args, err := createServerArgs(log, config, clusterConfig, myContainerDir, myPeer.ID, myHostAddress, strconv.Itoa(myPort), serverType, arangodConfig, containerSecretFileName, bsCfg.RecoveryAgentID, databaseAutoUpgrade)
+	args, err := createServerArgs(log, config, clusterConfig, myContainerDir, myContainerLogFile, myPeer.ID, myHostAddress, strconv.Itoa(myPort), serverType, arangodConfig, containerSecretFileName, bsCfg.RecoveryAgentID, databaseAutoUpgrade)
 	if err != nil {
 		return nil, false, maskAny(err)
 	}
@@ -174,13 +189,11 @@ func startServer(ctx context.Context, log *logging.Logger, runtimeContext runtim
 
 // showRecentLogs dumps the most recent log lines of the server of given type to the console.
 func (s *runtimeServerManager) showRecentLogs(log *logging.Logger, runtimeContext runtimeServerManagerContext, serverType ServerType) {
-	myHostDir, err := runtimeContext.serverHostDir(serverType)
+	logPath, err := runtimeContext.serverHostLogFile(serverType)
 	if err != nil {
-		log.Errorf("Cannot find server host dir: %#v", err)
+		log.Errorf("Cannot find server host log file: %#v", err)
 		return
 	}
-	logFileName := serverType.ProcessType().LogFileName()
-	logPath := filepath.Join(myHostDir, logFileName)
 	logFile, err := os.Open(logPath)
 	if os.IsNotExist(err) {
 		log.Infof("Log file for %s is empty", serverType)
@@ -363,13 +376,11 @@ func (s *runtimeServerManager) rotateLogFile(ctx context.Context, log *logging.L
 	}
 
 	// Prepare log path
-	myHostDir, err := runtimeContext.serverHostDir(serverType)
+	logPath, err := runtimeContext.serverHostLogFile(serverType)
 	if err != nil {
-		log.Debugf("Failed to get host directory for '%s': %s", serverType, err)
+		log.Debugf("Failed to get host log file for '%s': %s", serverType, err)
 		return
 	}
-	logFileName := serverType.ProcessType().LogFileName()
-	logPath := filepath.Join(myHostDir, logFileName)
 	log.Debugf("Rotating %s log file: %s", serverType, logPath)
 
 	// Move old files

service/server.go

@@ -32,7 +32,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"path/filepath"
 	"strconv"
 
 	"github.com/arangodb-helper/arangodb/client"
@@ -85,8 +84,8 @@ type httpServerContext interface {
 	// IsRunningMaster returns if the starter is the running master.
 	IsRunningMaster() (isRunningMaster, isRunning bool, masterURL string)
 
-	// serverHostDir returns the path of the folder (in host namespace) containing data for the given server.
-	serverHostDir(serverType ServerType) (string, error)
+	// serverHostLogFile returns the path of the logfile (in host namespace) to which the given server will write its logs.
+	serverHostLogFile(serverType ServerType) (string, error)
 
 	// sendMasterLeaveCluster informs the master that we're leaving for good.
 	// The master will remove the database servers from the cluster and update
@@ -496,14 +495,12 @@ func (s *httpServer) syncWorkerLogsHandler(w http.ResponseWriter, r *http.Reques
 
 func (s *httpServer) logsHandler(w http.ResponseWriter, r *http.Request, serverType ServerType) {
 	// Find log path
-	myHostDir, err := s.context.serverHostDir(serverType)
+	logPath, err := s.context.serverHostLogFile(serverType)
 	if err != nil {
 		// Not ready yet
 		w.WriteHeader(http.StatusServiceUnavailable)
 		return
 	}
-	logFileName := serverType.ProcessType().LogFileName()
-	logPath := filepath.Join(myHostDir, logFileName)
 	s.log.Debugf("Fetching logs in %s", logPath)
 	rd, err := os.Open(logPath)
 	if os.IsNotExist(err) {

service/server_config_builder.go

@@ -73,14 +73,14 @@ func collectServerConfigVolumes(serverType ServerType, config configFile) []Volu
 }
 
 // createServerArgs returns the command line arguments needed to run an arangod/arangosync server of given type.
-func createServerArgs(log *logging.Logger, config Config, clusterConfig ClusterConfig, myContainerDir string,
+func createServerArgs(log *logging.Logger, config Config, clusterConfig ClusterConfig, myContainerDir, myContainerLogFile string,
 	myPeerID, myAddress, myPort string, serverType ServerType, arangodConfig configFile,
 	clusterJWTSecretFile, agentRecoveryID string, databaseAutoUpgrade bool) ([]string, error) {
 	switch serverType.ProcessType() {
 	case ProcessTypeArangod:
-		return createArangodArgs(log, config, clusterConfig, myContainerDir, myPeerID, myAddress, myPort, serverType, arangodConfig, agentRecoveryID, databaseAutoUpgrade), nil
+		return createArangodArgs(log, config, clusterConfig, myContainerDir, myContainerLogFile, myPeerID, myAddress, myPort, serverType, arangodConfig, agentRecoveryID, databaseAutoUpgrade), nil
 	case ProcessTypeArangoSync:
-		return createArangoSyncArgs(log, config, clusterConfig, myContainerDir, myPeerID, myAddress, myPort, serverType, clusterJWTSecretFile)
+		return createArangoSyncArgs(log, config, clusterConfig, myContainerDir, myContainerLogFile, myPeerID, myAddress, myPort, serverType, clusterJWTSecretFile)
 	default:
 		return nil, nil
 	}