update more defaults, logs

Author: davidnewhall

fog.conf

@@ -19,19 +19,19 @@ log_file_mb = 0
 # FW_LOG_FILES: How many logs files to keep around.
 log_files = 0
 # FW_BUFFER_UDP: This is the UDP socket buffer in bytes.
-buffer_udp = 1048576
+buffer_udp = 4194304 # 4MB
 # FW_BUFFER_PACKET: This is the size of the read buffer per packet.
-buffer_packet = 8192
+buffer_packet = 262144 # 256KB
 # FW_BUFFER_CHAN: This is the channel buffer between the listeners and processors.
-buffer_chan = 10240
+buffer_chan = 32768 # 32 thousand
 # FW_LISTENERS: How many UDP socket listener threads to start.
 listeners = 1
 # FW_PROCESSORS: How many packet processor threads to start.
 processors = 1
 # FW_WRITERS: How many file system operations may happen in parallel. 2-5 is fine with fast disks.
 writers = 1
 # FW_BUFFER_FILE_SYS: How many file pointers can be buffered into the file system writer.
-buffer_file_sys = 10240
+buffer_file_sys = 32768 # 32 thousand
 # FW_DEBUG: Prints 1 line per packet when enabled.
 debug = false
 

pkg/fog/logger.go

@@ -64,10 +64,22 @@ func (c *Config) Errorf(msg string, v ...any) {
 // printConfig logs the current configuration information.
 func (c *Config) printConfig() {
 	c.Printf("=> Fog Willow Starting, pid: %d", os.Getpid())
-	c.Printf("=> Listen Address / Password: %s / %v", c.ListenAddr, c.Password != "")
+	c.Printf("=> UDP Listen Address / Password: %s / %v", c.ListenAddr, c.Password != "")
+
+	if c.HTTPServer.TLSCertPath != "" && c.HTTPServer.TLSKeyPath != "" {
+		c.Printf("=> HTTPS Listen Address: %s (Cert=%s, Key=%s)", c.HTTPServer.ListenAddr, c.HTTPServer.TLSCertPath, c.HTTPServer.TLSKeyPath)
+	} else {
+		c.Printf("=> HTTP Listen Address: %s", c.HTTPServer.ListenAddr)
+	}
+
+	c.Printf("=> HTTP Server; Read/Header/Write/Idle: %s/%s/%s/%s, Max Header Bytes: %d",
+		c.HTTPServer.ReadTimeout, c.HTTPServer.ReadHeaderTimeout, c.HTTPServer.WriteTimeout,
+		c.HTTPServer.IdleTimeout, c.HTTPServer.MaxHeaderBytes)
+
 	c.Printf("=> Output Path: %s", c.OutputPath)
 	c.Printf("=> Intervals; Flush/Group: %s/%s", c.FlushInterval, c.GroupInterval)
 	c.Printf("=> Buffers; UDP/Packet/Chan/FS: %d/%d/%d/%d", c.BufferUDP, c.BufferPacket, c.BufferChan, c.BufferFileSys)
+	c.Printf("=> Buffer Pool (File Buffers): %v", c.BufferPool)
 	c.Printf("=> Threads; Listen/Process/Writer: %d/%d/%d", c.Listeners, c.Processors, c.Writers)
 
 	if c.LogFile != "" {

pkg/fog/start.go

@@ -57,7 +57,7 @@ type Config struct {
 func LoadConfigFile(path string) (*Config, error) {
 	config := &Config{
 		OutputPath:   DefaultOutputPath,
-		HTTPServer:   &httpserver.Config{ListenAddr: httpserver.DefaultListenAddr},
+		HTTPServer:   httpserver.DefaultConfig(),
 		BufferUDP:    DefaultUDPBuffer,
 		BufferPacket: DefaultPacketBuffer,
 		BufferChan:   DefaultChanBuffer,
@@ -136,17 +136,23 @@ func (c *Config) setup() {
 	c.Logger = c
 	c.Metrics = c.metrics
 	c.willow = willow.NeWillow(c.Config)
+
+	if c.HTTPServer == nil {
+		c.HTTPServer = httpserver.DefaultConfig()
+	}
+
+	c.HTTPServer.Setup()
 }
 
 func (c *Config) setupSocket() error {
-	addr, err := net.ResolveUDPAddr("udp", c.HTTPServer.ListenAddr)
+	addr, err := net.ResolveUDPAddr("udp", c.ListenAddr)
 	if err != nil {
-		return fmt.Errorf("invalid listen_addr provided: %w", err)
+		return fmt.Errorf("invalid udp listen_addr: %w", err)
 	}
 
 	c.sock, err = net.ListenUDP("udp", addr)
 	if err != nil {
-		return fmt.Errorf("unable to use provided listen_addr: %w", err)
+		return fmt.Errorf("unable to use udp listen_addr: %w", err)
 	}
 
 	err = c.sock.SetReadBuffer(int(c.BufferUDP)) //nolint:gosec

pkg/httpserver/config.go

@@ -24,7 +24,8 @@ type Config struct {
 	TLSKeyPath        string        `toml:"tls_key_path"        xml:"tls_key_path"`
 }
 
-func defaultConfig() *Config {
+// DefaultConfig returns the default configuration for the HTTP server.
+func DefaultConfig() *Config {
 	return &Config{
 		ListenAddr:        DefaultListenAddr,
 		ReadTimeout:       DefaultReadTimeout,
@@ -35,24 +36,29 @@ func defaultConfig() *Config {
 	}
 }
 
-func validateConfig(config *Config) {
-	if config.ReadTimeout <= 0 {
-		config.ReadTimeout = DefaultReadTimeout
+// Setup fills empty listen address and any zero timeouts or limits so logs and runtime match.
+func (c *Config) Setup() {
+	if c.ListenAddr == "" {
+		c.ListenAddr = DefaultListenAddr
 	}
 
-	if config.ReadHeaderTimeout <= 0 {
-		config.ReadHeaderTimeout = DefaultReadHeaderTimeout
+	if c.ReadTimeout <= 0 {
+		c.ReadTimeout = DefaultReadTimeout
 	}
 
-	if config.WriteTimeout <= 0 {
-		config.WriteTimeout = DefaultWriteTimeout
+	if c.ReadHeaderTimeout <= 0 {
+		c.ReadHeaderTimeout = DefaultReadHeaderTimeout
 	}
 
-	if config.IdleTimeout <= 0 {
-		config.IdleTimeout = DefaultIdleTimeout
+	if c.WriteTimeout <= 0 {
+		c.WriteTimeout = DefaultWriteTimeout
 	}
 
-	if config.MaxHeaderBytes <= 0 {
-		config.MaxHeaderBytes = DefaultMaxHeaderBytes
+	if c.IdleTimeout <= 0 {
+		c.IdleTimeout = DefaultIdleTimeout
+	}
+
+	if c.MaxHeaderBytes <= 0 {
+		c.MaxHeaderBytes = DefaultMaxHeaderBytes
 	}
 }

pkg/httpserver/server.go

@@ -23,10 +23,10 @@ type Server struct {
 // to the same mux (e.g. mux.Handle("/api/", apiHandler)). /metrics is always registered.
 func New(config *Config, register func(mux *http.ServeMux)) *Server {
 	if config == nil {
-		config = defaultConfig()
+		config = DefaultConfig()
 	}
 
-	validateConfig(config)
+	config.Setup()
 
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())

pkg/willow/willow.go

@@ -12,7 +12,7 @@ import (
 
 // Defaults for the module.
 const (
-	DefaultFileSysBuffer = 1024 * 10
+	DefaultFileSysBuffer = 1 << 15 // 32 thousand
 	DefaultFlushInterval = 16 * time.Second
 )