Merge pull request #58 from d-Rickyy-b/reduce-memory-footprint

feat: make buffer sizes configurable

Author: d-Rickyy-b

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- New configuration for buffer sizes (#58)
 ### Changed
 ### Fixed
 ### Docs
@@ -17,9 +18,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support for non-browsers by implementing server initiated heartbeats (#39)
 - Start new ct-watchers as new ct logs become available (#42)
 - More logging to document currently watched logs (03d878e)
+ 
 ### Changed
 - Changed log output to be better grepable (5c055cc)
 - Update ct log update interval to once per hour instead of once per 6 hours as previously (9b6e77d)
+
 ### Fixed
 - Fixed a possible race condition when accessing metrics
 

config.sample.yaml

@@ -28,3 +28,12 @@ general:
     - url: https://dodo.ct.comodo.com/
       operator: "Comodo"
       description: "Comodo Dodo"
+  # To optimize the performance of the server, you can overwrite the size of different buffers
+  # For low CPU, low memory machines, you should reduce the buffer sizes to save memory in case the CPU is maxed.
+  buffer_sizes:
+    # Buffer for each websocket connection
+    websocket: 300
+    # Buffer for each CT log connection
+    ctlog: 1000
+    # Combined buffer for the broadcast manager
+    broadcastmanager: 10000

internal/certificatetransparency/ct-watcher.go

@@ -238,7 +238,7 @@ func (w *worker) runWorker(ctx context.Context) error {
 		Matcher:     scanner.MatchAll{},
 		PrecertOnly: false,
 		NumWorkers:  1,
-		BufferSize:  1000,
+		BufferSize:  config.AppConfig.General.BufferSizes.CTLog,
 	})
 
 	scanErr := certScanner.Scan(ctx, w.foundCertCallback, w.foundPrecertCallback)

internal/config/config.go

@@ -31,6 +31,12 @@ type LogConfig struct {
 	Description string `yaml:"description"`
 }
 
+type BufferSizes struct {
+	Websocket        int `yaml:"websocket"`
+	CTLog            int `yaml:"ctlog"`
+	BroadcastManager int `yaml:"broadcastmanager"`
+}
+
 type Config struct {
 	Webserver struct {
 		ServerConfig       `yaml:",inline"`
@@ -47,6 +53,7 @@ type Config struct {
 	}
 	General struct {
 		AdditionalLogs []LogConfig `yaml:"additional_logs"`
+		BufferSizes    BufferSizes `yaml:"buffer_sizes"`
 	}
 }
 
@@ -211,5 +218,17 @@ func validateConfig(config *Config) bool {
 
 	config.General.AdditionalLogs = validLogs
 
+	if config.General.BufferSizes.Websocket <= 0 {
+		config.General.BufferSizes.Websocket = 300
+	}
+
+	if config.General.BufferSizes.CTLog <= 0 {
+		config.General.BufferSizes.CTLog = 1000
+	}
+
+	if config.General.BufferSizes.BroadcastManager <= 0 {
+		config.General.BufferSizes.BroadcastManager = 10000
+	}
+
 	return true
 }

internal/web/server.go

@@ -168,7 +168,7 @@ func upgradeConnection(w http.ResponseWriter, r *http.Request) (*websocket.Conn,
 
 // setupClient initializes a client struct and starts the broadcastHandler and websocket listener.
 func setupClient(connection *websocket.Conn, subscriptionType SubscriptionType, name string) {
-	c := newClient(connection, subscriptionType, name, 300)
+	c := newClient(connection, subscriptionType, name, config.AppConfig.General.BufferSizes.Websocket)
 	go c.broadcastHandler()
 	go c.listenWebsocket()
 
@@ -275,7 +275,7 @@ func NewWebsocketServer(networkIf string, port int, certPath, keyPath string) *W
 	setupWebsocketRoutes(server.routes)
 	server.initServer()
 
-	ClientHandler.Broadcast = make(chan models.Entry, 10_000)
+	ClientHandler.Broadcast = make(chan models.Entry, config.AppConfig.General.BufferSizes.BroadcastManager)
 	go ClientHandler.broadcaster()
 
 	return server