bugfixes, cleanup, and add decay for max cache

Author: crimist

config/config.go

@@ -19,7 +19,11 @@ const (
 type Configuration struct {
 	LogLevel string
 	Cache    string
-	Stats    struct {
+	Maximums struct {
+		Interval time.Duration
+		Decay    int
+	}
+	Stats struct {
 		General  bool
 		IP       bool
 		Interval time.Duration
@@ -99,6 +103,13 @@ func (conf *Configuration) validate() error {
 		}
 	}
 
+	if conf.Maximums.Interval < 0 {
+		return errors.New("invalid configuration: Maximums.UpdateInterval must be >= 0")
+	}
+	if conf.Maximums.Decay < 0 {
+		return errors.New("invalid configuration: Maximums.DecayHalfLifeUpdates must be >= 0")
+	}
+
 	if conf.HTTP.Serve != "" {
 		stat, err = os.Stat(conf.HTTP.Serve)
 

config/embedded/trakx.yaml

@@ -9,6 +9,13 @@ loglevel: info
 # backup files. Leave blank for default of XDG_CACHE_HOME/trakx
 cache:
 
+# Record map size maxima for preallocation between runs.
+maximums:
+  # How often to update stored maxima. Set to 0 to disable maximums tracking.
+  update_interval: 5m
+  # Decay half-life in update ticks (larger = slower decay). Set to 0 to disable decay.
+  decay_half_life_updates: 2000
+
 # Trakx can track statistic such as the number of announces, seeds, unique ips, etc.
 # They are published via go's expvar standard at '/stats'.
 stats:

daemon/run.go

@@ -56,9 +56,14 @@ func RunWithOptions(conf *config.Configuration, opts RunOptions) {
 	baseIntervalSeconds := uint(conf.Announce.Base / time.Second)
 	fuzzIntervalSeconds := uint(conf.Announce.Fuzz / time.Second)
 
-	maxStore, err := maxcache.New(conf.Cache)
-	if err != nil {
-		zap.L().Warn("Failed to load maximums cache", zap.Error(err))
+	var maxStore *maxcache.Store
+	if conf.Maximums.Interval > 0 {
+		maxStore, err = maxcache.New(conf.Cache, maxcache.Options{
+			DecayHalfLifeUpdates: conf.Maximums.Decay,
+		})
+		if err != nil {
+			zap.L().Warn("Failed to load maximums cache", zap.Error(err))
+		}
 	}
 
 	ipCollectorSize := 0
@@ -202,6 +207,8 @@ func RunWithOptions(conf *config.Configuration, opts RunOptions) {
 			return
 		}
 
+		zap.L().Info("Updating maximums cache")
+
 		if conf.Stats.General && conf.Stats.IP {
 			ips := collector.IPs().Total()
 			zap.L().Debug("Setting maximum", zap.Int("ips", ips))
@@ -212,13 +219,13 @@ func RunWithOptions(conf *config.Configuration, opts RunOptions) {
 
 		torrents := db.Torrents()
 		zap.L().Debug("Setting maximum", zap.Int("torrents", torrents))
-		if err := maxStore.Update(maxKeyDBTorrents, db.Torrents()); err != nil {
+		if err := maxStore.Update(maxKeyDBTorrents, torrents); err != nil {
 			zap.L().Warn("Failed to update maximums cache", zap.String("key", maxKeyDBTorrents), zap.Error(err))
 		}
 	}
 
 	if maxStore != nil {
-		go utils.RunOn(maxcache.UpdateFrequency, updateMaximums)
+		go utils.RunOn(conf.Maximums.Interval, updateMaximums)
 	}
 
 	go signalHandler(trackers, func() error {

internal/maxcache/store.go

@@ -4,44 +4,52 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math"
 	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"sync"
-	"time"
 )
 
 const (
-	fileSuffix = ".maximum"
+	fileSuffix = ".json"
 	dirName    = "maximums"
-
-	UpdateFrequency = time.Minute
 )
 
 var ErrInvalidKey = errors.New("invalid maximums key")
 
 type Store struct {
-	dir    string
-	mu     sync.Mutex
-	values map[string]int
+	dir         string
+	decayFactor float64
+	mu          sync.Mutex
+	values      map[string]int
 }
 
 type record struct {
 	Max int `json:"max"`
 }
 
+type Options struct {
+	DecayHalfLifeUpdates int
+}
+
 // New creates a maximums store rooted at cacheDir/maximums.
 // It loads existing maxima if present.
-func New(cacheDir string) (*Store, error) {
+func New(cacheDir string, opts Options) (*Store, error) {
 	dir := filepath.Join(cacheDir, dirName)
 	if err := os.MkdirAll(dir, 0o700); err != nil {
 		return nil, err
 	}
 
+	decayFactor := 1.0
+	if opts.DecayHalfLifeUpdates > 0 {
+		decayFactor = math.Pow(0.5, 1.0/float64(opts.DecayHalfLifeUpdates))
+	}
+
 	store := &Store{
-		dir:    dir,
-		values: make(map[string]int),
+		dir:         dir,
+		decayFactor: decayFactor,
+		values:      make(map[string]int),
 	}
 
 	if err := store.load(); err != nil {
@@ -75,12 +83,20 @@ func (s *Store) Update(key string, value int) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	if value <= s.values[key] {
+	current := s.values[key]
+	next := current
+	if s.decayFactor < 1 && current > 0 {
+		next = int(math.Floor(float64(current) * s.decayFactor))
+	}
+	if value > next {
+		next = value
+	}
+	if next == current {
 		return nil
 	}
 
-	s.values[key] = value
-	return s.writeLocked(key, value)
+	s.values[key] = next
+	return s.writeLocked(key, next)
 }
 
 func (s *Store) load() error {
@@ -172,15 +188,10 @@ func (s *Store) writeLocked(key string, value int) error {
 
 func parseValue(data []byte) (int, error) {
 	var rec record
-	if err := json.Unmarshal(data, &rec); err == nil {
-		return rec.Max, nil
-	}
-
-	value, err := strconv.Atoi(strings.TrimSpace(string(data)))
-	if err != nil {
+	if err := json.Unmarshal(data, &rec); err != nil {
 		return 0, err
 	}
-	return value, nil
+	return rec.Max, nil
 }
 
 func validateKey(key string) error {