Southbound observation has TTL set according to which observation is about to expire

Possible to add safety margin to outgoing TTL

Author: zluudg

Makefile

@@ -1,17 +1,8 @@
-NAME:=observation-encoder
-
-#######################################
-# VERSION SOURCE OF TRUTH FOR PROJECT #
-#######################################
-VERSION:=0.0.0
-
-OUT:=./out
-DEFAULT_INSTALLDIR:=/usr/bin
-INSTALL:=install -p -m 0755
-COMMIT:=$$(cat COMMIT 2> /dev/null || git describe --dirty=+WiP --always 2> /dev/null)
-
-.PHONY: build install clean fmt vet coverage
+VERSION:=$$(cat ./VERSION)
+COMMIT:=$$(git describe --dirty=+WiP --always 2> /dev/null || echo "no-vcs")
+OUT:=$$(pwd)/out
 
+.PHONY:
 
 all: build
 
@@ -21,12 +12,6 @@ build: outdir
 outdir:
 	-mkdir -p $(OUT)
 
-clean:
-	-rm -rf $(OUT)
-
-install:
-	test -z "$(DESTDIR)" && $(INSTALL) $(OUT)/$(PROG) $(DEFAULT_INSTALLDIR) || $(INSTALL) $(OUT)/$(PROG) $(DESTDIR)$(prefix)/bin/
-
 test:
 	go test ./...
 
@@ -40,10 +25,8 @@ fmt:
 vet:
 	go vet ./...
 
-tarball: outdir
-	@test ! -f COMMIT || (echo "Trying to make tarball from extracted tarball?" && false)
-	@test ! -z $(COMMIT) || (echo "Not tracked by git?" && false)
-	@test -z $$(git status --porcelain) || (echo "won't make tarball from dirty history" && false)
-	echo "$(COMMIT)" > $(OUT)/COMMIT
-	git archive --format=tar.gz --prefix=$(NAME)/ -o $(OUT)/$(NAME)-$(VERSION).tar.gz --add-file $(OUT)/COMMIT HEAD
-	cd $(OUT) && sha256sum -b $(NAME)-$(VERSION).tar.gz > $(NAME)-$(VERSION).sha256.txt
+clean:
+	-rm -rf $(OUT)
+
+ko:
+	ko build -L -B ./cmd/observation-encoder

README.md

@@ -19,20 +19,29 @@ observation-encoder -config /path/to/config/file
 # Sample config file
 
 ```toml
+debug = true
+
 [cert]
-interval = 100
+interval = 86400
 cert_dir = "/path/to/certs/dir"
 
 [api]
-active = true
+active = false
 address = "127.0.0.1"
 port = "10001"
 
 [nats]
 url = "nats://127.0.0.1:4222"
-subject_prefix = "observations"
-subject_southbound = "test.subject"
-ttl = 3600
+subject_southbound = "test.observations.down.edge"
+observation_subject_prefix = "test.observations"
+
+[[nats.buckets]]
+name = "foo"
+ttl = 60
+
+[[nats.buckets]]
+name = "bar"
+ttl = 120
 
 [libtapir]
 debug = true

VERSION

@@ -0,0 +1 @@
+0.0.0

internal/app/app.go

@@ -13,14 +13,16 @@ const c_N_HANDLERS = 3
 const c_NATS_DELIM = common.NATS_DELIM
 
 type Conf struct {
-	Log            common.Logger
 	Debug          bool `toml:"debug"`
+	TtlMargin      int  `toml:"ttl_margin"`
+	Log            common.Logger
 	NatsHandle     nats
 	LibtapirHandle libtapir
 }
 
 type appHandle struct {
 	id             string
+	ttlMargin      int
 	log            common.Logger
 	natsHandle     nats
 	libtapirHandle libtapir
@@ -39,13 +41,13 @@ type job struct {
 type nats interface {
 	WatchObservations(context.Context) (<-chan common.NatsMsg, error)
 	RemovePrefix(string) string
-	GetObservations(context.Context, string) (uint32, error)
+	GetObservations(context.Context, string) (uint32, int, error)
 	SendSouthboundObservation(string) error
 	Shutdown() error
 }
 
 type libtapir interface {
-	GenerateObservationMsg(string, uint32) (string, error) // TODO set ttl?
+	GenerateObservationMsg(string, uint32, int) (string, error) // TODO set ttl?
 }
 
 func Create(conf Conf) (*appHandle, error) {
@@ -63,8 +65,14 @@ func Create(conf Conf) (*appHandle, error) {
 		return nil, common.ErrBadHandle
 	}
 
+	// TODO what is a reasonale safety margin (in seconds) for adding to outgoing TTL?
+	if conf.TtlMargin < 0 || conf.TtlMargin > 3600 {
+		return nil, common.ErrBadParam
+	}
+
 	a.log = conf.Log
 	a.id = "main app"
+	a.ttlMargin = conf.TtlMargin
 	a.natsHandle = conf.NatsHandle
 	a.libtapirHandle = conf.LibtapirHandle
 
@@ -143,15 +151,20 @@ func (a *appHandle) handleJob(ctx context.Context, j job) {
 
 	a.log.Debug("Extracted domain '%s'", domain)
 
-	obs, err := a.natsHandle.GetObservations(ctx, domain)
+	obs, ttl, err := a.natsHandle.GetObservations(ctx, domain)
 	if err != nil {
 		a.log.Error("Could not get observations for %s: %s", domain, err)
 		return
 	}
 
+	if obs == 0 || ttl == 0 {
+		a.log.Debug("Observation for %s has value %d and ttl %d, won't bother sending", domain, obs, ttl)
+		return
+	}
+
 	a.log.Debug("%s has observation vector %d", domain, obs)
 
-	obsJSON, err := a.libtapirHandle.GenerateObservationMsg(domain, obs)
+	obsJSON, err := a.libtapirHandle.GenerateObservationMsg(domain, obs, int(ttl)+a.ttlMargin)
 	if err != nil {
 		a.log.Error("Couldn't generate JSON observation: %s", err)
 		return

internal/libtapir/libtapir.go

@@ -29,18 +29,18 @@ func Create(conf Conf) (*libtapir, error) {
 	return lt, nil
 }
 
-func (lt *libtapir) GenerateObservationMsg(domainStr string, flags uint32) (string, error) {
+func (lt *libtapir) GenerateObservationMsg(domainStr string, flags uint32, ttl int) (string, error) {
 	domain := tapir.Domain{
 		Name:         domainStr,
 		TimeAdded:    time.Now(),
-		TTL:          3600,
+		TTL:          ttl,
 		TagMask:      tapir.TagMask(flags),
 		ExtendedTags: []string{},
 	}
 
 	tapirMsg := tapir.TapirMsg{
 		SrcName:   "dns-tapir",
-		Creator:   "tapir-analyse-new-qname",
+		Creator:   "observation-encoder",
 		MsgType:   "observation",
 		ListType:  "doubtlist",
 		Added:     []tapir.Domain{domain},