Skip to content

Commit 82b27ef

Browse files
zluudgzluudg
authored
Leon/dev (#1)
* Skeleton for handling incoming observations * Control debug logs for different components * Aggregate observations based on lookup of NATS subject * Keep subject prefix internal to nats, set bucket name, ttl and prefix in conf file * Generate observation JSON from flags * go fmt * Actual publish * go fmt * some GH workflow action pipeline stuff * dependabot integration * make sure go version 1.26.6 is used * prevent hot loop in MAIN_APP_LOOP * avoid duplicating observation encodings * fix fragile manipulation of NATS subjects * actually expire keys. skip history of key/val during connect * Add bounds check before checking observation flag type * go fmt * fixed typo that caused compilation failure * Do not accept negative TTL values in NATS * More robust manipulation of NATS keys/subjects. Also with tests! * go fmt
1 parent 98cae08 commit 82b27ef

File tree

18 files changed

+964
-92
lines changed

18 files changed

+964
-92
lines changed

‎.github/dependabot.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# To get started with Dependabot version updates, you'll need to specify which
2+
# package ecosystems to update and where the package manifests are located.
3+
# Please see the documentation for all configuration options:
4+
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
5+
6+
version: 2
7+
updates:
8+
- package-ecosystem: "gomod" # See documentation for possible values
9+
directory: "/" # Location of package manifests
10+
schedule:
11+
interval: "weekly"

‎.github/workflows/base-pipeline.yaml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: Base Pipeline
2+
3+
on:
4+
push:
5+
branches: [ "main" ]
6+
pull_request:
7+
branches: [ "main" ]
8+
9+
env:
10+
GO_VERSION: "1.25.6"
11+
jobs:
12+
format:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout repository
16+
uses: actions/checkout@v4
17+
18+
- name: Setup Go
19+
uses: actions/setup-go@v5.5.0
20+
with:
21+
go-version: ${{ env.GO_VERSION }}
22+
23+
- name: Check formatting
24+
run: |
25+
files=$(gofmt -l ./)
26+
27+
if [ -n "$files" ]; then
28+
while IFS= read -r file; do
29+
echo "::warning file=$file::File is not gofmt formatted"
30+
done <<< "$files"
31+
exit 1
32+
else
33+
echo "✅ All files are properly formatted."
34+
fi
35+
36+
test:
37+
runs-on: ubuntu-latest
38+
steps:
39+
- name: Checkout repository
40+
uses: actions/checkout@v4
41+
42+
- name: Setup Go
43+
uses: actions/setup-go@v5.5.0
44+
with:
45+
go-version: ${{ env.GO_VERSION }}
46+
47+
- name: Build
48+
run: |
49+
make build
50+
51+
- name: Test
52+
run: |
53+
make test

‎.github/workflows/container.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Build container
2+
3+
on:
4+
workflow_run:
5+
workflows:
6+
- "Base Pipeline"
7+
branches:
8+
- "main"
9+
types:
10+
- completed
11+
push:
12+
tags:
13+
- "v*.*.*"
14+
15+
env:
16+
GO_VERSION: "1.25.6"
17+
jobs:
18+
container:
19+
if: ${{ github.event.workflow_run.conclusion == 'success' }}
20+
name: Build and push container
21+
runs-on: ubuntu-latest
22+
permissions:
23+
actions: write
24+
contents: read
25+
packages: write
26+
steps:
27+
- uses: actions/checkout@v4
28+
- uses: actions/setup-go@v5
29+
with:
30+
go-version: ${{ env.GO_VERSION }}
31+
- uses: ko-build/setup-ko@v0.7
32+
- run: ko build --bare ./cmd/observation-encoder

‎.ko.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
defaultPlatforms:
2+
- linux/amd64
3+
- linux/arm64
4+
defaultLdflags:
5+
- -X main.commit={{.Git.FullCommit}}

‎README.md‎

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,25 @@ observation-encoder -config /path/to/config/file
1818

1919
# Sample config file
2020

21-
```json
22-
{
23-
"address": "127.0.0.1",
24-
"port": "10000",
25-
"cert":
26-
{
27-
"interval": 100,
28-
"cert_dir": "/path/to/certificates/dir"
29-
},
30-
"api":
31-
{
32-
"active": true,
33-
"address": "127.0.0.1",
34-
"port": "10001"
35-
}
36-
}
21+
```toml
22+
address = "127.0.0.1"
23+
port = "10000"
24+
25+
[cert]
26+
interval = 100
27+
cert_dir = "/path/to/certs/dir"
28+
29+
[api]
30+
active = true
31+
address = "127.0.0.1"
32+
port = "10001"
33+
34+
[nats]
35+
url = "nats://127.0.0.1:4222"
36+
subject_prefix = "observations"
37+
subject_southbound = "test.subject"
38+
ttl = 3600
39+
40+
[libtapir]
41+
debug = true
3742
```

‎cmd/observation-encoder/main.go‎

Lines changed: 88 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,23 +10,26 @@ import (
1010
"syscall"
1111
"time"
1212

13-
//"github.com/nats-io/nats.go"
1413
"github.com/pelletier/go-toml/v2"
1514

1615
"github.com/dnstapir/observation-encoder/internal/api"
1716
"github.com/dnstapir/observation-encoder/internal/app"
1817
"github.com/dnstapir/observation-encoder/internal/cert"
1918
"github.com/dnstapir/observation-encoder/internal/common"
19+
"github.com/dnstapir/observation-encoder/internal/libtapir"
2020
"github.com/dnstapir/observation-encoder/internal/logger"
21+
"github.com/dnstapir/observation-encoder/internal/nats"
2122
)
2223

2324
var commit = "BAD-BUILD"
2425

2526
type conf struct {
2627
app.Conf
27-
Debug bool `toml:"debug"`
28-
Api api.Conf `toml:"api"`
29-
Cert cert.Conf `toml:"cert"`
28+
Debug bool `toml:"debug"`
29+
Api api.Conf `toml:"api"`
30+
Cert cert.Conf `toml:"cert"`
31+
Nats nats.Conf `toml:"nats"`
32+
Libtapir libtapir.Conf `toml:"libtapir"`
3033
}
3134

3235
func main() {
@@ -87,32 +90,103 @@ func main() {
8790
confDecoder.Decode(&mainConf)
8891
file.Close() // TODO okay to close here while also using defer above?
8992

90-
// TODO create different loggers with different debug settings
93+
debugFlag = debugFlag || mainConf.Debug
94+
95+
/*
96+
******************************************************************
97+
********************** SET UP NATS *******************************
98+
******************************************************************
99+
*/
100+
natslog, err := logger.Create(
101+
logger.Conf{
102+
Debug: debugFlag || mainConf.Nats.Debug,
103+
})
104+
if err != nil {
105+
log.Error("Error creating nats log: %s", err)
106+
}
107+
108+
mainConf.Nats.Log = natslog
109+
natsHandle, err := nats.Create(mainConf.Nats)
110+
if err != nil {
111+
log.Error("Could not create NATS handle: %s", err)
112+
os.Exit(-1)
113+
}
114+
115+
/*
116+
******************************************************************
117+
********************** SET UP LIBTAPIR ***************************
118+
******************************************************************
119+
*/
120+
libtapirlog, err := logger.Create(
121+
logger.Conf{
122+
Debug: debugFlag || mainConf.Libtapir.Debug,
123+
})
124+
if err != nil {
125+
log.Error("Error creating libtapir log: %s", err)
126+
}
127+
128+
mainConf.Libtapir.Log = libtapirlog
129+
libtapirHandle, err := libtapir.Create(mainConf.Libtapir)
130+
if err != nil {
131+
log.Error("Could not create libtapir handle: %s", err)
132+
os.Exit(-1)
133+
}
134+
135+
/*
136+
******************************************************************
137+
********************** SET UP MAIN APP ***************************
138+
******************************************************************
139+
*/
91140
applog, err := logger.Create(
92141
logger.Conf{
93142
Debug: debugFlag || mainConf.Debug,
94143
})
95144
if err != nil {
96145
log.Error("Error creating app log: %s", err)
97-
} else {
98-
applog.Debug("Debug logging enabled")
99146
}
100147

101148
mainConf.Log = applog
149+
mainConf.NatsHandle = natsHandle
150+
mainConf.LibtapirHandle = libtapirHandle
102151
appHandle, err := app.Create(mainConf.Conf)
103152
if err != nil {
104153
log.Error("Error creating application: '%s'", err)
105154
os.Exit(-1)
106155
}
107156

108-
mainConf.Cert.Log = applog
157+
/*
158+
******************************************************************
159+
********************** SET UP CERT HANDLER ***********************
160+
******************************************************************
161+
*/
162+
certlog, err := logger.Create(
163+
logger.Conf{
164+
Debug: debugFlag || mainConf.Cert.Debug,
165+
})
166+
if err != nil {
167+
log.Error("Error creating cert log: %s", err)
168+
}
169+
170+
mainConf.Cert.Log = certlog
109171
certHandle, err := cert.Create(mainConf.Cert)
110172
if err != nil {
111173
log.Error("Error creating cert manager: '%s'", err)
112174
os.Exit(-1)
113175
}
114176

115-
mainConf.Api.Log = applog
177+
/*
178+
******************************************************************
179+
********************** SET UP API ********************************
180+
******************************************************************
181+
*/
182+
apilog, err := logger.Create(
183+
logger.Conf{
184+
Debug: debugFlag || mainConf.Api.Debug,
185+
})
186+
if err != nil {
187+
log.Error("Error creating API log: %s", err)
188+
}
189+
mainConf.Api.Log = apilog
116190
mainConf.Api.App = appHandle
117191
mainConf.Api.Certs = certHandle
118192
apiHandle, err := api.Create(mainConf.Api)
@@ -121,6 +195,11 @@ func main() {
121195
os.Exit(-1)
122196
}
123197

198+
/*
199+
******************************************************************
200+
********************** START RUNNING STUFF ***********************
201+
******************************************************************
202+
*/
124203
sigChan := make(chan os.Signal, 1)
125204
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
126205
defer close(sigChan)

‎go.mod‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,50 @@
11
module github.com/dnstapir/observation-encoder
22

3-
go 1.24.6
3+
go 1.25.6
44

5-
require github.com/pelletier/go-toml/v2 v2.2.4
5+
require (
6+
github.com/dnstapir/tapir v0.0.0-20260115113810-71a904f35f68
7+
github.com/nats-io/nats.go v1.48.0
8+
github.com/pelletier/go-toml/v2 v2.2.4
9+
)
10+
11+
require (
12+
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
13+
github.com/eclipse/paho.golang v0.23.0 // indirect
14+
github.com/fsnotify/fsnotify v1.9.0 // indirect
15+
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
16+
github.com/goccy/go-json v0.10.5 // indirect
17+
github.com/gookit/goutil v0.7.3 // indirect
18+
github.com/gorilla/websocket v1.5.3 // indirect
19+
github.com/klauspost/compress v1.18.4 // indirect
20+
github.com/lestrrat-go/blackmagic v1.0.4 // indirect
21+
github.com/lestrrat-go/httpcc v1.0.1 // indirect
22+
github.com/lestrrat-go/httprc v1.0.6 // indirect
23+
github.com/lestrrat-go/iter v1.0.2 // indirect
24+
github.com/lestrrat-go/jwx/v2 v2.1.6 // indirect
25+
github.com/lestrrat-go/option v1.0.1 // indirect
26+
github.com/miekg/dns v1.1.72 // indirect
27+
github.com/nats-io/nkeys v0.4.15 // indirect
28+
github.com/nats-io/nuid v1.0.1 // indirect
29+
github.com/rogpeppe/go-internal v1.14.1 // indirect
30+
github.com/ryanuber/columnize v2.1.2+incompatible // indirect
31+
github.com/sagikazarmark/locafero v0.12.0 // indirect
32+
github.com/segmentio/asm v1.2.1 // indirect
33+
github.com/smhanov/dawg v0.0.0-20220118194912-66057bdbf2e3 // indirect
34+
github.com/spf13/afero v1.15.0 // indirect
35+
github.com/spf13/cast v1.10.0 // indirect
36+
github.com/spf13/pflag v1.0.10 // indirect
37+
github.com/spf13/viper v1.21.0 // indirect
38+
github.com/subosito/gotenv v1.6.0 // indirect
39+
go.yaml.in/yaml/v3 v3.0.4 // indirect
40+
golang.org/x/crypto v0.48.0 // indirect
41+
golang.org/x/exp v0.0.0-20260209203927-2842357ff358 // indirect
42+
golang.org/x/mod v0.33.0 // indirect
43+
golang.org/x/net v0.50.0 // indirect
44+
golang.org/x/sync v0.19.0 // indirect
45+
golang.org/x/sys v0.41.0 // indirect
46+
golang.org/x/term v0.40.0 // indirect
47+
golang.org/x/text v0.34.0 // indirect
48+
golang.org/x/tools v0.42.0 // indirect
49+
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
50+
)

0 commit comments

Comments
 (0)