Merge pull request #18 from theopenlane/feat-configandintel

fix up the config setup, add new pipeline

Author: matoszz

.buildkite/pipeline.yaml

@@ -76,6 +76,24 @@ steps:
                 - "CGO_ENABLED=0"
                 - "GOOS=linux"
                 - "GOFLAGS=-buildvcs=false"
+      - label: ":golang: build release"
+        key: "gobuild-release"
+        cancel_on_build_failing: true
+        artifact_paths: "bin/${APP_NAME}"
+        if: build.tag != null
+        plugins:
+          - docker#v5.13.0:
+              image: "ghcr.io/theopenlane/build-image:latest"
+              always-pull: true
+              command: ["task", "go:build"]
+              propagate-environment: true
+              volumes:
+                - "$GOCACHE:$GOCACHE"
+                - "$GOMODCACHE:$GOMODCACHE"
+              environment:
+                - "CGO_ENABLED=0"
+                - "GOOS=linux"
+                - "GOFLAGS=-buildvcs=false"
   - group: ":docker: Docker Image"
     depends_on: "go-build"
     key: "docker-build"
@@ -117,7 +135,7 @@ steps:
         plugins:
           - artifacts#v1.9.4:
               download: "bin/${APP_NAME}"
-              step: "gobuild"
+              step: "gobuild-release"
           - docker-login#v3.0.0:
               username: openlane-bender
               password-env: SECRET_GHCR_PUBLISH_TOKEN
@@ -140,17 +158,87 @@ steps:
         depends_on: "docker-build-and-tag"
         if: build.tag != null
         plugins:
-          - docker#v5.13.0:
-              image: "ghcr.io/theopenlane/build-image:latest"
-              always-pull: true
-              command: ["sh", "-c", "apk add --no-cache bash >/dev/null 2>&1 && bash .buildkite/image-tag-automation.sh"]
-              propagate-environment: true
-              volumes:
-                - "$GOCACHE:$GOCACHE"
-                - "$GOMODCACHE:$GOMODCACHE"
-              environment:
-                - "GITHUB_TOKEN"
-                - "GITHUB_SSH_PRIVATE_KEY"
-                - "HELM_CHART_REPO"
-                - "HELM_CHART_PATH"
-                - "BUILDKITE_BUILD_CHECKOUT_PATH=/workdir"
+          - theopenlane/git#v0.1.5:
+              execute-phase: command
+              repository: "${HELM_CHART_REPO}"
+              branch: "sleuth-${BUILDKITE_TAG}"
+              preset: helm-sync
+              chart-name: sleuth
+              chart-version-env: BUILDKITE_TAG
+              auth:
+                mode: https-token
+                token-env: GITHUB_TOKEN
+              user:
+                name: theopenlane-bender
+                email: bender@theopenlane.io
+              sync:
+                - to: values.yaml
+                  type: merge-yaml
+                  merge-target-path: .sleuth.image.tag
+                  merge-source-env: BUILDKITE_TAG
+              commit-message: "chore: bump sleuth to ${BUILDKITE_TAG}"
+              pr:
+                enabled: true
+                title: "chore: bump sleuth to ${BUILDKITE_TAG}"
+  - group: ":helm: Helm Config"
+    key: "helm-config"
+    steps:
+      - label: ":helm: draft helm config PR"
+        key: "helm_config_draft_pr"
+        if: build.pull_request.id != null
+        if_changed:
+          - config/helm-values.yaml
+        plugins:
+          - theopenlane/git#v0.1.5:
+              execute-phase: command
+              repository: "${HELM_CHART_REPO}"
+              branch: "sleuth-config-${BUILDKITE_PULL_REQUEST}"
+              preset: helm-sync
+              chart-name: sleuth
+              auth:
+                mode: https-token
+                token-env: GITHUB_TOKEN
+              user:
+                name: theopenlane-bender
+                email: bender@theopenlane.io
+              sync:
+                - from: helm-values.yaml
+                  to: values.yaml
+                  type: merge-yaml
+                  merge-target-path: .sleuth.sleuthConfiguration
+                  merge-source-path: .sleuthConfiguration
+              commit-message: "chore: update sleuth helm config from build #{{BUILD_NUMBER}}"
+              pr:
+                enabled: true
+                draft: true
+                title: "chore: update sleuth helm config (PR #{{SOURCE_PR_NUMBER}})"
+                comment-on-source-pr: true
+                source-comment: "Downstream helm chart draft PR: {{TARGET_PR_URL}}"
+      - label: ":helm: update helm config"
+        key: "helm_config_merge"
+        if: build.branch == "main"
+        if_changed:
+          - config/helm-values.yaml
+        plugins:
+          - theopenlane/git#v0.1.5:
+              execute-phase: command
+              repository: "${HELM_CHART_REPO}"
+              branch-prefix: "sleuth-config"
+              preset: helm-sync
+              chart-name: sleuth
+              auth:
+                mode: https-token
+                token-env: GITHUB_TOKEN
+              user:
+                name: theopenlane-bender
+                email: bender@theopenlane.io
+              sync:
+                - from: helm-values.yaml
+                  to: values.yaml
+                  type: merge-yaml
+                  merge-target-path: .sleuth.sleuthConfiguration
+                  merge-source-path: .sleuthConfiguration
+              commit-message: "chore: update sleuth helm config from build #{{BUILD_NUMBER}}"
+              pr:
+                enabled: true
+                title: "chore: update sleuth helm config (Build #{{BUILD_NUMBER}})"

.gitignore

@@ -41,5 +41,5 @@ Thumbs.db
 *.log
 .tasktmp/
 .claude
-*.config*yaml
 config/.config.yaml
+data/

Taskfile.yaml

@@ -97,6 +97,11 @@ tasks:
       - task: specs:generate
       - task: client:generate
 
+  config:ci:
+    desc: Regenerate config schema and example files for CI
+    cmds:
+      - task: config:schema
+
   config:init:
     desc: Create a local .config.yaml from the example (run config:generate first)
     cmds:

cmd/serve.go

@@ -103,22 +103,11 @@ func serve(ctx context.Context) error {
 
 // setupIntel initializes the threat intelligence manager from config
 func setupIntel(ctx context.Context, cfg *config.Config) (*intel.Manager, error) {
-	feedCfg, err := intel.LoadFeedConfig(cfg.Intel.FeedConfig)
-	if err != nil {
-		return nil, fmt.Errorf("loading feed config from %s: %w", cfg.Intel.FeedConfig, err)
-	}
-
-	intelClient := &http.Client{Timeout: cfg.Intel.RequestTimeout}
-
 	emailAuthAnalyzer := emailauth.NewAnalyzer()
 	rdapClient := rdap.NewClient()
 
 	manager, err := intel.NewManager(
-		feedCfg,
-		intel.WithStorageDir(cfg.Intel.StorageDir),
-		intel.WithHTTPClient(intelClient),
-		intel.WithResolverTimeout(cfg.Intel.ResolverTimeout),
-		intel.WithDNSCacheTTL(cfg.Intel.DNSCacheTTL),
+		cfg.Intel,
 		intel.WithEmailAuthAnalyzer(emailAuthAnalyzer),
 		intel.WithRDAPAnalyzer(rdapClient),
 	)
@@ -145,16 +134,7 @@ func setupIntel(ctx context.Context, cfg *config.Config) (*intel.Manager, error)
 
 // setupScanner initializes the domain scanner from config
 func setupScanner(cfg *config.Config) (*scanner.Scanner, error) {
-	opts := []scanner.ScanOption{
-		scanner.WithVerbose(cfg.Scanner.Verbose),
-		scanner.WithMaxSubdomains(cfg.Scanner.MaxSubdomains),
-	}
-
-	if len(cfg.Scanner.NucleiSeverity) > 0 {
-		opts = append(opts, scanner.WithNucleiSeverity(cfg.Scanner.NucleiSeverity))
-	}
-
-	return scanner.New(opts...)
+	return scanner.New(cfg.Scanner)
 }
 
 // setupCloudflare initializes the Cloudflare client from config, returning nil when unconfigured
@@ -166,11 +146,7 @@ func setupCloudflare(cfg *config.Config) *cloudflare.Client {
 		return nil
 	}
 
-	client, err := cloudflare.New(
-		cfg.Cloudflare.AccountID,
-		cfg.Cloudflare.APIToken,
-		cloudflare.WithHTTPClient(&http.Client{Timeout: cfg.Cloudflare.RequestTimeout}),
-	)
+	client, err := cloudflare.New(cfg.Cloudflare)
 	if err != nil {
 		log.Warn().Err(err).Msg("failed to initialize cloudflare client")
 		return nil
@@ -195,10 +171,7 @@ func setupSlack(cfg *config.Config) *slack.Client {
 		return nil
 	}
 
-	client, err := slack.New(
-		cfg.Slack.WebhookURL,
-		slack.WithHTTPClient(&http.Client{Timeout: cfg.Slack.RequestTimeout}),
-	)
+	client, err := slack.New(cfg.Slack)
 	if err != nil {
 		log.Warn().Err(err).Msg("failed to initialize slack client")
 		return nil

config/config.example.yaml

@@ -15,6 +15,8 @@ scanner:
         - critical
         - high
         - medium
+        - low
+        - info
     timeout: 2m0s
     verbose: false
 server:

config/config.go

@@ -10,6 +10,10 @@ import (
 	"github.com/knadh/koanf/v2"
 	"github.com/mcuadros/go-defaults"
 	"github.com/rs/zerolog/log"
+	"github.com/theopenlane/sleuth/internal/cloudflare"
+	"github.com/theopenlane/sleuth/internal/intel"
+	"github.com/theopenlane/sleuth/internal/scanner"
+	"github.com/theopenlane/sleuth/internal/slack"
 )
 
 // envVarPrefix is the prefix for environment variables used to override config values
@@ -23,13 +27,13 @@ type Config struct {
 	// Server contains HTTP server settings
 	Server Server `json:"server" koanf:"server"`
 	// Scanner contains domain scanner settings
-	Scanner Scanner `json:"scanner" koanf:"scanner"`
+	Scanner scanner.Config `json:"scanner" koanf:"scanner"`
 	// Intel contains threat intelligence settings
-	Intel Intel `json:"intel" koanf:"intel"`
+	Intel intel.Config `json:"intel" koanf:"intel"`
 	// Cloudflare contains Cloudflare API settings
-	Cloudflare Cloudflare `json:"cloudflare" koanf:"cloudflare"`
+	Cloudflare cloudflare.Config `json:"cloudflare" koanf:"cloudflare"`
 	// Slack contains Slack webhook settings
-	Slack Slack `json:"slack" koanf:"slack"`
+	Slack slack.Config `json:"slack" koanf:"slack"`
 }
 
 // Server contains HTTP server configuration
@@ -50,52 +54,6 @@ type Server struct {
 	MaxBodySize int64 `json:"maxbodysize" koanf:"maxbodysize" default:"102400"`
 }
 
-// Scanner contains domain scanner configuration
-type Scanner struct {
-	// Timeout is the maximum duration for a scan operation
-	Timeout time.Duration `json:"timeout" koanf:"timeout" default:"120s"`
-	// MaxSubdomains is the maximum number of subdomains to enumerate
-	MaxSubdomains int `json:"maxsubdomains" koanf:"maxsubdomains" default:"50"`
-	// NucleiSeverity is the list of nuclei severity levels to scan for
-	NucleiSeverity []string `json:"nucleiseverity" koanf:"nucleiseverity"`
-	// Verbose enables verbose scanner output
-	Verbose bool `json:"verbose" koanf:"verbose" default:"false"`
-}
-
-// Intel contains threat intelligence configuration
-type Intel struct {
-	// FeedConfig is the path to the feed configuration JSON file
-	FeedConfig string `json:"feedconfig" koanf:"feedconfig" default:"config/feed_config.json"`
-	// StorageDir is the directory for storing downloaded feed data
-	StorageDir string `json:"storagedir" koanf:"storagedir" default:"data/intel"`
-	// AutoHydrate enables automatic feed hydration on startup
-	AutoHydrate bool `json:"autohydrate" koanf:"autohydrate" default:"false"`
-	// RequestTimeout is the timeout for individual feed download requests
-	RequestTimeout time.Duration `json:"requesttimeout" koanf:"requesttimeout" default:"90s"`
-	// ResolverTimeout is the timeout for DNS lookups during scoring
-	ResolverTimeout time.Duration `json:"resolvertimeout" koanf:"resolvertimeout" default:"10s"`
-	// DNSCacheTTL is the TTL for cached DNS responses
-	DNSCacheTTL time.Duration `json:"dnscachettl" koanf:"dnscachettl" default:"5m"`
-}
-
-// Cloudflare contains Cloudflare API configuration
-type Cloudflare struct {
-	// AccountID is the Cloudflare account identifier
-	AccountID string `json:"accountid" koanf:"accountid"`
-	// APIToken is the bearer token for Cloudflare API authentication
-	APIToken string `json:"apitoken" koanf:"apitoken" sensitive:"true"`
-	// RequestTimeout is the timeout for Cloudflare API requests
-	RequestTimeout time.Duration `json:"requesttimeout" koanf:"requesttimeout" default:"30s"`
-}
-
-// Slack contains Slack webhook configuration
-type Slack struct {
-	// WebhookURL is the Slack incoming webhook URL
-	WebhookURL string `json:"webhookurl" koanf:"webhookurl" sensitive:"true"`
-	// RequestTimeout is the timeout for Slack webhook requests
-	RequestTimeout time.Duration `json:"requesttimeout" koanf:"requesttimeout" default:"10s"`
-}
-
 // Option configures the Config
 type Option func(*Config)
 

config/config_test.go

@@ -42,7 +42,7 @@ func TestNew(t *testing.T) {
 	}
 
 	if cfg.Intel.StorageDir != "data/intel" {
-		t.Errorf("expected default storage dir, got %s", cfg.Intel.StorageDir)
+		t.Errorf("expected default storage dir data/intel, got %s", cfg.Intel.StorageDir)
 	}
 
 	if cfg.Cloudflare.RequestTimeout != 30*time.Second {

config/helm-values.yaml

@@ -0,0 +1,26 @@
+sleuthConfiguration:
+  server:
+    listen: ":17710"
+    readtimeout: "30s"
+    writetimeout: "3m0s"
+    shutdowngraceperiod: "30s"
+    maxbodysize: 102400
+  scanner:
+    timeout: "2m0s"
+    maxsubdomains: 50
+    nucleiseverity:
+      - critical
+      - high
+      - medium
+    verbose: false
+  intel:
+    feedconfig: "config/feed_config.json"
+    storagedir: "data/intel"
+    autohydrate: false
+    requesttimeout: "1m30s"
+    resolvertimeout: "10s"
+    dnscachettl: "5m0s"
+  cloudflare:
+    requesttimeout: "30s"
+  slack:
+    requesttimeout: "10s"