Skip to content

Commit bfb6b3a

Browse files
okankoAMZItielOlenick
okankoAMZ
and
ItielOlenick
authored
Target allocator TLS Implementation (#239)
* Ta https server (#2921) * Added https server, tests, secret marshalling --------- Co-authored-by: ItielOlenick <[email protected]>
1 parent 2dc2f2b commit bfb6b3a

File tree

22 files changed

+452
-219
lines changed

22 files changed

+452
-219
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ IMG ?= ${IMG_PREFIX}/${IMG_REPO}:${VERSION}
1818
ARCH ?= $(shell go env GOARCH)
1919

2020
TARGET_ALLOCATOR_IMG_REPO ?= target-allocator
21-
TARGET_ALLOCATOR_IMG ?= ${IMG_PREFIX}/${TARGET_ALLOCATOR_IMG_REPO}:${TARGET_ALLOCATOR_VERSION}
21+
TARGET_ALLOCATOR_IMG ?=${IMG_PREFIX}/${TARGET_ALLOCATOR_IMG_REPO}:${TARGET_ALLOCATOR_VERSION}
2222

2323
# Options for 'bundle-build'
2424
ifneq ($(origin CHANNELS), undefined)
@@ -103,7 +103,7 @@ manager: generate fmt vet
103103
# Build target allocator binary
104104
.PHONY: targetallocator
105105
targetallocator:
106-
cd cmd/amazon-cloudwatch-agent-target-allocator && CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(ARCH) go build -installsuffix cgo -o bin/targetallocator_${ARCH} -ldflags "${LDFLAGS}" .
106+
cd cmd/amazon-cloudwatch-agent-target-allocator && CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(ARCH) go build -installsuffix cgo -o bin/targetallocator_${ARCH} -ldflags "${LDFLAGS}" .
107107

108108
# Run against the configured Kubernetes cluster in ~/.kube/config
109109
.PHONY: run

‎cmd/amazon-cloudwatch-agent-target-allocator/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,6 @@ COPY --from=certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-ce
1616
# Copy binary built on the host
1717
COPY bin/targetallocator_${TARGETARCH} ./main
1818

19+
EXPOSE 8443
20+
1921
ENTRYPOINT ["./main"]

‎cmd/amazon-cloudwatch-agent-target-allocator/config/config.go‎

Lines changed: 89 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
package config
55

66
import (
7+
"crypto/tls"
8+
"crypto/x509"
79
"errors"
810
"fmt"
911
"io/fs"
@@ -16,38 +18,59 @@ import (
1618
_ "github.com/prometheus/prometheus/discovery/install"
1719
"github.com/spf13/pflag"
1820
"gopkg.in/yaml.v2"
21+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1922
"k8s.io/client-go/rest"
2023
"k8s.io/client-go/tools/clientcmd"
2124
"k8s.io/klog/v2"
2225
ctrl "sigs.k8s.io/controller-runtime"
2326
"sigs.k8s.io/controller-runtime/pkg/log/zap"
27+
28+
tamanifest "github.com/aws/amazon-cloudwatch-agent-operator/internal/manifests/targetallocator"
2429
)
2530

26-
const DefaultResyncTime = 5 * time.Minute
27-
const DefaultConfigFilePath string = "/conf/targetallocator.yaml"
28-
const DefaultCRScrapeInterval model.Duration = model.Duration(time.Second * 30)
29-
const DefaultAllocationStrategy string = "consistent-hashing"
31+
const (
32+
DefaultResyncTime = 5 * time.Minute
33+
DefaultConfigFilePath string = "/conf/targetallocator.yaml"
34+
DefaultCRScrapeInterval model.Duration = model.Duration(time.Second * 30)
35+
DefaultAllocationStrategy = "consistent-hashing"
36+
DefaultFilterStrategy = "relabel-config"
37+
DefaultListenAddr = ":8443"
38+
DefaultCertMountPath = tamanifest.TACertMountPath
39+
DefaultTLSKeyPath = DefaultCertMountPath + "/server.key"
40+
DefaultTLSCertPath = DefaultCertMountPath + "/server.crt"
41+
DefaultCABundlePath = ""
42+
)
3043

3144
type Config struct {
32-
ListenAddr string `yaml:"listen_addr,omitempty"`
33-
KubeConfigFilePath string `yaml:"kube_config_file_path,omitempty"`
34-
ClusterConfig *rest.Config `yaml:"-"`
35-
RootLogger logr.Logger `yaml:"-"`
36-
ReloadConfig bool `yaml:"-"`
37-
LabelSelector map[string]string `yaml:"label_selector,omitempty"`
38-
PromConfig *promconfig.Config `yaml:"config"`
39-
AllocationStrategy *string `yaml:"allocation_strategy,omitempty"`
40-
FilterStrategy *string `yaml:"filter_strategy,omitempty"`
41-
PrometheusCR PrometheusCRConfig `yaml:"prometheus_cr,omitempty"`
42-
PodMonitorSelector map[string]string `yaml:"pod_monitor_selector,omitempty"`
43-
ServiceMonitorSelector map[string]string `yaml:"service_monitor_selector,omitempty"`
45+
ListenAddr string `yaml:"listen_addr,omitempty"`
46+
KubeConfigFilePath string `yaml:"kube_config_file_path,omitempty"`
47+
ClusterConfig *rest.Config `yaml:"-"`
48+
RootLogger logr.Logger `yaml:"-"`
49+
ReloadConfig bool `yaml:"-"`
50+
LabelSelector map[string]string `yaml:"label_selector,omitempty"`
51+
PromConfig *promconfig.Config `yaml:"config"`
52+
AllocationStrategy *string `yaml:"allocation_strategy,omitempty"`
53+
FilterStrategy *string `yaml:"filter_strategy,omitempty"`
54+
PrometheusCR PrometheusCRConfig `yaml:"prometheus_cr,omitempty"`
55+
PodMonitorSelector map[string]string `yaml:"pod_monitor_selector,omitempty"`
56+
ServiceMonitorSelector map[string]string `yaml:"service_monitor_selector,omitempty"`
57+
CollectorSelector *metav1.LabelSelector `yaml:"collector_selector,omitempty"`
58+
HTTPS HTTPSServerConfig `yaml:"https,omitempty"`
4459
}
4560

4661
type PrometheusCRConfig struct {
4762
Enabled bool `yaml:"enabled,omitempty"`
4863
ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"`
4964
}
5065

66+
type HTTPSServerConfig struct {
67+
Enabled bool `yaml:"enabled,omitempty"`
68+
ListenAddr string `yaml:"listen_addr,omitempty"`
69+
CAFilePath string `yaml:"ca_file_path,omitempty"`
70+
TLSCertFilePath string `yaml:"tls_cert_file_path,omitempty"`
71+
TLSKeyFilePath string `yaml:"tls_key_file_path,omitempty"`
72+
}
73+
5174
func (c Config) GetAllocationStrategy() string {
5275
if c.AllocationStrategy != nil {
5376
return *c.AllocationStrategy
@@ -91,17 +114,32 @@ func LoadFromCLI(target *Config, flagSet *pflag.FlagSet) error {
91114
}
92115
target.ClusterConfig = clusterConfig
93116

94-
target.ListenAddr, err = getListenAddr(flagSet)
117+
target.ReloadConfig, err = getConfigReloadEnabled(flagSet)
95118
if err != nil {
96119
return err
97120
}
98121

99-
target.PrometheusCR.Enabled, err = getPrometheusCREnabled(flagSet)
122+
target.HTTPS.Enabled, err = getHttpsEnabled(flagSet)
100123
if err != nil {
101124
return err
102125
}
103126

104-
target.ReloadConfig, err = getConfigReloadEnabled(flagSet)
127+
target.HTTPS.ListenAddr, err = getHttpsListenAddr(flagSet)
128+
if err != nil {
129+
return err
130+
}
131+
132+
target.HTTPS.CAFilePath, err = getHttpsCAFilePath(flagSet)
133+
if err != nil {
134+
return err
135+
}
136+
137+
target.HTTPS.TLSCertFilePath, err = getHttpsTLSCertFilePath(flagSet)
138+
if err != nil {
139+
return err
140+
}
141+
142+
target.HTTPS.TLSKeyFilePath, err = getHttpsTLSKeyFilePath(flagSet)
105143
if err != nil {
106144
return err
107145
}
@@ -128,6 +166,13 @@ func CreateDefaultConfig() Config {
128166
ScrapeInterval: DefaultCRScrapeInterval,
129167
},
130168
AllocationStrategy: &allocation_strategy,
169+
HTTPS: HTTPSServerConfig{
170+
Enabled: true,
171+
ListenAddr: DefaultListenAddr,
172+
CAFilePath: DefaultCABundlePath,
173+
TLSCertFilePath: DefaultTLSCertPath,
174+
TLSKeyFilePath: DefaultTLSKeyPath,
175+
},
131176
}
132177
}
133178

@@ -168,3 +213,28 @@ func ValidateConfig(config *Config) error {
168213
}
169214
return nil
170215
}
216+
217+
func (c HTTPSServerConfig) NewTLSConfig() (*tls.Config, error) {
218+
cert, err := tls.LoadX509KeyPair(c.TLSCertFilePath, c.TLSKeyFilePath)
219+
if err != nil {
220+
return nil, err
221+
}
222+
tlsConfig := &tls.Config{
223+
Certificates: []tls.Certificate{cert},
224+
ClientAuth: tls.NoClientCert,
225+
MinVersion: tls.VersionTLS12,
226+
}
227+
if c.CAFilePath == "" {
228+
return tlsConfig, nil
229+
}
230+
caCert, err := os.ReadFile(c.CAFilePath)
231+
caCertPool := x509.NewCertPool()
232+
if err != nil {
233+
return nil, err
234+
}
235+
caCertPool.AppendCertsFromPEM(caCert)
236+
tlsConfig.ClientCAs = caCertPool
237+
tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
238+
239+
return tlsConfig, nil
240+
}

‎cmd/amazon-cloudwatch-agent-target-allocator/config/config_test.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,13 @@ func TestLoad(t *testing.T) {
4141
PrometheusCR: PrometheusCRConfig{
4242
ScrapeInterval: model.Duration(time.Second * 60),
4343
},
44+
HTTPS: HTTPSServerConfig{
45+
Enabled: true,
46+
ListenAddr: DefaultListenAddr,
47+
CAFilePath: "/path/to/ca.pem",
48+
TLSCertFilePath: "/path/to/cert.pem",
49+
TLSKeyFilePath: "/path/to/key.pem",
50+
},
4451
PromConfig: &promconfig.Config{
4552
GlobalConfig: promconfig.GlobalConfig{
4653
ScrapeInterval: model.Duration(60 * time.Second),
@@ -106,6 +113,13 @@ func TestLoad(t *testing.T) {
106113
PrometheusCR: PrometheusCRConfig{
107114
ScrapeInterval: DefaultCRScrapeInterval,
108115
},
116+
HTTPS: HTTPSServerConfig{
117+
Enabled: true,
118+
ListenAddr: DefaultListenAddr,
119+
CAFilePath: DefaultCABundlePath,
120+
TLSCertFilePath: DefaultTLSCertPath,
121+
TLSKeyFilePath: DefaultTLSKeyPath,
122+
},
109123
PromConfig: &promconfig.Config{
110124
GlobalConfig: promconfig.GlobalConfig{
111125
ScrapeInterval: model.Duration(60 * time.Second),

‎cmd/amazon-cloudwatch-agent-target-allocator/config/flags.go‎

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,17 @@ import (
1414

1515
// Flag names.
1616
const (
17-
targetAllocatorName = "target-allocator"
18-
configFilePathFlagName = "config-file"
19-
listenAddrFlagName = "listen-addr"
20-
prometheusCREnabledFlagName = "enable-prometheus-cr-watcher"
21-
kubeConfigPathFlagName = "kubeconfig-path"
22-
reloadConfigFlagName = "reload-config"
17+
targetAllocatorName = "target-allocator"
18+
configFilePathFlagName = "config-file"
19+
listenAddrFlagName = "listen-addr"
20+
prometheusCREnabledFlagName = "enable-prometheus-cr-watcher"
21+
kubeConfigPathFlagName = "kubeconfig-path"
22+
reloadConfigFlagName = "reload-config"
23+
httpsEnabledFlagName = "enable-https-server"
24+
listenAddrHttpsFlagName = "listen-addr-https"
25+
httpsCAFilePathFlagName = "https-ca-file"
26+
httpsTLSCertFilePathFlagName = "https-tls-cert-file"
27+
httpsTLSKeyFilePathFlagName = "https-tls-key-file"
2328
)
2429

2530
// We can't bind this flag to our FlagSet, so we need to handle it separately.
@@ -28,10 +33,13 @@ var zapCmdLineOpts zap.Options
2833
func getFlagSet(errorHandling pflag.ErrorHandling) *pflag.FlagSet {
2934
flagSet := pflag.NewFlagSet(targetAllocatorName, errorHandling)
3035
flagSet.String(configFilePathFlagName, DefaultConfigFilePath, "The path to the config file.")
31-
flagSet.String(listenAddrFlagName, ":8080", "The address where this service serves.")
32-
flagSet.Bool(prometheusCREnabledFlagName, false, "Enable Prometheus CRs as target sources")
3336
flagSet.String(kubeConfigPathFlagName, filepath.Join(homedir.HomeDir(), ".kube", "config"), "absolute path to the KubeconfigPath file")
3437
flagSet.Bool(reloadConfigFlagName, false, "Enable automatic configuration reloading. This functionality is deprecated and will be removed in a future release.")
38+
flagSet.Bool(httpsEnabledFlagName, true, "Enable HTTPS additional server")
39+
flagSet.String(listenAddrHttpsFlagName, ":8443", "The address where this service serves over HTTPS.")
40+
flagSet.String(httpsCAFilePathFlagName, DefaultCABundlePath, "The path to the HTTPS server TLS CA file.")
41+
flagSet.String(httpsTLSCertFilePathFlagName, DefaultTLSCertPath, "The path to the HTTPS server TLS certificate file.")
42+
flagSet.String(httpsTLSKeyFilePathFlagName, DefaultTLSKeyPath, "The path to the HTTPS server TLS key file.")
3543
zapFlagSet := flag.NewFlagSet("", flag.ErrorHandling(errorHandling))
3644
zapCmdLineOpts.BindFlags(zapFlagSet)
3745
flagSet.AddGoFlagSet(zapFlagSet)
@@ -46,14 +54,26 @@ func getKubeConfigFilePath(flagSet *pflag.FlagSet) (string, error) {
4654
return flagSet.GetString(kubeConfigPathFlagName)
4755
}
4856

49-
func getListenAddr(flagSet *pflag.FlagSet) (string, error) {
50-
return flagSet.GetString(listenAddrFlagName)
57+
func getConfigReloadEnabled(flagSet *pflag.FlagSet) (bool, error) {
58+
return flagSet.GetBool(reloadConfigFlagName)
5159
}
5260

53-
func getPrometheusCREnabled(flagSet *pflag.FlagSet) (bool, error) {
54-
return flagSet.GetBool(prometheusCREnabledFlagName)
61+
func getHttpsListenAddr(flagSet *pflag.FlagSet) (string, error) {
62+
return flagSet.GetString(listenAddrHttpsFlagName)
5563
}
5664

57-
func getConfigReloadEnabled(flagSet *pflag.FlagSet) (bool, error) {
58-
return flagSet.GetBool(reloadConfigFlagName)
65+
func getHttpsEnabled(flagSet *pflag.FlagSet) (bool, error) {
66+
return flagSet.GetBool(httpsEnabledFlagName)
67+
}
68+
69+
func getHttpsCAFilePath(flagSet *pflag.FlagSet) (string, error) {
70+
return flagSet.GetString(httpsCAFilePathFlagName)
71+
}
72+
73+
func getHttpsTLSCertFilePath(flagSet *pflag.FlagSet) (string, error) {
74+
return flagSet.GetString(httpsTLSCertFilePathFlagName)
75+
}
76+
77+
func getHttpsTLSKeyFilePath(flagSet *pflag.FlagSet) (string, error) {
78+
return flagSet.GetString(httpsTLSKeyFilePathFlagName)
5979
}

‎cmd/amazon-cloudwatch-agent-target-allocator/config/flags_test.go‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ func TestGetFlagSet(t *testing.T) {
1616

1717
// Check if each flag exists
1818
assert.NotNil(t, fs.Lookup(configFilePathFlagName), "Flag %s not found", configFilePathFlagName)
19-
assert.NotNil(t, fs.Lookup(listenAddrFlagName), "Flag %s not found", listenAddrFlagName)
20-
assert.NotNil(t, fs.Lookup(prometheusCREnabledFlagName), "Flag %s not found", prometheusCREnabledFlagName)
2119
assert.NotNil(t, fs.Lookup(kubeConfigPathFlagName), "Flag %s not found", kubeConfigPathFlagName)
2220
}
2321

@@ -41,18 +39,6 @@ func TestFlagGetters(t *testing.T) {
4139
expectedValue: filepath.Join("~", ".kube", "config"),
4240
getterFunc: func(fs *pflag.FlagSet) (interface{}, error) { return getKubeConfigFilePath(fs) },
4341
},
44-
{
45-
name: "GetListenAddr",
46-
flagArgs: []string{"--" + listenAddrFlagName, ":8081"},
47-
expectedValue: ":8081",
48-
getterFunc: func(fs *pflag.FlagSet) (interface{}, error) { return getListenAddr(fs) },
49-
},
50-
{
51-
name: "GetPrometheusCREnabled",
52-
flagArgs: []string{"--" + prometheusCREnabledFlagName, "true"},
53-
expectedValue: true,
54-
getterFunc: func(fs *pflag.FlagSet) (interface{}, error) { return getPrometheusCREnabled(fs) },
55-
},
5642
{
5743
name: "GetConfigReloadEnabled",
5844
flagArgs: []string{"--" + reloadConfigFlagName, "true"},
@@ -65,6 +51,18 @@ func TestFlagGetters(t *testing.T) {
6551
expectedErr: true,
6652
getterFunc: func(fs *pflag.FlagSet) (interface{}, error) { return getConfigFilePath(fs) },
6753
},
54+
{
55+
name: "HttpsServer",
56+
flagArgs: []string{"--" + httpsEnabledFlagName, "true"},
57+
expectedValue: true,
58+
getterFunc: func(fs *pflag.FlagSet) (interface{}, error) { return getHttpsEnabled(fs) },
59+
},
60+
{
61+
name: "HttpsServerKey",
62+
flagArgs: []string{"--" + httpsTLSKeyFilePathFlagName, "/path/to/tls.key"},
63+
expectedValue: "/path/to/tls.key",
64+
getterFunc: func(fs *pflag.FlagSet) (interface{}, error) { return getHttpsTLSKeyFilePath(fs) },
65+
},
6866
}
6967

7068
for _, tt := range tests {

‎cmd/amazon-cloudwatch-agent-target-allocator/config/testdata/config_test.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ label_selector:
33
app.kubernetes.io/managed-by: amazon-cloudwatch-agent-operator
44
prometheus_cr:
55
scrape_interval: 60s
6+
https:
7+
enabled: true
8+
ca_file_path: /path/to/ca.pem
9+
tls_cert_file_path: /path/to/cert.pem
10+
tls_key_file_path: /path/to/key.pem
611
config:
712
scrape_configs:
813
- job_name: prometheus

‎cmd/amazon-cloudwatch-agent-target-allocator/main.go‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,12 @@ func main() {
5555
errChan = make(chan error)
5656
)
5757
cfg, configFilePath, err := config.Load()
58+
5859
if err != nil {
5960
fmt.Printf("Failed to load config: %v", err)
6061
os.Exit(1)
6162
}
63+
setupLog.Info("init config", "Config-Http", cfg.HTTPS)
6264
ctrl.SetLogger(cfg.RootLogger)
6365

6466
if validationErr := config.ValidateConfig(cfg); validationErr != nil {
@@ -76,7 +78,15 @@ func main() {
7678
setupLog.Error(err, "Unable to initialize allocation strategy")
7779
os.Exit(1)
7880
}
79-
srv := server.NewServer(log, allocator, cfg.ListenAddr)
81+
82+
httpOptions := []server.Option{}
83+
tlsConfig, confErr := cfg.HTTPS.NewTLSConfig()
84+
if confErr != nil {
85+
setupLog.Error(confErr, "Unable to initialize TLS configuration", "Config", cfg.HTTPS)
86+
os.Exit(1)
87+
}
88+
httpOptions = append(httpOptions, server.WithTLSConfig(tlsConfig, cfg.HTTPS.ListenAddr))
89+
srv := server.NewServer(log, allocator, cfg.ListenAddr, httpOptions...)
8090

8191
discoveryCtx, discoveryCancel := context.WithCancel(ctx)
8292
discoveryManager = discovery.NewManager(discoveryCtx, gokitlog.NewNopLogger())
@@ -171,14 +181,14 @@ func main() {
171181
})
172182
runGroup.Add(
173183
func() error {
174-
err := srv.Start()
175-
setupLog.Info("Server failed to start")
184+
err := srv.StartHTTPS()
185+
setupLog.Info("HTTPS Server failed to start", "error", err)
176186
return err
177187
},
178-
func(_ error) {
179-
setupLog.Info("Closing server")
180-
if shutdownErr := srv.Shutdown(ctx); shutdownErr != nil {
181-
setupLog.Error(shutdownErr, "Error on server shutdown")
188+
func(intrpError error) {
189+
setupLog.Info("Closing HTTPS server", "intrp", intrpError)
190+
if shutdownErr := srv.ShutdownHTTPS(ctx); shutdownErr != nil {
191+
setupLog.Error(shutdownErr, "Error on HTTPS server shutdown")
182192
}
183193
})
184194
runGroup.Add(

0 commit comments

Comments
 (0)