diff --git a/jobs/loggr-syslog-binding-cache/spec b/jobs/loggr-syslog-binding-cache/spec index d09c781b8..2a082c7ba 100644 --- a/jobs/loggr-syslog-binding-cache/spec +++ b/jobs/loggr-syslog-binding-cache/spec @@ -14,6 +14,9 @@ templates: metrics.key.erb: config/certs/metrics.key aggregate_drains.yml.erb: config/aggregate_drains.yml prom_scraper_config.yml.erb: config/prom_scraper_config.yml + agent.crt.erb: config/certs/agent.crt + agent.key.erb: config/certs/agent.key + agent_ca.crt.erb: config/certs/agent_ca.crt packages: - binding-cache @@ -134,3 +137,27 @@ properties: logging.format.timestamp: description: "Format for timestamp in component logs. Valid values are 'deprecated' and 'rfc3339'." default: "deprecated" + + agent.port: + description: "Port the agent is serving gRPC via mTLS" + default: 3458 + agent.ca_cert: + description: | + TLS loggregator root CA certificate. It is required for key/cert + verification. + agent.cert: + description: "TLS certificate for syslog signed by the loggregator CA" + agent.key: + description: "TLS private key for syslog signed by the loggregator CA" + agent.cipher_suites: + description: | + An ordered list of supported SSL cipher suites. Allowed cipher suites are + TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 and TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384. + default: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" + + blacklisted_syslog_ranges: + description: | + A list of IP address ranges that are not allowed to be specified in + syslog drain binding URLs. + default: [] + example: [{start: 10.10.10.1, end: 10.10.10.10}] diff --git a/jobs/loggr-syslog-binding-cache/templates/agent.crt.erb b/jobs/loggr-syslog-binding-cache/templates/agent.crt.erb new file mode 100644 index 000000000..f01f3182a --- /dev/null +++ b/jobs/loggr-syslog-binding-cache/templates/agent.crt.erb @@ -0,0 +1 @@ +<%= p("agent.cert") %> diff --git a/jobs/loggr-syslog-binding-cache/templates/agent.key.erb b/jobs/loggr-syslog-binding-cache/templates/agent.key.erb new file mode 100644 index 000000000..211918fdd --- /dev/null +++ b/jobs/loggr-syslog-binding-cache/templates/agent.key.erb @@ -0,0 +1 @@ +<%= p("agent.key") %> diff --git a/jobs/loggr-syslog-binding-cache/templates/agent_ca.crt.erb b/jobs/loggr-syslog-binding-cache/templates/agent_ca.crt.erb new file mode 100644 index 000000000..19f95732d --- /dev/null +++ b/jobs/loggr-syslog-binding-cache/templates/agent_ca.crt.erb @@ -0,0 +1 @@ +<%= p("agent.ca_cert") %> diff --git a/jobs/loggr-syslog-binding-cache/templates/bpm.yml.erb b/jobs/loggr-syslog-binding-cache/templates/bpm.yml.erb index a91c6fe2b..106ec07e8 100644 --- a/jobs/loggr-syslog-binding-cache/templates/bpm.yml.erb +++ b/jobs/loggr-syslog-binding-cache/templates/bpm.yml.erb @@ -1,4 +1,9 @@ <% + blacklisted_ranges = p("blacklisted_syslog_ranges") + blacklisted_ips = blacklisted_ranges.map do |range| + "#{range['start']}-#{range['end']}" + end.join(",") + certs_dir = "/var/vcap/jobs/loggr-syslog-binding-cache/config/certs" api_url = link("cloud_controller").address if_p("api.override_url") { @@ -32,6 +37,14 @@ "DEBUG_METRICS" => "#{p("metrics.debug")}", "PPROF_PORT" => "#{p("metrics.pprof_port")}", "USE_RFC3339" => "#{p("logging.format.timestamp") == "rfc3339"}", + + "AGENT_CA_FILE_PATH" => "#{certs_dir}/agent_ca.crt", + "AGENT_CERT_FILE_PATH" => "#{certs_dir}/agent.crt", + "AGENT_KEY_FILE_PATH" => "#{certs_dir}/agent.key", + "AGENT_CIPHER_SUITES" => "#{p("agent.cipher_suites").split(":").join(",")}", + "AGENT_PORT" => "#{p("agent.port")}", + + "BLACKLISTED_SYSLOG_RANGES" => "#{blacklisted_ips}", } } bpm = {"processes" => [process] } diff --git a/src/cmd/syslog-agent/app/syslog_agent.go b/src/cmd/syslog-agent/app/syslog_agent.go index 277bbdeb1..ca21d0a1f 100644 --- a/src/cmd/syslog-agent/app/syslog_agent.go +++ b/src/cmd/syslog-agent/app/syslog_agent.go @@ -13,6 +13,7 @@ import ( gendiodes "code.cloudfoundry.org/go-diodes" "code.cloudfoundry.org/go-loggregator/v10" metrics "code.cloudfoundry.org/go-metric-registry" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" "code.cloudfoundry.org/tlsconfig" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/binding" @@ -56,19 +57,9 @@ func NewSyslogAgent( cfg Config, m Metrics, l *log.Logger, + appLogEmitterFactory applog.LogEmitterFactory, ) *SyslogAgent { internalTlsConfig, externalTlsConfig := drainTLSConfig(cfg) - writerFactory := syslog.NewWriterFactory( - internalTlsConfig, - externalTlsConfig, - syslog.NetworkTimeoutConfig{ - Keepalive: 10 * time.Second, - DialTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - }, - m, - ) - ingressTLSConfig, err := loggregator.NewIngressTLSConfig( cfg.GRPC.CAFile, cfg.GRPC.CertFile, @@ -81,17 +72,24 @@ func NewSyslogAgent( logClient, err := loggregator.NewIngressClient( ingressTLSConfig, loggregator.WithLogger(log.New(os.Stderr, "", log.LstdFlags)), + loggregator.WithAddr(fmt.Sprintf("localhost:%d", cfg.GRPC.Port)), ) if err != nil { l.Panicf("failed to create log client for syslog connector: %q", err) } + writerFactory := syslog.NewWriterFactory(internalTlsConfig, externalTlsConfig, syslog.NetworkTimeoutConfig{ + Keepalive: 10 * time.Second, + DialTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + }, m) + connector := syslog.NewSyslogConnector( cfg.DrainSkipCertVerify, timeoutwaitgroup.New(time.Minute), writerFactory, m, - syslog.WithLogClient(logClient, "syslog_agent"), + syslog.WithLogEmitter(appLogEmitterFactory.NewLogEmitter(logClient, "syslog_agent")), ) var cacheClient *cache.CacheClient diff --git a/src/cmd/syslog-agent/app/syslog_agent_mtls_test.go b/src/cmd/syslog-agent/app/syslog_agent_mtls_test.go index 4b98a5b78..1bb1204be 100644 --- a/src/cmd/syslog-agent/app/syslog_agent_mtls_test.go +++ b/src/cmd/syslog-agent/app/syslog_agent_mtls_test.go @@ -10,6 +10,8 @@ import ( "os" "time" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -154,7 +156,9 @@ var _ = Describe("SyslogAgent with mTLS", func() { agentCfg.Cache.PollingInterval = 10 * time.Millisecond } - agent = app.NewSyslogAgent(agentCfg, agentMetrics, agentLogr) + factory := applog.NewAppLogEmitterFactory() + + agent = app.NewSyslogAgent(agentCfg, agentMetrics, agentLogr, &factory) go agent.Run() }) diff --git a/src/cmd/syslog-agent/app/syslog_agent_test.go b/src/cmd/syslog-agent/app/syslog_agent_test.go index 46a6d2c24..37b20b388 100644 --- a/src/cmd/syslog-agent/app/syslog_agent_test.go +++ b/src/cmd/syslog-agent/app/syslog_agent_test.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -48,6 +50,8 @@ var _ = Describe("SyslogAgent", func() { agentMetrics *metricsHelpers.SpyMetricsRegistry agentLogr *log.Logger agent *app.SyslogAgent + + factory applog.LogEmitterFactory ) BeforeEach(func() { @@ -134,7 +138,9 @@ var _ = Describe("SyslogAgent", func() { agentCfg.Cache.PollingInterval = 10 * time.Millisecond } - agent = app.NewSyslogAgent(agentCfg, agentMetrics, agentLogr) + factory := applog.NewAppLogEmitterFactory() + + agent = app.NewSyslogAgent(agentCfg, agentMetrics, agentLogr, &factory) go agent.Run() }) @@ -238,6 +244,14 @@ var _ = Describe("SyslogAgent", func() { Eventually(agentMetrics.GetDebugMetricsEnabled).Should(BeFalse()) }) + It("configures appLogEmitter", func() { + spyFactory := testhelper.SpyAppLogEmitterFactory{} + app.NewSyslogAgent(agentCfg, agentMetrics, agentLogr, &spyFactory) + + Expect(spyFactory.SourceIndex()).Should(Equal("syslog_agent")) + Expect(spyFactory.LogClient()).ShouldNot(BeNil()) + }) + Context("when debug configuration is enabled", func() { BeforeEach(func() { agentCfg.MetricsServer.DebugMetrics = true @@ -423,7 +437,7 @@ var _ = Describe("SyslogAgent", func() { cfgCopy.GRPC.KeyFile = "invalid" msg := `failed to configure client TLS: "failed to load keypair: open invalid: no such file or directory"` - Expect(func() { app.NewSyslogAgent(cfgCopy, agentMetrics, agentLogr) }).To(PanicWith(msg)) + Expect(func() { app.NewSyslogAgent(cfgCopy, agentMetrics, agentLogr, factory) }).To(PanicWith(msg)) }) }) }) diff --git a/src/cmd/syslog-agent/main.go b/src/cmd/syslog-agent/main.go index 24fe4f5d8..87de2431a 100644 --- a/src/cmd/syslog-agent/main.go +++ b/src/cmd/syslog-agent/main.go @@ -5,6 +5,8 @@ import ( _ "net/http/pprof" //nolint:gosec "os" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" + metrics "code.cloudfoundry.org/go-metric-registry" "code.cloudfoundry.org/loggregator-agent-release/src/cmd/syslog-agent/app" @@ -33,5 +35,7 @@ func main() { ), ) - app.NewSyslogAgent(cfg, m, logger).Run() + factory := applog.NewAppLogEmitterFactory() + + app.NewSyslogAgent(cfg, m, logger, &factory).Run() } diff --git a/src/cmd/syslog-binding-cache/app/config.go b/src/cmd/syslog-binding-cache/app/config.go index 0a7f00d76..d9659f9f8 100644 --- a/src/cmd/syslog-binding-cache/app/config.go +++ b/src/cmd/syslog-binding-cache/app/config.go @@ -5,6 +5,7 @@ import ( "time" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/config" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/ingress/bindings" envstruct "code.cloudfoundry.org/go-envstruct" ) @@ -31,6 +32,20 @@ type Config struct { CachePort int `env:"CACHE_PORT, required, report"` MetricsServer config.MetricsServer + + ForwarderAgentAddress string `env:"FORWARDER_AGENT_ADDR"` + GRPC GRPC + Blacklist bindings.BlacklistRanges `env:"BLACKLISTED_SYSLOG_RANGES, report"` +} + +// GRPC stores the configuration for the router as a server using a PORT +// with mTLS certs and as a client. +type GRPC struct { + Port int `env:"AGENT_PORT, report"` + CAFile string `env:"AGENT_CA_FILE_PATH, required, report"` + CertFile string `env:"AGENT_CERT_FILE_PATH, required, report"` + KeyFile string `env:"AGENT_KEY_FILE_PATH, required, report"` + CipherSuites []string `env:"AGENT_CIPHER_SUITES, report"` } // LoadConfig will load the configuration for the syslog binding cache from the @@ -38,7 +53,8 @@ type Config struct { // panic. func LoadConfig() Config { cfg := Config{ - APIPollingInterval: 15 * time.Second, + APIPollingInterval: 15 * time.Second, + ForwarderAgentAddress: "localhost:3458", } if err := envstruct.Load(&cfg); err != nil { log.Panicf("Failed to load config from environment: %s", err) diff --git a/src/cmd/syslog-binding-cache/app/syslog_binding_cache.go b/src/cmd/syslog-binding-cache/app/syslog_binding_cache.go index ad2dba59d..b6262a517 100644 --- a/src/cmd/syslog-binding-cache/app/syslog_binding_cache.go +++ b/src/cmd/syslog-binding-cache/app/syslog_binding_cache.go @@ -4,12 +4,16 @@ import ( "crypto/tls" "fmt" "log" + "net" "net/http" _ "net/http/pprof" //nolint:gosec + "os" "sync" "time" + "code.cloudfoundry.org/go-loggregator/v10" metrics "code.cloudfoundry.org/go-metric-registry" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" "code.cloudfoundry.org/tlsconfig" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/binding" @@ -19,13 +23,20 @@ import ( "github.com/go-chi/chi/v5" ) +type IPChecker interface { + ResolveAddr(host string) (net.IP, error) + CheckBlacklist(ip net.IP) error +} + type SyslogBindingCache struct { config Config pprofServer *http.Server server *http.Server - log *log.Logger + logger *log.Logger metrics Metrics mu sync.Mutex + emitter applog.LogEmitter + checker IPChecker } type Metrics interface { @@ -34,11 +45,33 @@ type Metrics interface { RegisterDebugMetrics() } -func NewSyslogBindingCache(config Config, metrics Metrics, log *log.Logger) *SyslogBindingCache { +func NewSyslogBindingCache(config Config, metrics Metrics, logger *log.Logger) *SyslogBindingCache { + ingressTLSConfig, err := loggregator.NewIngressTLSConfig( + config.GRPC.CAFile, + config.GRPC.CertFile, + config.GRPC.KeyFile, + ) + if err != nil { + logger.Panicf("failed to configure client TLS: %q", err) + } + + logClient, err := loggregator.NewIngressClient( + ingressTLSConfig, + loggregator.WithLogger(log.New(os.Stderr, "", log.LstdFlags)), + loggregator.WithAddr(config.ForwarderAgentAddress), + ) + if err != nil { + logger.Panicf("failed to create logger client for syslog connector: %q", err) + } + factory := applog.NewAppLogEmitterFactory() + emitter := factory.NewLogEmitter(logClient, "syslog_binding_cache") + return &SyslogBindingCache{ config: config, - log: log, + logger: logger, metrics: metrics, + emitter: emitter, + checker: &config.Blacklist, } } @@ -50,11 +83,11 @@ func (sbc *SyslogBindingCache) Run() { Handler: http.DefaultServeMux, ReadHeaderTimeout: 2 * time.Second, } - go func() { sbc.log.Println("PPROF SERVER STOPPED " + sbc.pprofServer.ListenAndServe().Error()) }() + go func() { sbc.logger.Println("PPROF SERVER STOPPED " + sbc.pprofServer.ListenAndServe().Error()) }() } store := binding.NewStore(sbc.metrics) aggregateStore := binding.NewAggregateStore(sbc.config.AggregateDrainsFile) - poller := binding.NewPoller(sbc.apiClient(), sbc.config.APIPollingInterval, store, sbc.metrics, sbc.log) + poller := binding.NewPoller(sbc.apiClient(), sbc.config.APIPollingInterval, store, sbc.metrics, sbc.logger, sbc.emitter, &sbc.config.Blacklist) go poller.Poll() @@ -103,7 +136,7 @@ func (sbc *SyslogBindingCache) startServer(router chi.Router) { sbc.mu.Unlock() err := sbc.server.ListenAndServeTLS("", "") if err != http.ErrServerClosed { - sbc.log.Panicf("error creating listener: %s", err) + sbc.logger.Panicf("error creating listener: %s", err) } } @@ -115,7 +148,7 @@ func (sbc *SyslogBindingCache) tlsConfig() *tls.Config { tlsconfig.WithClientAuthenticationFromFile(sbc.config.CacheCAFile), ) if err != nil { - sbc.log.Panicf("failed to load server TLS config: %s", err) + sbc.logger.Panicf("failed to load server TLS config: %s", err) } if len(sbc.config.CipherSuites) > 0 { diff --git a/src/cmd/syslog-binding-cache/app/syslog_binding_cache_test.go b/src/cmd/syslog-binding-cache/app/syslog_binding_cache_test.go index 3561ec5c6..93e542a92 100644 --- a/src/cmd/syslog-binding-cache/app/syslog_binding_cache_test.go +++ b/src/cmd/syslog-binding-cache/app/syslog_binding_cache_test.go @@ -107,6 +107,7 @@ var _ = Describe("App", func() { err = aggDrainFile.Close() Expect(err).ToNot(HaveOccurred()) sbcCerts = testhelper.GenerateCerts("binding-cache-ca") + grpcPort := 30000 + GinkgoParallelProcess() sbcCfg = app.Config{ APIURL: capi.URL, APIPollingInterval: 10 * time.Millisecond, @@ -128,6 +129,12 @@ var _ = Describe("App", func() { KeyFile: sbcCerts.Key("metron"), PprofPort: uint16(pprofPort), }, + GRPC: app.GRPC{ + Port: grpcPort, + CAFile: sbcCerts.CA(), + CertFile: sbcCerts.Cert("metron"), + KeyFile: sbcCerts.Key("metron"), + }, } sbcMetrics = metricsHelpers.NewMetricsRegistry() sbcLogr = log.New(GinkgoWriter, "", log.LstdFlags) diff --git a/src/internal/testhelper/spy_app_log_emitter.go b/src/internal/testhelper/spy_app_log_emitter.go new file mode 100644 index 000000000..f4e1e7e33 --- /dev/null +++ b/src/internal/testhelper/spy_app_log_emitter.go @@ -0,0 +1,25 @@ +package testhelper + +import ( + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" +) + +type SpyAppLogEmitterFactory struct { + logClient applog.LogClient + sourceIndex string +} + +func (factory *SpyAppLogEmitterFactory) LogClient() applog.LogClient { + return factory.logClient +} + +func (factory *SpyAppLogEmitterFactory) SourceIndex() string { + return factory.sourceIndex +} + +func (factory *SpyAppLogEmitterFactory) NewLogEmitter(logClient applog.LogClient, sourceIndex string) applog.LogEmitter { + factory.logClient = logClient + factory.sourceIndex = sourceIndex + emitterFactory := applog.NewAppLogEmitterFactory() + return emitterFactory.NewLogEmitter(logClient, sourceIndex) +} diff --git a/src/internal/testhelper/spy_log_client.go b/src/internal/testhelper/spy_log_client.go new file mode 100644 index 000000000..6fbbd9f5d --- /dev/null +++ b/src/internal/testhelper/spy_log_client.go @@ -0,0 +1,82 @@ +package testhelper + +import ( + "code.cloudfoundry.org/go-loggregator/v10" + v2 "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" + "sync" +) + +type spyLogClient struct { + mu sync.Mutex + _message []string + _appID []string + + // We use maps to ensure that we can query the keys + _sourceType map[string]struct{} + _sourceInstance map[string]struct{} +} + +func NewSpyLogClient() *spyLogClient { + return &spyLogClient{ + _sourceType: make(map[string]struct{}), + _sourceInstance: make(map[string]struct{}), + } +} + +func (s *spyLogClient) EmitLog(message string, opts ...loggregator.EmitLogOption) { + s.mu.Lock() + defer s.mu.Unlock() + + env := &v2.Envelope{ + Tags: make(map[string]string), + } + + for _, o := range opts { + o(env) + } + + s._message = append(s._message, message) + s._appID = append(s._appID, env.SourceId) + s._sourceType[env.GetTags()["source_type"]] = struct{}{} + s._sourceInstance[env.GetInstanceId()] = struct{}{} +} + +func (s *spyLogClient) Message() []string { + s.mu.Lock() + defer s.mu.Unlock() + + return s._message +} + +func (s *spyLogClient) AppID() []string { + s.mu.Lock() + defer s.mu.Unlock() + + return s._appID +} + +func (s *spyLogClient) SourceType() map[string]struct{} { + s.mu.Lock() + defer s.mu.Unlock() + + // Copy map so the original does not escape the mutex and induce a race. + m := make(map[string]struct{}) + for k := range s._sourceType { + m[k] = struct{}{} + } + + return m +} + +func (s *spyLogClient) SourceInstance() map[string]struct{} { + s.mu.Lock() + defer s.mu.Unlock() + + // Copy map so the original does not escape the mutex and induce a race. + m := make(map[string]struct{}) + for k := range s._sourceInstance { + m[k] = struct{}{} + } + + return m +} diff --git a/src/pkg/ingress/bindings/bindingsfakes/fake_ipchecker.go b/src/pkg/binding/bindingfakes/fake_ipchecker.go similarity index 97% rename from src/pkg/ingress/bindings/bindingsfakes/fake_ipchecker.go rename to src/pkg/binding/bindingfakes/fake_ipchecker.go index 251bb4f7d..4e6e77bf0 100644 --- a/src/pkg/ingress/bindings/bindingsfakes/fake_ipchecker.go +++ b/src/pkg/binding/bindingfakes/fake_ipchecker.go @@ -1,11 +1,11 @@ // Code generated by counterfeiter. DO NOT EDIT. -package bindingsfakes +package bindingfakes import ( "net" "sync" - "code.cloudfoundry.org/loggregator-agent-release/src/pkg/ingress/bindings" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/binding" ) type FakeIPChecker struct { @@ -188,4 +188,4 @@ func (fake *FakeIPChecker) recordInvocation(key string, args []interface{}) { fake.invocations[key] = append(fake.invocations[key], args) } -var _ bindings.IPChecker = new(FakeIPChecker) +var _ binding.IPChecker = new(FakeIPChecker) diff --git a/src/pkg/binding/poller.go b/src/pkg/binding/poller.go index 188f9b18d..29d03a6c3 100644 --- a/src/pkg/binding/poller.go +++ b/src/pkg/binding/poller.go @@ -1,12 +1,19 @@ package binding import ( + "crypto/tls" + "crypto/x509" "encoding/json" + "fmt" "log" + "net" "net/http" + "net/url" "time" metrics "code.cloudfoundry.org/go-metric-registry" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/simplecache" ) type Poller struct { @@ -17,6 +24,15 @@ type Poller struct { logger *log.Logger bindingRefreshErrorCounter metrics.Counter lastBindingCount metrics.Gauge + emitter applog.LogEmitter + checker IPChecker + failedHostsCache *simplecache.SimpleCache[string, bool] +} + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . IPChecker +type IPChecker interface { + ResolveAddr(host string) (net.IP, error) + CheckBlacklist(ip net.IP) error } type client interface { @@ -51,7 +67,7 @@ type Setter interface { Set(bindings []Binding, bindingCount int) } -func NewPoller(ac client, pi time.Duration, s Setter, m Metrics, logger *log.Logger) *Poller { +func NewPoller(ac client, pi time.Duration, s Setter, m Metrics, logger *log.Logger, emitter applog.LogEmitter, checker IPChecker) *Poller { p := &Poller{ apiClient: ac, pollingInterval: pi, @@ -65,6 +81,9 @@ func NewPoller(ac client, pi time.Duration, s Setter, m Metrics, logger *log.Log "last_binding_refresh_count", "Current number of bindings received from binding provider during last refresh.", ), + emitter: emitter, + checker: checker, + failedHostsCache: simplecache.New[string, bool](120 * time.Second), } p.poll() return p @@ -107,11 +126,112 @@ func (p *Poller) poll() { } } + checkBindings(bindings, p.emitter, p.checker, p.logger, p.failedHostsCache) + bindingCount := CalculateBindingCount(bindings) p.lastBindingCount.Set(float64(bindingCount)) p.store.Set(bindings, bindingCount) } +func checkBindings(bindings []Binding, emitter applog.LogEmitter, checker IPChecker, logger *log.Logger, failedHostsCache *simplecache.SimpleCache[string, bool]) { + logger.Printf("checking bindings - found %d bindings", len(bindings)) + for _, b := range bindings { + if len(b.Credentials) == 0 { + logger.Printf("no credentials for %s", b.Url) + continue + } + + //todo provide Prometheus metrics for invalid/blacklisted drains + u, err := url.Parse(b.Url) + + for _, cred := range b.Credentials { + if err != nil { + sendAppLogMessage(fmt.Sprintf("Cannot parse syslog drain url %s", b.Url), cred.Apps, emitter, logger) + continue + } + + anonymousUrl := u + anonymousUrl.User = nil + anonymousUrl.RawQuery = "" + + if invalidScheme(u.Scheme) { + sendAppLogMessage(fmt.Sprintf("Invalid Scheme for syslog drain url %s", b.Url), cred.Apps, emitter, logger) + continue + } + + if len(u.Host) == 0 { + sendAppLogMessage(fmt.Sprintf("No hostname found in syslog drain url %s", b.Url), cred.Apps, emitter, logger) + continue + } + + _, exists := failedHostsCache.Get(u.Host) + if exists { + //invalidDrains += 1 + sendAppLogMessage(fmt.Sprintf("Skipped resolve ip address for syslog drain with url %s due to prior failure", anonymousUrl.String()), cred.Apps, emitter, logger) + continue + } + + ip, err := checker.ResolveAddr(u.Host) + if err != nil { + //invalidDrains += 1 + failedHostsCache.Set(u.Host, true) + sendAppLogMessage(fmt.Sprintf("Cannot resolve ip address for syslog drain with url %s", anonymousUrl.String()), cred.Apps, emitter, logger) + continue + } + + err = checker.CheckBlacklist(ip) + if err != nil { + //invalidDrains += 1 + //blacklistedDrains += 1 + sendAppLogMessage(fmt.Sprintf("Resolved ip address for syslog drain with url %s is blacklisted", anonymousUrl.String()), cred.Apps, emitter, logger) + continue + } + + if len(cred.Cert) > 0 && len(cred.Key) > 0 { + _, err := tls.X509KeyPair([]byte(cred.Cert), []byte(cred.Key)) + if err != nil { + errorMessage := err.Error() + sendAppLogMessage(fmt.Sprintf("failed to load certificate: %s", errorMessage), cred.Apps, emitter, logger) + continue + } + } + + if len(cred.CA) > 0 { + certPool := x509.NewCertPool() + ok := certPool.AppendCertsFromPEM([]byte(cred.CA)) + if !ok { + sendAppLogMessage("failed to load root CA", cred.Apps, emitter, logger) + continue + } + } + } + } +} + +func sendAppLogMessage(msg string, apps []App, emitter applog.LogEmitter, logger *log.Logger) { + for _, app := range apps { + appId := app.AppID + if appId == "" { + continue + } + emitter.EmitAppLog(appId, msg) + emitter.EmitPlatformLog(fmt.Sprintf("%s for app %s", msg, appId)) + logger.Printf("%s for app %s", msg, appId) + } +} + +var allowedSchemes = []string{"syslog", "syslog-tls", "https", "https-batch"} + +func invalidScheme(scheme string) bool { + for _, s := range allowedSchemes { + if s == scheme { + return false + } + } + + return true +} + func CalculateBindingCount(bindings []Binding) int { apps := make(map[string]bool) for _, b := range bindings { diff --git a/src/pkg/binding/poller_test.go b/src/pkg/binding/poller_test.go index bc6212e37..645213ea8 100644 --- a/src/pkg/binding/poller_test.go +++ b/src/pkg/binding/poller_test.go @@ -6,11 +6,14 @@ import ( "errors" "io" "log" + "net" "net/http" "sync/atomic" "time" metricsHelpers "code.cloudfoundry.org/go-metric-registry/testhelpers" + "code.cloudfoundry.org/loggregator-agent-release/src/internal/testhelper" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -24,16 +27,20 @@ var _ = Describe("Poller", func() { store *fakeStore metrics *metricsHelpers.SpyMetricsRegistry logger = log.New(GinkgoWriter, "", 0) + emitter applog.LogEmitter ) BeforeEach(func() { apiClient = newFakeAPIClient() store = newFakeStore() metrics = metricsHelpers.NewMetricsRegistry() + factory := applog.NewAppLogEmitterFactory() + logClient := testhelper.NewSpyLogClient() + emitter = factory.NewLogEmitter(logClient, "test") }) It("polls for bindings on an interval", func() { - p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger) + p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger, emitter, &dummyIPChecker{}) go p.Poll() Eventually(apiClient.called).Should(BeNumerically(">=", 2)) @@ -67,7 +74,7 @@ var _ = Describe("Poller", func() { }, } - p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger) + p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger, emitter, &dummyIPChecker{}) go p.Poll() var expectedBindings []binding.Binding @@ -142,7 +149,7 @@ var _ = Describe("Poller", func() { }, } - p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger) + p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger, emitter, &dummyIPChecker{}) go p.Poll() var expectedBindings []binding.Binding @@ -188,7 +195,7 @@ var _ = Describe("Poller", func() { }) It("tracks the number of API errors", func() { - p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger) + p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger, emitter, &dummyIPChecker{}) go p.Poll() apiClient.errors <- errors.New("expected") @@ -201,7 +208,7 @@ var _ = Describe("Poller", func() { It("does not update the stores if the response code is bad", func() { apiClient.statusCode <- 404 - p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger) + p := binding.NewPoller(apiClient, 10*time.Millisecond, store, metrics, logger, emitter, &dummyIPChecker{}) go p.Poll() Eventually(store.bindings).Should(BeEmpty()) @@ -228,7 +235,7 @@ var _ = Describe("Poller", func() { }, }, } - binding.NewPoller(apiClient, time.Hour, store, metrics, logger) + binding.NewPoller(apiClient, time.Hour, store, metrics, logger, emitter, &dummyIPChecker{}) Expect(metrics.GetMetric("last_binding_refresh_count", nil).Value()). To(BeNumerically("==", 2)) @@ -346,3 +353,13 @@ type response struct { Results []binding.Binding NextID int `json:"next_id"` } + +type dummyIPChecker struct{} + +func (d *dummyIPChecker) ResolveAddr(host string) (net.IP, error) { + return net.IPv4(127, 0, 0, 1), nil +} + +func (*dummyIPChecker) CheckBlacklist(ip net.IP) error { + return nil +} diff --git a/src/pkg/egress/applog/log_emitter.go b/src/pkg/egress/applog/log_emitter.go new file mode 100644 index 000000000..8beedb92a --- /dev/null +++ b/src/pkg/egress/applog/log_emitter.go @@ -0,0 +1,66 @@ +package applog + +import ( + "code.cloudfoundry.org/go-loggregator/v10" +) + +// LogClient is used to emit logs. +type LogClient interface { + EmitLog(message string, opts ...loggregator.EmitLogOption) +} + +// LogEmitter abstracts the sending of a log to the application log stream. +type LogEmitter struct { + logClient LogClient + sourceIndex string +} + +// EmitAppLog writes a message in the application log stream using a LogClient. +func (logEmitter *LogEmitter) EmitAppLog(appID string, message string) { + if logEmitter.logClient == nil || appID == "" { + return + } + + option := loggregator.WithAppInfo(appID, "LGR", "") + logEmitter.logClient.EmitLog(message, option) + + option = loggregator.WithAppInfo( + appID, + "SYS", + logEmitter.sourceIndex, + ) + logEmitter.logClient.EmitLog(message, option) +} + +func (logEmitter *LogEmitter) EmitPlatformLog(message string) { + option := loggregator.WithAppInfo("", "LGR", "") + logEmitter.logClient.EmitLog(message, option) + + option = loggregator.WithAppInfo( + "", + "SYS", + logEmitter.sourceIndex, + ) + logEmitter.logClient.EmitLog(message, option) +} + +// LogEmitterFactory is used to create new instances of LogEmitter +type LogEmitterFactory interface { + NewLogEmitter(logClient LogClient, sourceIndex string) LogEmitter +} + +// DefaultLogEmitterFactory implementation of LogEmitterFactory to produce DefaultAppLogEmitter. +type DefaultLogEmitterFactory struct { +} + +// NewAppLogEmitter creates a new LogEmitter. +func (factory *DefaultLogEmitterFactory) NewLogEmitter(logClient LogClient, sourceIndex string) LogEmitter { + return LogEmitter{ + logClient: logClient, + sourceIndex: sourceIndex, + } +} + +func NewAppLogEmitterFactory() DefaultLogEmitterFactory { + return DefaultLogEmitterFactory{} +} diff --git a/src/pkg/egress/applog/log_emitter_test.go b/src/pkg/egress/applog/log_emitter_test.go new file mode 100644 index 000000000..37eb6c209 --- /dev/null +++ b/src/pkg/egress/applog/log_emitter_test.go @@ -0,0 +1,72 @@ +package applog_test + +import ( + "code.cloudfoundry.org/loggregator-agent-release/src/internal/testhelper" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Loggregator Emitter", func() { + Describe("DefaultAppLogEmitter", func() { + It("emits a log message", func() { + logClient := testhelper.NewSpyLogClient() + factory := applog.NewAppLogEmitterFactory() + emitter := factory.NewLogEmitter(logClient, "0") + + emitter.EmitAppLog("app-id", "some-message") + + messages := logClient.Message() + appIDs := logClient.AppID() + sourceTypes := logClient.SourceType() + Expect(messages).To(HaveLen(2)) + Expect(messages[0]).To(Equal("some-message")) + Expect(messages[1]).To(Equal("some-message")) + Expect(appIDs[0]).To(Equal("app-id")) + Expect(appIDs[1]).To(Equal("app-id")) + Expect(sourceTypes).To(HaveKey("LGR")) + Expect(sourceTypes).To(HaveKey("SYS")) + }) + + It("does not emit a log message if the appID is empty", func() { + logClient := testhelper.NewSpyLogClient() + factory := applog.NewAppLogEmitterFactory() + emitter := factory.NewLogEmitter(logClient, "0") + + emitter.EmitAppLog("", "some-message") + + messages := logClient.Message() + appIDs := logClient.AppID() + sourceTypes := logClient.SourceType() + Expect(messages).To(HaveLen(0)) + Expect(appIDs).To(HaveLen(0)) + Expect(sourceTypes).ToNot(HaveKey("LGR")) + Expect(sourceTypes).ToNot(HaveKey("SYS")) + }) + }) + + Describe("DefaultLogEmitterFactory", func() { + It("produces a LogEmitter", func() { + factory := applog.NewAppLogEmitterFactory() + logClient := testhelper.NewSpyLogClient() + sourceIndex := "test-index" + + emitter := factory.NewLogEmitter(logClient, sourceIndex) + emitter.EmitAppLog("app-id", "some-message") + + messages := logClient.Message() + appIDs := logClient.AppID() + sourceTypes := logClient.SourceType() + sourceInstance := logClient.SourceInstance() + Expect(messages).To(HaveLen(2)) + Expect(messages[0]).To(Equal("some-message")) + Expect(messages[1]).To(Equal("some-message")) + Expect(appIDs[0]).To(Equal("app-id")) + Expect(appIDs[1]).To(Equal("app-id")) + Expect(sourceTypes).To(HaveKey("LGR")) + Expect(sourceTypes).To(HaveKey("SYS")) + Expect(sourceInstance).To(HaveKey("")) + Expect(sourceInstance).To(HaveKey("test-index")) + }) + }) +}) diff --git a/src/pkg/egress/syslog/retry_writer_test.go b/src/pkg/egress/syslog/retry_writer_test.go index 65c4752f2..73060abf3 100644 --- a/src/pkg/egress/syslog/retry_writer_test.go +++ b/src/pkg/egress/syslog/retry_writer_test.go @@ -3,11 +3,9 @@ package syslog_test import ( "errors" "net/url" - "sync" "sync/atomic" "time" - "code.cloudfoundry.org/go-loggregator/v10" v2 "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/syslog" @@ -165,81 +163,6 @@ func (s *spyWriteCloser) WriteAttempts() int { return int(atomic.LoadInt64(&s.writeAttempts)) } -type spyLogClient struct { - mu sync.Mutex - _message []string - _appID []string - - // We use maps to ensure that we can query the keys - _sourceType map[string]struct{} - _sourceInstance map[string]struct{} -} - -func newSpyLogClient() *spyLogClient { - return &spyLogClient{ - _sourceType: make(map[string]struct{}), - _sourceInstance: make(map[string]struct{}), - } -} - -func (s *spyLogClient) EmitLog(message string, opts ...loggregator.EmitLogOption) { - s.mu.Lock() - defer s.mu.Unlock() - - env := &v2.Envelope{ - Tags: make(map[string]string), - } - - for _, o := range opts { - o(env) - } - - s._message = append(s._message, message) - s._appID = append(s._appID, env.SourceId) - s._sourceType[env.GetTags()["source_type"]] = struct{}{} - s._sourceInstance[env.GetInstanceId()] = struct{}{} -} - -func (s *spyLogClient) message() []string { - s.mu.Lock() - defer s.mu.Unlock() - - return s._message -} - -func (s *spyLogClient) appID() []string { - s.mu.Lock() - defer s.mu.Unlock() - - return s._appID -} - -func (s *spyLogClient) sourceType() map[string]struct{} { - s.mu.Lock() - defer s.mu.Unlock() - - // Copy map so the orig does not escape the mutex and induce a race. - m := make(map[string]struct{}) - for k := range s._sourceType { - m[k] = struct{}{} - } - - return m -} - -func (s *spyLogClient) sourceInstance() map[string]struct{} { - s.mu.Lock() - defer s.mu.Unlock() - - // Copy map so the orig does not escape the mutex and induce a race. - m := make(map[string]struct{}) - for k := range s._sourceInstance { - m[k] = struct{}{} - } - - return m -} - func buildDelay(multiplier time.Duration) func(int) time.Duration { return func(attempt int) time.Duration { return time.Duration(attempt) * multiplier diff --git a/src/pkg/egress/syslog/syslog_connector.go b/src/pkg/egress/syslog/syslog_connector.go index bb06b34a1..47c635315 100644 --- a/src/pkg/egress/syslog/syslog_connector.go +++ b/src/pkg/egress/syslog/syslog_connector.go @@ -4,12 +4,12 @@ import ( "fmt" "log" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" "golang.org/x/net/context" metrics "code.cloudfoundry.org/go-metric-registry" "code.cloudfoundry.org/go-diodes" - "code.cloudfoundry.org/go-loggregator/v10" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress" ) @@ -33,32 +33,20 @@ type Credentials struct { CA string `json:"ca"` } -// LogClient is used to emit logs. -type LogClient interface { - EmitLog(message string, opts ...loggregator.EmitLogOption) -} - -// nullLogClient ensures that the LogClient is in fact optional. -type nullLogClient struct{} - -// EmitLog drops all messages into /dev/null. -func (nullLogClient) EmitLog(message string, opts ...loggregator.EmitLogOption) { -} - type writerFactory interface { - NewWriter(*URLBinding) (egress.WriteCloser, error) + NewWriter(*URLBinding, applog.LogEmitter) (egress.WriteCloser, error) } // SyslogConnector creates the various egress syslog writers. type SyslogConnector struct { skipCertVerify bool - logClient LogClient wg egress.WaitGroup - sourceIndex string writerFactory writerFactory + metricClient metricClient - metricClient metricClient droppedMetric metrics.Counter + + appLogEmitter applog.LogEmitter } // NewSyslogConnector configures and returns a new SyslogConnector. @@ -78,7 +66,6 @@ func NewSyslogConnector( sc := &SyslogConnector{ skipCertVerify: skipCertVerify, wg: wg, - logClient: nullLogClient{}, writerFactory: f, metricClient: m, @@ -93,12 +80,11 @@ func NewSyslogConnector( // ConnectorOption allows a syslog connector to be customized. type ConnectorOption func(*SyslogConnector) -// WithLogClient returns a ConnectorOption that will set up logging for any +// WithLogEmitter returns a ConnectorOption that will set up logging for any // information about a binding. -func WithLogClient(logClient LogClient, sourceIndex string) ConnectorOption { +func WithLogEmitter(emitter applog.LogEmitter) ConnectorOption { return func(sc *SyslogConnector) { - sc.logClient = logClient - sc.sourceIndex = sourceIndex + sc.appLogEmitter = emitter } } @@ -110,7 +96,7 @@ func (w *SyslogConnector) Connect(ctx context.Context, b Binding) (egress.Writer return nil, err } - writer, err := w.writerFactory.NewWriter(urlBinding) + writer, err := w.writerFactory.NewWriter(urlBinding, w.appLogEmitter) if err != nil { return nil, err } @@ -138,8 +124,8 @@ func (w *SyslogConnector) Connect(ctx context.Context, b Binding) (egress.Writer w.droppedMetric.Add(float64(missed)) drainDroppedMetric.Add(float64(missed)) - w.emitLoggregatorErrorLog(b.AppId, fmt.Sprintf("%d messages lost for application %s in user provided syslog drain with url %s", missed, b.AppId, anonymousUrl.String())) w.emitStandardOutErrorLog(b.AppId, urlBinding.Scheme(), anonymousUrl.String(), missed) + w.appLogEmitter.EmitAppLog(b.AppId, fmt.Sprintf("%d messages lost for application %s in user provided syslog drain with url %s", missed, b.AppId, anonymousUrl.String())) }), w.wg) filteredWriter, err := NewFilteringDrainWriter(b, dw) @@ -151,20 +137,6 @@ func (w *SyslogConnector) Connect(ctx context.Context, b Binding) (egress.Writer return filteredWriter, nil } -func (w *SyslogConnector) emitLoggregatorErrorLog(appID, message string) { - if appID == "" { - return - } - option := loggregator.WithAppInfo(appID, "LGR", "") - w.logClient.EmitLog(message, option) - - option = loggregator.WithAppInfo( - appID, - "SYS", - w.sourceIndex, - ) - w.logClient.EmitLog(message, option) -} func (w *SyslogConnector) emitStandardOutErrorLog(appID, scheme, url string, missed int) { errorAppOrAggregate := fmt.Sprintf("for %s's app drain", appID) if appID == "" { diff --git a/src/pkg/egress/syslog/syslog_connector_test.go b/src/pkg/egress/syslog/syslog_connector_test.go index cb58a740d..ce4dcde75 100644 --- a/src/pkg/egress/syslog/syslog_connector_test.go +++ b/src/pkg/egress/syslog/syslog_connector_test.go @@ -1,6 +1,9 @@ package syslog_test import ( + "code.cloudfoundry.org/loggregator-agent-release/src/internal/testhelper" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" + "errors" "fmt" "io" @@ -172,13 +175,14 @@ var _ = Describe("SyslogConnector", func() { }) It("emits a LGR and SYS log to the log client about logs that have been dropped", func() { - logClient := newSpyLogClient() + logClient := testhelper.NewSpyLogClient() + factory := applog.NewAppLogEmitterFactory() connector := syslog.NewSyslogConnector( true, spyWaitGroup, writerFactory, sm, - syslog.WithLogClient(logClient, "3"), + syslog.WithLogEmitter(factory.NewLogEmitter(logClient, "3")), ) binding := syslog.Binding{AppId: "app-id", @@ -201,26 +205,27 @@ var _ = Describe("SyslogConnector", func() { } }(writer) - Eventually(logClient.message).Should(ContainElement(MatchRegexp("\\d messages lost for application (.*) in user provided syslog drain with url"))) - Eventually(logClient.appID).Should(ContainElement("app-id")) + Eventually(logClient.Message).Should(ContainElement(MatchRegexp("\\d messages lost for application (.*) in user provided syslog drain with url"))) + Eventually(logClient.AppID).Should(ContainElement("app-id")) - Eventually(logClient.sourceType).Should(HaveLen(2)) - Eventually(logClient.sourceType).Should(HaveKey("LGR")) - Eventually(logClient.sourceType).Should(HaveKey("SYS")) + Eventually(logClient.SourceType).Should(HaveLen(2)) + Eventually(logClient.SourceType).Should(HaveKey("LGR")) + Eventually(logClient.SourceType).Should(HaveKey("SYS")) - Eventually(logClient.sourceInstance).Should(HaveLen(2)) - Eventually(logClient.sourceInstance).Should(HaveKey("")) - Eventually(logClient.sourceInstance).Should(HaveKey("3")) + Eventually(logClient.SourceInstance).Should(HaveLen(2)) + Eventually(logClient.SourceInstance).Should(HaveKey("")) + Eventually(logClient.SourceInstance).Should(HaveKey("3")) }) It("doesn't emit LGR and SYS log to the log client about aggregate drains drops", func() { - logClient := newSpyLogClient() + logClient := testhelper.NewSpyLogClient() + factory := applog.NewAppLogEmitterFactory() connector := syslog.NewSyslogConnector( true, spyWaitGroup, writerFactory, sm, - syslog.WithLogClient(logClient, "3"), + syslog.WithLogEmitter(factory.NewLogEmitter(logClient, "3")), ) binding := syslog.Binding{Drain: syslog.Drain{Url: "dropping://"}} @@ -239,7 +244,7 @@ var _ = Describe("SyslogConnector", func() { } }(writer) - Consistently(logClient.message).ShouldNot(ContainElement(MatchRegexp("\\d messages lost for application (.*) in user provided syslog drain with url"))) + Consistently(logClient.Message()).ShouldNot(ContainElement(MatchRegexp("\\d messages lost for application (.*) in user provided syslog drain with url"))) }) It("does not panic on unknown dropped metrics", func() { @@ -276,6 +281,7 @@ type stubWriterFactory struct { func (f *stubWriterFactory) NewWriter( urlBinding *syslog.URLBinding, + emitter applog.LogEmitter, ) (egress.WriteCloser, error) { f.called = true return f.writer, f.err diff --git a/src/pkg/egress/syslog/tcp.go b/src/pkg/egress/syslog/tcp.go index fc1cd47c4..1ded3c6c2 100644 --- a/src/pkg/egress/syslog/tcp.go +++ b/src/pkg/egress/syslog/tcp.go @@ -13,6 +13,7 @@ import ( "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" metrics "code.cloudfoundry.org/go-metric-registry" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" ) // DialFunc represents a method for creating a connection, either TCP or TLS. @@ -32,6 +33,8 @@ type TCPWriter struct { syslogConverter *Converter egressMetric metrics.Counter + + emitter applog.LogEmitter } // NewTCPWriter creates a new TCP syslog writer. @@ -40,6 +43,7 @@ func NewTCPWriter( netConf NetworkTimeoutConfig, egressMetric metrics.Counter, c *Converter, + emitter applog.LogEmitter, ) egress.WriteCloser { dialer := &net.Dialer{ Timeout: netConf.DialTimeout, @@ -58,6 +62,7 @@ func NewTCPWriter( scheme: "syslog", egressMetric: egressMetric, syslogConverter: c, + emitter: emitter, } return w @@ -104,6 +109,9 @@ func (w *TCPWriter) connection() (net.Conn, error) { func (w *TCPWriter) connect() (net.Conn, error) { conn, err := w.dialFunc(w.url.Host) if err != nil { + logMessage := fmt.Sprintf("Failed to connect to %s", w.url.String()) + w.emitter.EmitAppLog(w.appID, logMessage) + w.emitter.EmitPlatformLog(fmt.Sprintf("%s for app %s", logMessage, w.appID)) return nil, err } w.conn = conn diff --git a/src/pkg/egress/syslog/tcp_test.go b/src/pkg/egress/syslog/tcp_test.go index b40d5c334..f2de1da69 100644 --- a/src/pkg/egress/syslog/tcp_test.go +++ b/src/pkg/egress/syslog/tcp_test.go @@ -2,6 +2,10 @@ package syslog_test import ( "bufio" + + "code.cloudfoundry.org/loggregator-agent-release/src/internal/testhelper" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" + "fmt" "io" "net" @@ -51,12 +55,16 @@ var _ = Describe("TCPWriter", func() { BeforeEach(func() { var err error egressCounter = &metricsHelpers.SpyMetric{} + factory := applog.NewAppLogEmitterFactory() + logClient := testhelper.NewSpyLogClient() + emitter := factory.NewLogEmitter(logClient, "3") writer = syslog.NewTCPWriter( binding, netConf, egressCounter, syslog.NewConverter(), + emitter, ) Expect(err).ToNot(HaveOccurred()) }) @@ -183,12 +191,16 @@ var _ = Describe("TCPWriter", func() { It("write returns an error", func() { env := buildLogEnvelope("APP", "2", "just a test", loggregator_v2.Log_OUT) binding.URL, _ = url.Parse("syslog://localhost-garbage:9999") + factory := applog.NewAppLogEmitterFactory() + logClient := testhelper.NewSpyLogClient() + emitter := factory.NewLogEmitter(logClient, "3") writer := syslog.NewTCPWriter( binding, netConf, &metricsHelpers.SpyMetric{}, syslog.NewConverter(), + emitter, ) errs := make(chan error, 1) @@ -208,11 +220,15 @@ var _ = Describe("TCPWriter", func() { Context("with a happy dialer", func() { BeforeEach(func() { var err error + factory := applog.NewAppLogEmitterFactory() + logClient := testhelper.NewSpyLogClient() + emitter := factory.NewLogEmitter(logClient, "3") writer = syslog.NewTCPWriter( binding, netConf, &metricsHelpers.SpyMetric{}, syslog.NewConverter(), + emitter, ) Expect(err).ToNot(HaveOccurred()) diff --git a/src/pkg/egress/syslog/tls.go b/src/pkg/egress/syslog/tls.go index 1610efc4f..6a18dd2b7 100644 --- a/src/pkg/egress/syslog/tls.go +++ b/src/pkg/egress/syslog/tls.go @@ -7,6 +7,7 @@ import ( metrics "code.cloudfoundry.org/go-metric-registry" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" ) // TLSWriter represents a syslog writer that connects over unencrypted TCP. @@ -27,6 +28,7 @@ func NewTLSWriter( tlsConf *tls.Config, egressMetric metrics.Counter, syslogConverter *Converter, + emitter applog.LogEmitter, ) egress.WriteCloser { dialer := &net.Dialer{ @@ -48,6 +50,7 @@ func NewTLSWriter( scheme: "syslog-tls", egressMetric: egressMetric, syslogConverter: syslogConverter, + emitter: emitter, }, } diff --git a/src/pkg/egress/syslog/tls_test.go b/src/pkg/egress/syslog/tls_test.go index 0f3adf4c4..30f831981 100644 --- a/src/pkg/egress/syslog/tls_test.go +++ b/src/pkg/egress/syslog/tls_test.go @@ -10,6 +10,7 @@ import ( metricsHelpers "code.cloudfoundry.org/go-metric-registry/testhelpers" "code.cloudfoundry.org/loggregator-agent-release/src/internal/testhelper" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/syslog" "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" @@ -55,6 +56,9 @@ var _ = Describe("TLSWriter", func() { Hostname: "test-hostname", URL: url, } + factory := applog.NewAppLogEmitterFactory() + logClient := testhelper.NewSpyLogClient() + emitter := factory.NewLogEmitter(logClient, "3") writer := syslog.NewTLSWriter( binding, netConf, @@ -63,6 +67,7 @@ var _ = Describe("TLSWriter", func() { }, egressCounter, syslog.NewConverter(), + emitter, ) defer writer.Close() diff --git a/src/pkg/egress/syslog/writer_factory.go b/src/pkg/egress/syslog/writer_factory.go index 8ec8249ab..7f40618cd 100644 --- a/src/pkg/egress/syslog/writer_factory.go +++ b/src/pkg/egress/syslog/writer_factory.go @@ -7,6 +7,7 @@ import ( metrics "code.cloudfoundry.org/go-metric-registry" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" ) type metricClient interface { @@ -52,7 +53,7 @@ func NewWriterFactory(internalTlsConfig *tls.Config, externalTlsConfig *tls.Conf } } -func (f WriterFactory) NewWriter(ub *URLBinding) (egress.WriteCloser, error) { +func (f WriterFactory) NewWriter(ub *URLBinding, emitter applog.LogEmitter) (egress.WriteCloser, error) { tlsCfg := f.externalTlsConfig.Clone() if ub.InternalTls { tlsCfg = f.internalTlsConfig.Clone() @@ -60,7 +61,8 @@ func (f WriterFactory) NewWriter(ub *URLBinding) (egress.WriteCloser, error) { if len(ub.Certificate) > 0 && len(ub.PrivateKey) > 0 { cert, err := tls.X509KeyPair(ub.Certificate, ub.PrivateKey) if err != nil { - err = NewWriterFactoryErrorf(ub.URL, "failed to load certificate: %s", err.Error()) + errorMessage := err.Error() + err = NewWriterFactoryErrorf(ub.URL, "failed to load certificate: %s", errorMessage) return nil, err } tlsCfg.Certificates = []tls.Certificate{cert} @@ -120,6 +122,7 @@ func (f WriterFactory) NewWriter(ub *URLBinding) (egress.WriteCloser, error) { f.netConf, egressMetric, converter, + emitter, ) case "syslog-tls": w = NewTLSWriter( @@ -128,6 +131,7 @@ func (f WriterFactory) NewWriter(ub *URLBinding) (egress.WriteCloser, error) { tlsCfg, egressMetric, converter, + emitter, ) } diff --git a/src/pkg/egress/syslog/writer_factory_test.go b/src/pkg/egress/syslog/writer_factory_test.go index 5707bbf56..7f3df1a9a 100644 --- a/src/pkg/egress/syslog/writer_factory_test.go +++ b/src/pkg/egress/syslog/writer_factory_test.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "net/url" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/applog" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -13,8 +14,9 @@ import ( var _ = Describe("EgressFactory", func() { var ( - f syslog.WriterFactory - sm *metricsHelpers.SpyMetricsRegistry + f syslog.WriterFactory + sm *metricsHelpers.SpyMetricsRegistry + emitter applog.LogEmitter ) BeforeEach(func() { @@ -30,7 +32,7 @@ var _ = Describe("EgressFactory", func() { URL: url, } - writer, err := f.NewWriter(urlBinding) + writer, err := f.NewWriter(urlBinding, emitter) Expect(err).ToNot(HaveOccurred()) retryWriter, ok := writer.(*syslog.RetryWriter) @@ -49,7 +51,7 @@ var _ = Describe("EgressFactory", func() { URL: url, } - writer, err := f.NewWriter(urlBinding) + writer, err := f.NewWriter(urlBinding, emitter) Expect(err).ToNot(HaveOccurred()) retryWriter, ok := writer.(*syslog.RetryWriter) @@ -68,7 +70,7 @@ var _ = Describe("EgressFactory", func() { URL: url, } - writer, err := f.NewWriter(urlBinding) + writer, err := f.NewWriter(urlBinding, emitter) Expect(err).ToNot(HaveOccurred()) retryWriter, ok := writer.(*syslog.RetryWriter) @@ -87,7 +89,7 @@ var _ = Describe("EgressFactory", func() { URL: url, } - writer, err := f.NewWriter(urlBinding) + writer, err := f.NewWriter(urlBinding, emitter) Expect(err).ToNot(HaveOccurred()) retryWriter, ok := writer.(*syslog.RetryWriter) @@ -106,7 +108,7 @@ var _ = Describe("EgressFactory", func() { Certificate: []byte("invalid-certificate"), } - _, err = f.NewWriter(urlBinding) + _, err = f.NewWriter(urlBinding, emitter) Expect(err).ToNot(HaveOccurred()) }) }) @@ -120,7 +122,7 @@ var _ = Describe("EgressFactory", func() { PrivateKey: []byte("invalid-private-key"), } - _, err = f.NewWriter(urlBinding) + _, err = f.NewWriter(urlBinding, emitter) Expect(err).ToNot(HaveOccurred()) }) }) @@ -143,7 +145,7 @@ var _ = Describe("EgressFactory", func() { urlBinding.CA = []byte("invalid-ca") } - _, err = f.NewWriter(urlBinding) + _, err = f.NewWriter(urlBinding, emitter) Expect(err).To(MatchError(expectedErr)) }, @@ -169,7 +171,7 @@ var _ = Describe("EgressFactory", func() { AppID: appID, } - _, err = f.NewWriter(urlBinding) + _, err = f.NewWriter(urlBinding, emitter) Expect(err).ToNot(HaveOccurred()) metric := sm.GetMetric("egress", tags) diff --git a/src/pkg/ingress/bindings/filtered_binding_fetcher.go b/src/pkg/ingress/bindings/filtered_binding_fetcher.go index c496ba901..db0dbc7c7 100644 --- a/src/pkg/ingress/bindings/filtered_binding_fetcher.go +++ b/src/pkg/ingress/bindings/filtered_binding_fetcher.go @@ -2,7 +2,6 @@ package bindings import ( "log" - "net" "net/url" "time" @@ -14,19 +13,13 @@ import ( var allowedSchemes = []string{"syslog", "syslog-tls", "https", "https-batch"} -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . IPChecker -type IPChecker interface { - ResolveAddr(host string) (net.IP, error) - CheckBlacklist(ip net.IP) error -} - // Metrics is the client used to expose gauge and counter metricsClient. type metricsClient interface { NewGauge(name, helpText string, opts ...metrics.MetricOption) metrics.Gauge } type FilteredBindingFetcher struct { - ipChecker IPChecker + ipChecker binding.IPChecker br binding.Fetcher warn bool logger *log.Logger @@ -35,7 +28,7 @@ type FilteredBindingFetcher struct { failedHostsCache *simplecache.SimpleCache[string, bool] } -func NewFilteredBindingFetcher(c IPChecker, b binding.Fetcher, m metricsClient, warn bool, lc *log.Logger) *FilteredBindingFetcher { +func NewFilteredBindingFetcher(c binding.IPChecker, b binding.Fetcher, m metricsClient, warn bool, lc *log.Logger) *FilteredBindingFetcher { opt := metrics.WithMetricLabels(map[string]string{"unit": "total"}) invalidDrains := m.NewGauge( diff --git a/src/pkg/ingress/bindings/filtered_binding_fetcher_test.go b/src/pkg/ingress/bindings/filtered_binding_fetcher_test.go index a17d25fdb..fbc5e2cc5 100644 --- a/src/pkg/ingress/bindings/filtered_binding_fetcher_test.go +++ b/src/pkg/ingress/bindings/filtered_binding_fetcher_test.go @@ -7,10 +7,9 @@ import ( "net" metricsHelpers "code.cloudfoundry.org/go-metric-registry/testhelpers" + "code.cloudfoundry.org/loggregator-agent-release/src/pkg/binding/bindingfakes" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/egress/syslog" "code.cloudfoundry.org/loggregator-agent-release/src/pkg/ingress/bindings" - "code.cloudfoundry.org/loggregator-agent-release/src/pkg/ingress/bindings/bindingsfakes" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -196,13 +195,13 @@ var _ = Describe("FilteredBindingFetcher", func() { Context("when the drain host fails to resolve", func() { var logBuffer bytes.Buffer var warn bool - var mockic *bindingsfakes.FakeIPChecker + var mockic *bindingfakes.FakeIPChecker BeforeEach(func() { logBuffer = bytes.Buffer{} log.SetOutput(&logBuffer) warn = true - mockic = &bindingsfakes.FakeIPChecker{} + mockic = &bindingfakes.FakeIPChecker{} mockic.ResolveAddrReturns(net.IP{}, errors.New("oof ouch ip not resolved")) })