Add logging configuration

Author: MateSaary

README.md

@@ -43,26 +43,26 @@ CAD consists of:
 
 1) [PagerDuty Webhooks](https://support.pagerduty.com/docs/webhooks) are used to trigger Configuration-Anomaly-Detection when a [PagerDuty incident](https://support.pagerduty.com/docs/incidents) is created
 2) The webhook routes to a [Tekton EventListener](https://tekton.dev/docs/triggers/eventlisteners/)
-3) Received webhooks are filtered by a [Tekton Interceptor](https://tekton.dev/docs/triggers/interceptors/) that uses the payload to evaluate whether the alert has an implemented handler function in `cadctl` or not. If there is no handler implemented, the alert is directly forwarded to a human SRE. 
+3) Received webhooks are filtered by a [Tekton Interceptor](https://tekton.dev/docs/triggers/interceptors/) that uses the payload to evaluate whether the alert has an implemented handler function in `cadctl` or not, and validates the webhook against the `X-PagerDuty-Signature` header. If there is no handler implemented, the alert is directly forwarded to a human SRE.
 4) If `cadctl` implements a handler for the received payload/alert, a [Tekton PipelineRun](https://tekton.dev/docs/pipelines/pipelineruns/) is started.
-5) The pipeline runs `cadctl` which determines the handler function by itself based on the payload. 
+5) The pipeline runs `cadctl` which determines the handler function by itself based on the payload.
 
 ![CAD Overview](./images/cad_overview/cad_architecture_dark.png#gh-dark-mode-only)
 ![CAD Overview](./images/cad_overview/cad_architecture_light.png#gh-light-mode-only)
 
-## Contributing 
+## Contributing
 
 ### Building
 
-For build targets, see `make help`. 
+For build targets, see `make help`.
 
 ### Adding a new investigation
 
 CAD investigations are triggered by PagerDuty webhooks. Currently, CAD supports the following two formats of webhooks:
--  WebhookV3 
+-  WebhookV3
 -  EventOrchestrationWebhook
 
-The required investigation is identified by CAD based on the incident and its payload. 
+The required investigation is identified by CAD based on the incident and its payload.
 As PagerDuty itself does not provide finer granularity for webhooks than service-based, CAD filters out the alerts it should investigate. For more information, please refer to https://support.pagerduty.com/docs/webhooks.
 
 To add a new alert investigation:
@@ -75,7 +75,7 @@ To add a new alert investigation:
 - an existing cluster
 - an existing PagerDuty incident for the cluster and alert type that is being tested
 
-To quickly create an incident for a cluster_id, you can run `./test/generate_incident.sh <alertname> <clusterid>`. 
+To quickly create an incident for a cluster_id, you can run `./test/generate_incident.sh <alertname> <clusterid>`.
 Example usage:`./test/generate_incident.sh ClusterHasGoneMissing 2b94brrrrrrrrrrrrrrrrrrhkaj`.
 
 ### Running cadctl for an incident ID
@@ -90,6 +90,11 @@ Example usage:`./test/generate_incident.sh ClusterHasGoneMissing 2b94brrrrrrrrrr
   ./bin/cadctl investigate --payload-path payload
   ```
 
+### Logging levels
+
+CAD allows for different logging levels (debug, info, warn, error, fatal, panic). The log level is determind through a hierarchy, where the cli flag `log-level`
+is checked first, and if not set the optional environment variable `LOG_LEVEL` is used. If neither is set, the log level defaults to `info`.
+
 ## Documentation
 
 ### Investigations
@@ -101,7 +106,7 @@ Investigation specific documentation can be found in the according investigation
 ### Integrations
 
 * [AWS](https://github.com/aws/aws-sdk-go) -- Logging into the cluster, retreiving instance info and AWS CloudTrail events.
-* [PagerDuty](https://github.com/PagerDuty/go-pagerduty) -- Retrieving alert info, esclating or silencing incidents, and adding notes. 
+* [PagerDuty](https://github.com/PagerDuty/go-pagerduty) -- Retrieving alert info, esclating or silencing incidents, and adding notes.
 * [OCM](https://github.com/openshift-online/ocm-sdk-go) -- Retrieving cluster info, sending service logs, and managing (post, delete) limited support reasons.
 * [osd-network-verifier](https://github.com/openshift/osd-network-verifier) -- Tool to verify the pre-configured networking components for ROSA and OSD CCS clusters.
 
@@ -159,3 +164,5 @@ Grafana dashboard configmaps are stored in the [Dashboards](./dashboards/) direc
 - `CAD_EXPERIMENTAL_ENABLED`: enables experimental investigations when set to `true`, see mapping.go
 
 For Red Hat employees, these environment variables can be found in the SRE-P vault.
+
+- `LOG_LEVEL`: refers to the CAD log level, if not set, the default is `info`. See

cadctl/cmd/investigate/investigate.go

@@ -43,21 +43,26 @@ var InvestigateCmd = &cobra.Command{
 }
 
 var (
-	logLevelString = "info"
-	payloadPath    = "./payload.json"
+	logLevelFlag = ""
+	payloadPath  = "./payload.json"
 )
 
 func init() {
 	InvestigateCmd.Flags().StringVarP(&payloadPath, "payload-path", "p", payloadPath, "the path to the payload")
-	InvestigateCmd.Flags().StringVarP(&logLevelString, "log-level", "l", logLevelString, "the log level [debug,info,warn,error,fatal], default = info")
+	InvestigateCmd.Flags().StringVarP(&logging.LogLevelString, "log-level", "l", "", "the log level [debug,info,warn,error,fatal], default = info")
 
 	err := InvestigateCmd.MarkFlagRequired("payload-path")
 	if err != nil {
 		logging.Warn("Could not mark flag 'payload-path' as required")
 	}
 }
 
-func run(_ *cobra.Command, _ []string) error {
+func run(cmd *cobra.Command, _ []string) error {
+	// early init of logger for logs before clusterID is known
+	if cmd.Flags().Changed("log-level") {
+		flagValue, _ := cmd.Flags().GetString("log-level")
+		logging.RawLogger = logging.InitLogger(flagValue, "")
+	}
 	payload, err := os.ReadFile(payloadPath)
 	if err != nil {
 		return fmt.Errorf("failed to read webhook payload: %w", err)
@@ -106,8 +111,13 @@ func run(_ *cobra.Command, _ []string) error {
 	// For installing clusters, externalID can be empty.
 	internalClusterID := cluster.ID()
 
-	// initialize logger for the internal-cluster-id context
-	logging.RawLogger = logging.InitLogger(logLevelString, internalClusterID)
+	// re-initialize logger for the internal-cluster-id context
+	// if log-level flag is set, take priority over env + default
+	if cmd.Flags().Changed("log-level") {
+		logging.RawLogger = logging.InitLogger(logLevelFlag, internalClusterID)
+	} else {
+		logging.RawLogger = logging.InitLogger(logging.LogLevelString, internalClusterID)
+	}
 
 	requiresInvestigation, err := clusterRequiresInvestigation(cluster, pdClient, ocmClient)
 	if err != nil || !requiresInvestigation {

deploy/interceptor.yaml

@@ -25,6 +25,8 @@ spec:
         env:
         - name: CAD_EXPERIMENTAL_ENABLED
           value: ${CAD_EXPERIMENTAL_ENABLED}
+        - name: LOG_LEVEL
+          value: ${LOG_LEVEL}
         resources:
           limits:
             cpu: "100m"

deploy/task-cad-checks.yaml

@@ -32,6 +32,8 @@ spec:
           value: aggregation-pushgateway:9091
         - name: CAD_EXPERIMENTAL_ENABLED
           value: ${CAD_EXPERIMENTAL_ENABLED}
+        - name: LOG_LEVEL
+          value: ${LOG_LEVEL}
       # envFrom should pull all the secret information as envvars, so key names should be uppercase
       envFrom:
         - secretRef:
@@ -48,4 +50,4 @@ spec:
           memory: 64Mi
         limits:
           cpu: 100m
-          memory: 256Mi
+          memory: 256Mi

hack/update-template/main.go

@@ -32,6 +32,7 @@ var saasTemplateFile = Template{
 		{Name: "REGISTRY_IMG", Value: "quay.io/app-sre/configuration-anomaly-detection"},
 		{Name: "NAMESPACE_NAME", Value: "configuration-anomaly-detection"},
 		{Name: "CAD_EXPERIMENTAL_ENABLED", Value: "FALSE"},
+		{Name: "LOG_LEVEL", Value: "info"},
 	},
 }
 

interceptor/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"log"
 	"net"
 	"net/http"
 	"os"
@@ -12,8 +11,7 @@ import (
 	"time"
 
 	"github.com/openshift/configuration-anomaly-detection/interceptor/pkg/interceptor"
-	"go.uber.org/zap"
-	"knative.dev/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
 	"knative.dev/pkg/signals"
 )
 
@@ -24,25 +22,13 @@ const (
 	idleTimeout  = 60 * time.Second
 )
 
-var logger = &zap.SugaredLogger{}
+var logger = logging.InitLogger(logging.LogLevelString, "")
 
 func main() {
 	// set up signals so we handle the first shutdown signal gracefully
 	ctx := signals.NewContext()
 
-	zap, err := zap.NewProduction()
-	if err != nil {
-		log.Fatalf("failed to initialize logger: %s", err)
-	}
-	logger = zap.Sugar()
-	ctx = logging.WithLogger(ctx, logger)
-	defer func() {
-		if err := logger.Sync(); err != nil {
-			log.Fatalf("failed to sync the logger: %s", err)
-		}
-	}()
-
-	service := interceptor.PagerDutyInterceptor{Logger: logger}
+	service := interceptor.PagerDutyInterceptor{}
 	mux := http.NewServeMux()
 	mux.Handle("/", service)
 	mux.HandleFunc("/ready", readinessHandler)

interceptor/pkg/interceptor/pdinterceptor.go

@@ -13,37 +13,34 @@ import (
 
 	"github.com/PagerDuty/go-pagerduty/webhookv3"
 	investigations "github.com/openshift/configuration-anomaly-detection/pkg/investigations"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
 	"github.com/openshift/configuration-anomaly-detection/pkg/pagerduty"
 	triggersv1 "github.com/tektoncd/triggers/pkg/apis/triggers/v1beta1"
 	"github.com/tektoncd/triggers/pkg/interceptors"
 	"google.golang.org/grpc/codes"
-
-	"go.uber.org/zap"
 )
 
 // ErrInvalidContentType is returned when the content-type is not a JSON body.
 var ErrInvalidContentType = errors.New("form parameter encoding not supported, please change the hook to send JSON payloads")
 
-type PagerDutyInterceptor struct {
-	Logger *zap.SugaredLogger
-}
+type PagerDutyInterceptor struct{}
 
 func (pdi PagerDutyInterceptor) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	b, err := pdi.executeInterceptor(r)
 	if err != nil {
 		var e Error
 		if errors.As(err, &e) {
-			pdi.Logger.Infof("HTTP %d - %s", e.Status(), e)
+			logging.Infof("HTTP %d - %s", e.Status(), e)
 			http.Error(w, e.Error(), e.Status())
 		} else {
-			pdi.Logger.Errorf("Non Status Error: %s", err)
+			logging.Errorf("Non Status Error: %s", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 		}
 	}
 
 	w.Header().Add("Content-Type", "application/json")
 	if _, err := w.Write(b); err != nil {
-		pdi.Logger.Errorf("failed to write response: %s", err)
+		logging.Errorf("failed to write response: %s", err)
 	}
 }
 
@@ -114,10 +111,8 @@ func (pdi *PagerDutyInterceptor) executeInterceptor(r *http.Request) ([]byte, er
 	var ireq triggersv1.InterceptorRequest
 
 	// logging request
-	pdi.Logger.Info("Wrapped Request header: %v", r.Header)
-	pdi.Logger.Info("Wrapped Request body: ", body.String())
-	pdi.Logger.Info("Unwrapped Request header: %v", extractedRequest.Header)
-	pdi.Logger.Info("Unwrapped Request body: ", originalReq.Body)
+	logging.Debug("Unwrapped Request header: %v", extractedRequest.Header)
+	logging.Debug("Unwrapped Request body: ", originalReq.Body)
 
 	token, _ := os.LookupEnv("PD_SIGNATURE")
 
@@ -126,13 +121,15 @@ func (pdi *PagerDutyInterceptor) executeInterceptor(r *http.Request) ([]byte, er
 		return nil, badRequest(fmt.Errorf("failed to verify signature: %w", err))
 	}
 
+	logging.Info("Signature verified successfully")
+
 	if err := json.Unmarshal(body.Bytes(), &ireq); err != nil {
 		return nil, badRequest(fmt.Errorf("failed to parse body as InterceptorRequest: %w", err))
 	}
-	pdi.Logger.Debugf("Interceptor request body is: %s", ireq.Body)
+	logging.Debugf("Interceptor request body is: %s", ireq.Body)
 
 	iresp := pdi.Process(ctx, &ireq)
-	pdi.Logger.Debugf("Interceptor response is: %+v", iresp)
+	logging.Debugf("Interceptor response is: %+v", iresp)
 	respBytes, err := json.Marshal(iresp)
 	if err != nil {
 		return nil, internal(err)
@@ -151,18 +148,18 @@ func (pdi *PagerDutyInterceptor) Process(ctx context.Context, r *triggersv1.Inte
 	// If the alert is not in the whitelist, return `Continue: false` as interceptor response
 	// and escalate the alert to SRE
 	if investigation == nil {
-		pdi.Logger.Infof("Incident %s is not mapped to an investigation, escalating incident and returning InterceptorResponse `Continue: false`.", pdClient.GetIncidentID())
+		logging.Infof("Incident %s is not mapped to an investigation, escalating incident and returning InterceptorResponse `Continue: false`.", pdClient.GetIncidentID())
 		err = pdClient.EscalateIncidentWithNote("🤖 No automation implemented for this alert; escalated to SRE. 🤖")
 		if err != nil {
-			pdi.Logger.Errorf("failed to escalate incident '%s': %w", pdClient.GetIncidentID(), err)
+			logging.Errorf("failed to escalate incident '%s': %w", pdClient.GetIncidentID(), err)
 		}
 
 		return &triggersv1.InterceptorResponse{
 			Continue: false,
 		}
 	}
 
-	pdi.Logger.Infof("Incident %s is mapped to investigation '%s', returning InterceptorResponse `Continue: true`.", pdClient.GetIncidentID(), investigation.Name)
+	logging.Infof("Incident %s is mapped to investigation '%s', returning InterceptorResponse `Continue: true`.", pdClient.GetIncidentID(), investigation.Name)
 	return &triggersv1.InterceptorResponse{
 		Continue: true,
 	}

openshift/template.yaml

@@ -11,6 +11,8 @@ parameters:
   value: configuration-anomaly-detection
 - name: CAD_EXPERIMENTAL_ENABLED
   value: "FALSE"
+- name: LOG_LEVEL
+  value: info
 objects:
 - apiVersion: apps/v1
   kind: Deployment
@@ -35,6 +37,8 @@ objects:
           env:
           - name: CAD_EXPERIMENTAL_ENABLED
             value: ${CAD_EXPERIMENTAL_ENABLED}
+          - name: LOG_LEVEL
+            value: ${LOG_LEVEL}
           envFrom:
           - secretRef:
               name: cad-pd-token
@@ -318,6 +322,8 @@ objects:
         value: aggregation-pushgateway:9091
       - name: CAD_EXPERIMENTAL_ENABLED
         value: ${CAD_EXPERIMENTAL_ENABLED}
+      - name: LOG_LEVEL
+        value: ${LOG_LEVEL}
       envFrom:
       - secretRef:
           name: cad-aws-credentials

pkg/logging/logging.go

@@ -10,8 +10,10 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
+var LogLevelString = getLogLevel()
+
 // RawLogger is the raw global logger object used for calls wrapped by the logging package
-var RawLogger = InitLogger("info", "")
+var RawLogger = InitLogger(LogLevelString, "")
 
 // InitLogger initializes a cluster-id specific child logger
 func InitLogger(logLevelString string, clusterID string) *zap.SugaredLogger {
@@ -92,3 +94,11 @@ func Errorf(template string, args ...interface{}) {
 func Fatalf(template string, args ...interface{}) {
 	RawLogger.Fatalf(template, args...)
 }
+
+// getLogLevel returns the log level from the environment variable LOG_LEVEL
+func getLogLevel() string {
+	if envLogLevel, exists := os.LookupEnv("LOG_LEVEL"); exists {
+		return envLogLevel
+	}
+	return "info"
+}