feat: support notifications on rollout events using notifications-engine (argoproj#1175)

feat: support notifications on rollout events using notifications-engine (argoproj#1175)

Signed-off-by: Alexander Matyushentsev <[email protected]>

Author: Alexander Matyushentsev

Makefile

@@ -197,9 +197,9 @@ plugin-darwin: ui/dist
 	cp -r ui/dist/app/* server/static
 	CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME}-darwin-amd64 ./cmd/kubectl-argo-rollouts
 
-.PHONY: plugin-docs
-plugin-docs:
-	go run ./hack/gen-plugin-docs/main.go
+.PHONY: docs
+docs:
+	go run ./hack/gen-docs/main.go
 
 .PHONY: builder-image
 builder-image:
@@ -259,7 +259,7 @@ clean:
 precheckin: test lint
 
 .PHONY: release-docs
-release-docs: plugin-docs
+release-docs: docs
 	docker run --rm -it \
 		-v ~/.ssh:/root/.ssh \
 		-v ${CURRENT_DIR}:/docs \

cmd/rollouts-controller/main.go

@@ -135,6 +135,13 @@ func newCommand() *cobra.Command {
 			clusterDynamicInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, resyncDuration, metav1.NamespaceAll, instanceIDTweakListFunc)
 			// 3. We finally need an istio dynamic informer factory which does not use a tweakListFunc.
 			istioDynamicInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, resyncDuration, namespace, nil)
+
+			controllerNamespaceInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
+				kubeClient,
+				resyncDuration,
+				kubeinformers.WithNamespace(defaults.Namespace()))
+			configMapInformer := controllerNamespaceInformerFactory.Core().V1().ConfigMaps()
+			secretInformer := controllerNamespaceInformerFactory.Core().V1().Secrets()
 			cm := controller.NewManager(
 				namespace,
 				kubeClient,
@@ -153,6 +160,8 @@ func newCommand() *cobra.Command {
 				tolerantinformer.NewTolerantClusterAnalysisTemplateInformer(clusterDynamicInformerFactory),
 				istioDynamicInformerFactory.ForResource(istioutil.GetIstioVirtualServiceGVR()).Informer(),
 				istioDynamicInformerFactory.ForResource(istioutil.GetIstioDestinationRuleGVR()).Informer(),
+				configMapInformer,
+				secretInformer,
 				resyncDuration,
 				instanceID,
 				metricsPort,
@@ -166,6 +175,7 @@ func newCommand() *cobra.Command {
 				clusterDynamicInformerFactory.Start(stopCh)
 			}
 			kubeInformerFactory.Start(stopCh)
+			controllerNamespaceInformerFactory.Start(stopCh)
 			jobInformerFactory.Start(stopCh)
 
 			// Check if Istio installed on cluster before starting dynamicInformerFactory

controller/controller.go

@@ -1,15 +1,17 @@
 package controller
 
 import (
+	"encoding/json"
 	"fmt"
 	"time"
 
-	"github.com/argoproj/argo-rollouts/utils/queue"
-
+	"github.com/argoproj/notifications-engine/pkg/api"
+	"github.com/argoproj/notifications-engine/pkg/controller"
 	"github.com/pkg/errors"
 	smiclientset "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned"
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -28,11 +30,14 @@ import (
 	"github.com/argoproj/argo-rollouts/controller/metrics"
 	"github.com/argoproj/argo-rollouts/experiments"
 	"github.com/argoproj/argo-rollouts/ingress"
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
 	clientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned"
 	rolloutscheme "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/scheme"
 	informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions/rollouts/v1alpha1"
 	"github.com/argoproj/argo-rollouts/rollout"
 	"github.com/argoproj/argo-rollouts/service"
+	"github.com/argoproj/argo-rollouts/utils/defaults"
+	"github.com/argoproj/argo-rollouts/utils/queue"
 	"github.com/argoproj/argo-rollouts/utils/record"
 )
 
@@ -61,12 +66,13 @@ const (
 
 // Manager is the controller implementation for Argo-Rollout resources
 type Manager struct {
-	metricsServer        *metrics.MetricsServer
-	rolloutController    *rollout.Controller
-	experimentController *experiments.Controller
-	analysisController   *analysis.Controller
-	serviceController    *service.Controller
-	ingressController    *ingress.Controller
+	metricsServer           *metrics.MetricsServer
+	rolloutController       *rollout.Controller
+	experimentController    *experiments.Controller
+	analysisController      *analysis.Controller
+	serviceController       *service.Controller
+	ingressController       *ingress.Controller
+	notificationsController controller.NotificationController
 
 	rolloutSynced                 cache.InformerSynced
 	experimentSynced              cache.InformerSynced
@@ -77,6 +83,8 @@ type Manager struct {
 	ingressSynced                 cache.InformerSynced
 	jobSynced                     cache.InformerSynced
 	replicasSetSynced             cache.InformerSynced
+	configMapSynced               cache.InformerSynced
+	secretSynced                  cache.InformerSynced
 
 	rolloutWorkqueue     workqueue.RateLimitingInterface
 	serviceWorkqueue     workqueue.RateLimitingInterface
@@ -110,6 +118,8 @@ func NewManager(
 	clusterAnalysisTemplateInformer informers.ClusterAnalysisTemplateInformer,
 	istioVirtualServiceInformer cache.SharedIndexInformer,
 	istioDestinationRuleInformer cache.SharedIndexInformer,
+	configMapInformer coreinformers.ConfigMapInformer,
+	secretInformer coreinformers.SecretInformer,
 	resyncPeriod time.Duration,
 	instanceID string,
 	metricsPort int,
@@ -139,8 +149,22 @@ func NewManager(
 	ingressWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Ingresses")
 
 	refResolver := rollout.NewInformerBasedWorkloadRefResolver(namespace, dynamicclientset, discoveryClient, rolloutWorkqueue, rolloutsInformer.Informer())
-
-	recorder := record.NewEventRecorder(kubeclientset, metrics.MetricRolloutEventsTotal)
+	apiFactory := api.NewFactory(record.NewAPIFactorySettings(), defaults.Namespace(), secretInformer.Informer(), configMapInformer.Informer())
+	recorder := record.NewEventRecorder(kubeclientset, metrics.MetricRolloutEventsTotal, apiFactory)
+	notificationsController := controller.NewController(dynamicclientset.Resource(v1alpha1.RolloutGVR), rolloutsInformer.Informer(), apiFactory,
+		controller.WithToUnstructured(func(obj metav1.Object) (*unstructured.Unstructured, error) {
+			data, err := json.Marshal(obj)
+			if err != nil {
+				return nil, err
+			}
+			res := &unstructured.Unstructured{}
+			err = json.Unmarshal(data, res)
+			if err != nil {
+				return nil, err
+			}
+			return res, nil
+		}),
+	)
 
 	rolloutController := rollout.NewController(rollout.ControllerConfig{
 		Namespace:                       namespace,
@@ -229,6 +253,8 @@ func NewManager(
 		analysisTemplateSynced:        analysisTemplateInformer.Informer().HasSynced,
 		clusterAnalysisTemplateSynced: clusterAnalysisTemplateInformer.Informer().HasSynced,
 		replicasSetSynced:             replicaSetInformer.Informer().HasSynced,
+		configMapSynced:               configMapInformer.Informer().HasSynced,
+		secretSynced:                  secretInformer.Informer().HasSynced,
 		rolloutWorkqueue:              rolloutWorkqueue,
 		experimentWorkqueue:           experimentWorkqueue,
 		analysisRunWorkqueue:          analysisRunWorkqueue,
@@ -239,6 +265,7 @@ func NewManager(
 		ingressController:             ingressController,
 		experimentController:          experimentController,
 		analysisController:            analysisController,
+		notificationsController:       notificationsController,
 		dynamicClientSet:              dynamicclientset,
 		refResolver:                   refResolver,
 		namespace:                     namespace,
@@ -260,7 +287,7 @@ func (c *Manager) Run(rolloutThreadiness, serviceThreadiness, ingressThreadiness
 
 	// Wait for the caches to be synced before starting workers
 	log.Info("Waiting for controller's informer caches to sync")
-	if ok := cache.WaitForCacheSync(stopCh, c.serviceSynced, c.ingressSynced, c.jobSynced, c.rolloutSynced, c.experimentSynced, c.analysisRunSynced, c.analysisTemplateSynced, c.replicasSetSynced); !ok {
+	if ok := cache.WaitForCacheSync(stopCh, c.serviceSynced, c.ingressSynced, c.jobSynced, c.rolloutSynced, c.experimentSynced, c.analysisRunSynced, c.analysisTemplateSynced, c.replicasSetSynced, c.configMapSynced, c.secretSynced); !ok {
 		return fmt.Errorf("failed to wait for caches to sync")
 	}
 	// only wait for cluster scoped informers to sync if we are running in cluster-wide mode
@@ -277,6 +304,8 @@ func (c *Manager) Run(rolloutThreadiness, serviceThreadiness, ingressThreadiness
 	go wait.Until(func() { c.ingressController.Run(ingressThreadiness, stopCh) }, time.Second, stopCh)
 	go wait.Until(func() { c.experimentController.Run(experimentThreadiness, stopCh) }, time.Second, stopCh)
 	go wait.Until(func() { c.analysisController.Run(analysisThreadiness, stopCh) }, time.Second, stopCh)
+	go wait.Until(func() { c.notificationsController.Run(rolloutThreadiness, stopCh) }, time.Second, stopCh)
+
 	log.Info("Started controller")
 
 	go func() {

docs/features/notifications.md

@@ -0,0 +1,134 @@
+# Notifications
+
+!!! important
+    Available since v1.1
+
+Argo Rollouts provides notifications powered by the [Notifications Engine](https://github.com/argoproj/notifications-engine).
+Controller administrators can leverage flexible systems of triggers and templates to configure notifications requested
+by the end users. The end-users can subscribe to the configured triggers by adding an annotation to the Rollout objects.
+
+## Configuration
+
+The trigger defines the condition when the notification should be sent as well as the notification content template.
+Default Argo Rollouts comes with a list of built-in triggers that cover the most important events of Argo Rollout live-cycle.
+Both triggers and templates are configured in the `argo-rollouts-notification-configmap` ConfigMap. In order to get
+started quickly, you can use pre-configured notification templates defined in [notifications-install.yaml](../../manifests/notifications-install.yaml).
+
+If you are leveraging Kustomize it is recommended to include [notifications-install.yaml](../../manifests/notifications-install.yaml) as a remote
+resource into your `kustomization.yaml` file:
+
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- https://github.com/argoproj/argo-rollouts/releases/latest/download/install.yaml
+- https://github.com/argoproj/argo-rollouts/releases/latest/download/notifications-install.yaml
+```
+
+After including the `argo-rollouts-notification-configmap` ConfigMap the administrator needs to configure integration
+with the required notifications service such as Slack or MS Teams. An example below demonstrates Slack integration:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argo-rollouts-notification-configmap
+data:
+  service.slack: |
+    token: $slack-token
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: argo-rollouts-notification-secret
+stringData:
+  slack-token: <my-slack-token>
+```
+
+Learn more about supported services and configuration settings in services [documentation](../generated/notification-services/overview.md).
+
+## Subscriptions
+
+The end-users can start leveraging notifications using `notifications.argoproj.io/subscribe.<trigger>.<service>: <recipient>` annotation.
+For example, the following annotation subscribes two Slack channels to notifications about canary rollout step completion:
+
+```yaml
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  name: rollout-canary
+  annotations:
+    notifications.argoproj.io/subscribe.on-rollout-step-completed.slack: my-channel1;my-channel2
+
+```
+
+Annotation key consists of following parts:
+
+* `on-rollout-step-completed` - trigger name
+* `slack` - notification service name
+* `my-channel1;my-channel2` - a semicolon separated list of recipients
+
+## Customization
+
+The Rollout administrator can customize the notifications by configuring notification templates and custom triggers
+in `argo-rollouts-notification-configmap` ConfigMap.
+
+### Templates
+
+The notification template is a stateless function that generates the notification content. The template is leveraging
+[html/template](https://golang.org/pkg/html/template/) golang package. It is meant to be reusable and can be referenced by multiple triggers.
+
+An example below demonstrates a sample template:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argo-rollouts-notification-configmap
+data:
+  template.my-purple-template: |
+    message: |
+      Rollout {{.rollout.metadata.name}} has purple image
+    slack:
+        attachments: |
+            [{
+              "title": "{{ .rollout.metadata.name}}",
+              "color": "#800080"
+            }]
+```
+
+Each template has access to the following fields:
+
+- `rollout` holds the rollout object.
+- `recipient` holds the recipient name.
+
+The `message` field of the template definition allows creating a basic notification for any notification service. You can
+leverage notification service-specific fields to create complex notifications. For example using service-specific you can
+add blocks and attachments for Slack, subject for Email or URL path, and body for Webhook. See corresponding service
+[documentation](../generated/notification-services/overview.md) for more information.
+
+### Custom Triggers
+
+In addition to custom notification template administrator and configure custom triggers. Custom trigger defines the
+condition when the notification should be sent. The definition includes name, condition and notification templates reference.
+The condition is a predicate expression that returns true if the notification should be sent. The trigger condition
+evaluation is powered by [antonmedv/expr](https://github.com/antonmedv/expr).
+The condition language syntax is described at [Language-Definition.md](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md).
+
+The trigger is configured in `argo-rollouts-notification-configmap` ConfigMap. For example the following trigger sends a notification
+when rollout pod spec uses `argoproj/rollouts-demo:purple` image:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argocd-notifications-cm
+data:
+  trigger.on-purple: |
+    - send: [my-purple-template]
+      when: rollout.spec.template.spec.containers[0].image == 'argoproj/rollouts-demo:purple'
+```
+
+Each condition might use several templates. Typically each template is responsible for generating a service-specific notification part.

examples/notifications/configmap.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argo-rollouts-notification-configmap
+data:
+  service.slack: |
+    token: $slack-token
+  ## Custom Trigger
+  trigger.on-purple: |
+    - send: [my-purple-template]
+      when: rollout.spec.template.spec.containers[0].image == 'argoproj/rollouts-demo:purple'
+  template.my-purple-template: |
+    message: |
+      Rollout {{.rollout.metadata.name}} has purple image
+    slack:
+        attachments: |
+            [{
+              "title": "{{ .rollout.metadata.name}}",
+              "color": "#800080"
+            }]

examples/notifications/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../manifests/notifications
+
+patchesStrategicMerge:
+- configmap.yaml