Support and document the use of Kubernetes network policies with the Operator and Coherence clusters (#537)

Author: thegridman

.github/workflows/prometheus-tests.yaml

@@ -99,6 +99,7 @@ jobs:
 
     - name: E2E Prometheus Tests
       shell: bash
+      timeout-minutes: 45
       run: |
         make all
         make kind-load

Makefile

@@ -250,7 +250,6 @@ TOOLS_DIRECTORY   = $(CURRDIR)/build/tools
 TOOLS_BIN         = $(TOOLS_DIRECTORY)/bin
 OPERATOR_SDK_HOME = $(TOOLS_DIRECTORY)/sdk/$(UNAME_S)-$(UNAME_M)
 OPERATOR_SDK      = $(OPERATOR_SDK_HOME)/operator-sdk
-PROMETHEUS_HOME   = $(TOOLS_DIRECTORY)/prometheus
 
 # ----------------------------------------------------------------------------------------------------------------------
 # The ttl.sh images used in integration tests
@@ -309,7 +308,8 @@ TEST_SSL_SECRET := coherence-ssl-secret
 # ----------------------------------------------------------------------------------------------------------------------
 # Prometheus Operator settings (used in integration tests)
 # ----------------------------------------------------------------------------------------------------------------------
-PROMETHEUS_VERSION           ?= v0.8.0
+PROMETHEUS_VERSION           ?= v0.10.0
+PROMETHEUS_HOME               = $(TOOLS_DIRECTORY)/prometheus/$(PROMETHEUS_VERSION)
 PROMETHEUS_NAMESPACE         ?= monitoring
 PROMETHEUS_ADAPTER_VERSION   ?= 2.5.0
 GRAFANA_DASHBOARDS           ?= dashboards/grafana/
@@ -323,7 +323,7 @@ KIBANA_INDEX_PATTERN := "6abb1220-3feb-11e9-a9a3-4b1c09db6e6a"
 # ----------------------------------------------------------------------------------------------------------------------
 # MetalLB load balancer settings
 # ----------------------------------------------------------------------------------------------------------------------
-METALLB_VERSION ?= v0.10.2
+METALLB_VERSION ?= v0.12.1
 
 # ----------------------------------------------------------------------------------------------------------------------
 # Istio settings
@@ -1919,22 +1919,25 @@ get-prometheus: $(PROMETHEUS_HOME)/$(PROMETHEUS_VERSION).txt ## Download Prometh
 
 $(PROMETHEUS_HOME)/$(PROMETHEUS_VERSION).txt: $(BUILD_PROPS)
 	curl -sL https://github.com/prometheus-operator/kube-prometheus/archive/refs/tags/$(PROMETHEUS_VERSION).tar.gz -o $(BUILD_OUTPUT)/prometheus.tar.gz
-	mkdir $(PROMETHEUS_HOME)
+	mkdir -p $(PROMETHEUS_HOME)
 	tar -zxf $(BUILD_OUTPUT)/prometheus.tar.gz --directory $(PROMETHEUS_HOME) --strip-components=1
 	rm $(BUILD_OUTPUT)/prometheus.tar.gz
 	touch $(PROMETHEUS_HOME)/$(PROMETHEUS_VERSION).txt
 
 .PHONY: install-prometheus
 install-prometheus: get-prometheus ## Install Prometheus and Grafana
 	kubectl create -f $(PROMETHEUS_HOME)/manifests/setup
+	sleep 10
 	until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
 #   We create additional custom RBAC rules because the defaults do not work
 #   in an RBAC enabled cluster such as KinD
 #   See: https://prometheus-operator.dev/docs/operator/rbac/
 	kubectl create -f hack/prometheus-rbac.yaml
 	kubectl create -f $(PROMETHEUS_HOME)/manifests
 	sleep 10
+	kubectl -n monitoring get all
 	@echo "Waiting for Prometheus StatefulSet to be ready"
+	until kubectl -n monitoring get statefulset/prometheus-k8s ; do date; sleep 1; echo ""; done
 	kubectl -n monitoring rollout status statefulset/prometheus-k8s --timeout=5m
 	@echo "Waiting for Grafana Deployment to be ready"
 	kubectl -n monitoring rollout status deployment/grafana --timeout=5m

THIRD_PARTY_LICENSES.txt

@@ -333,7 +333,7 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 *****
-github.com/coreos/prometheus-operator/pkg/apis/[email protected]
+github.com/prometheus-operator/prometheus-operator/pkg/apis/[email protected]
 
 Apache License
                            Version 2.0, January 2004
@@ -538,7 +538,7 @@ Apache License
    limitations under the License.
 
 *****
-github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1@v0.38.1-0.20200424145508-7e176fda06cc
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1@v0.56.3
 
 Apache License
                            Version 2.0, January 2004
@@ -743,7 +743,7 @@ Apache License
    limitations under the License.
 
 *****
-github.com/coreos/prometheus-operator/pkg/client/versioned/scheme@v0.38.1-0.20200424145508-7e176fda06cc
+github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/scheme@v0.56.3
 
 Apache License
                            Version 2.0, January 2004
@@ -948,7 +948,7 @@ Apache License
    limitations under the License.
 
 *****
-github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1@v0.38.1-0.20200424145508-7e176fda06cc
+github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1@v0.56.3
 
 Apache License
                            Version 2.0, January 2004

api/v1/coherence_types.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * http://oss.oracle.com/licenses/upl.
  */
@@ -11,10 +11,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/go-logr/logr"
 	"github.com/go-test/deep"
 	"github.com/pkg/errors"
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -286,6 +286,17 @@ type CoherenceSpec struct {
 	// Enables the Coherence IP Monitor feature.
 	// The Operator disables the IP Monitor by default.
 	EnableIPMonitor *bool `json:"enableIpMonitor,omitempty"`
+	// LocalPort sets the Coherence unicast port.
+	// When manually configuring unicast ports, a single port is specified and the second port is automatically selected.
+	// If either of the ports are not available, then the default behavior is to select the next available port.
+	// For example, if port 9000 is configured for the first port (port1) and it is not available, then the next
+	// available port is automatically selected. The second port (port2) is automatically opened and defaults to
+	// the next available port after port1 (port1 + 1 if available).
+	LocalPort *int32 `json:"localPort,omitempty"`
+	// LocalPortAdjust sets the Coherence unicast port adjust value.
+	// To specify a range of unicast ports from which ports are selected, include a port value that represents the
+	// upper limit of the port range.
+	LocalPortAdjust *intstr.IntOrString `json:"localPortAdjust,omitempty"`
 }
 
 // IsWKAMember returns true if this deployment is a WKA list member.
@@ -371,6 +382,16 @@ func (in *CoherenceSpec) UpdateStatefulSet(deployment *Coherence, sts *appsv1.St
 		c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCohOverride, Value: *in.OverrideConfig})
 	}
 
+	// Always set the unicast ports, as we default them if not specifically set
+	if in.LocalPort != nil {
+		c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCoherenceLocalPort, Value: Int32PtrToString(in.LocalPort)})
+	}
+
+	if in.LocalPortAdjust != nil {
+		lpa := in.LocalPortAdjust
+		c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCoherenceLocalPortAdjust, Value: lpa.String()})
+	}
+
 	if in.LogLevel != nil {
 		c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCohLogLevel, Value: Int32PtrToString(in.LogLevel)})
 	}
@@ -1171,11 +1192,11 @@ type ServiceMonitorSpec struct {
 	// Interval at which metrics should be scraped
 	// See https://coreos.com/operators/prometheus/docs/latest/api.html#endpoint
 	// +optional
-	Interval string `json:"interval,omitempty"`
+	Interval monitoringv1.Duration `json:"interval,omitempty"`
 	// Timeout after which the scrape is ended
 	// See https://coreos.com/operators/prometheus/docs/latest/api.html#endpoint
 	// +optional
-	ScrapeTimeout string `json:"scrapeTimeout,omitempty"`
+	ScrapeTimeout monitoringv1.Duration `json:"scrapeTimeout,omitempty"`
 	// TLS configuration to use when scraping the endpoint
 	// See https://coreos.com/operators/prometheus/docs/latest/api.html#endpoint
 	// +optional

api/v1/coherence_webhook.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * http://oss.oracle.com/licenses/upl.
  */
@@ -11,6 +11,8 @@ import (
 	"github.com/go-test/deep"
 	"github.com/oracle/coherence-operator/pkg/operator"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -51,12 +53,29 @@ func (in *Coherence) Default() {
 			in.Spec.SetReplicas(3)
 		}
 
+		// set the default Coherence local port and local port adjust if not present
+		if in.Spec.Coherence == nil {
+			lpa := intstr.FromInt(int(DefaultUnicastPortAdjust))
+			in.Spec.Coherence = &CoherenceSpec{
+				LocalPort:       pointer.Int32(DefaultUnicastPort),
+				LocalPortAdjust: &lpa,
+			}
+		} else {
+			if in.Spec.Coherence.LocalPort == nil {
+				in.Spec.Coherence.LocalPort = pointer.Int32(DefaultUnicastPort)
+			}
+			if in.Spec.Coherence.LocalPort == nil {
+				lpa := intstr.FromInt(int(DefaultUnicastPortAdjust))
+				in.Spec.Coherence.LocalPortAdjust = &lpa
+			}
+		}
+
 		// apply a label with the hash of the spec
 		if hash, applied := EnsureHashLabel(in); applied {
 			logger.Info(fmt.Sprintf("Applied %s label", LabelCoherenceHash), "hash", hash)
 		}
 
-		// only set defaults for image names new Coherence instances
+		// only set defaults for image names in new Coherence instances
 		coherenceImage := operator.GetDefaultCoherenceImage()
 		in.Spec.EnsureCoherenceImage(&coherenceImage)
 		utilsImage := operator.GetDefaultUtilsImage()

api/v1/coherence_webhook_test.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * http://oss.oracle.com/licenses/upl.
  *
@@ -14,6 +14,7 @@ import (
 	"github.com/spf13/viper"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
 	"testing"
 )
@@ -80,6 +81,76 @@ func TestDefaultReplicasIsNotOverriddenWhenAlreadySet(t *testing.T) {
 	g.Expect(*c.Spec.Replicas).To(Equal(replicas))
 }
 
+func TestCoherenceLocalPortIsSet(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	c := coh.Coherence{}
+	c.Default()
+	g.Expect(c.Spec.Coherence).NotTo(BeNil())
+	g.Expect(*c.Spec.Coherence.LocalPort).To(Equal(coh.DefaultUnicastPort))
+}
+
+func TestCoherenceLocalPortIsNotOverridden(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	var port int32 = 1234
+
+	c := coh.Coherence{
+		Spec: coh.CoherenceResourceSpec{
+			Coherence: &coh.CoherenceSpec{
+				LocalPort: int32Ptr(port),
+			},
+		},
+	}
+	c.Default()
+	g.Expect(c.Spec.Coherence).NotTo(BeNil())
+	g.Expect(*c.Spec.Coherence.LocalPort).To(Equal(port))
+}
+
+func TestCoherenceLocalPortIsNotSetOnUpdate(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	c := coh.Coherence{}
+	c.Status.Phase = coh.ConditionTypeReady
+	c.Default()
+	g.Expect(c.Spec.Coherence).To(BeNil())
+}
+
+func TestCoherenceLocalPortAdjustIsSet(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	lpa := intstr.FromInt(int(coh.DefaultUnicastPortAdjust))
+	c := coh.Coherence{}
+	c.Default()
+	g.Expect(c.Spec.Coherence).NotTo(BeNil())
+	g.Expect(*c.Spec.Coherence.LocalPortAdjust).To(Equal(lpa))
+}
+
+func TestCoherenceLocalPortAdjustIsNotOverridden(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	lpa := intstr.FromInt(9876)
+	c := coh.Coherence{
+		Spec: coh.CoherenceResourceSpec{
+			Coherence: &coh.CoherenceSpec{
+				LocalPortAdjust: &lpa,
+			},
+		},
+	}
+	c.Default()
+	g.Expect(c.Spec.Coherence).NotTo(BeNil())
+	g.Expect(*c.Spec.Coherence.LocalPortAdjust).To(Equal(lpa))
+}
+
+func TestCoherenceLocalPortAdjustIsNotSetOnUpdate(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	c := coh.Coherence{}
+	c.Status.Phase = coh.ConditionTypeReady
+	c.Default()
+	g.Expect(c.Spec.Coherence).To(BeNil())
+}
+
 func TestCoherenceImageIsSet(t *testing.T) {
 	g := NewGomegaWithT(t)
 

api/v1/coherenceresource_utils.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * http://oss.oracle.com/licenses/upl.
  */

api/v1/coherenceresourcespec_types.go

@@ -987,6 +987,16 @@ func (in *CoherenceResourceSpec) AddEnvVarIfAbsent(envVar corev1.EnvVar) {
 	in.Env = append(in.Env, envVar)
 }
 
+// AddEnvVarIfAbsent adds the specified EnvVar to the destination slice if one with the same name does not already exist.
+func AddEnvVarIfAbsent(dest []corev1.EnvVar, envVar corev1.EnvVar) []corev1.EnvVar {
+	for _, e := range dest {
+		if e.Name == envVar.Name {
+			return dest
+		}
+	}
+	return append(dest, envVar)
+}
+
 // CreateDefaultEnv creates the default environment variables for the Coherence container.
 func (in *CoherenceResourceSpec) CreateDefaultEnv(deployment *Coherence) []corev1.EnvVar {
 	var siteURL string

api/v1/common_test.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates.
  * Licensed under the Universal Permissive License v 1.0 as shown at
  * http://oss.oracle.com/licenses/upl.
  */

api/v1/constants.go

@@ -138,6 +138,10 @@ const (
 	DefaultJmxmpPort int32 = 9099
 	// DefaultHealthPort is the default health port
 	DefaultHealthPort int32 = 6676
+	// DefaultUnicastPort is the default Coherence unicast port
+	DefaultUnicastPort int32 = 7575
+	// DefaultUnicastPortAdjust is the default Coherence unicast port adjust value
+	DefaultUnicastPortAdjust int32 = 7576
 
 	// OperatorConfigName is the Operator configuration Secret name
 	OperatorConfigName = "coherence-operator-config"
@@ -193,6 +197,8 @@ const (
 	EnvVarCohMetricsPrefix            = "COH_METRICS"
 	EnvVarCohEnabledSuffix            = "_ENABLED"
 	EnvVarCohPortSuffix               = "_PORT"
+	EnvVarCoherenceLocalPort          = "COHERENCE_LOCALPORT"
+	EnvVarCoherenceLocalPortAdjust    = "COHERENCE_LOCALPORT_ADJUST"
 	EnvVarEnableIPMonitor             = "COH_ENABLE_IPMONITOR"
 	EnvVarSuffixSSLEnabled            = "_SSL_ENABLED"
 	EnvVarSuffixSSLCerts              = "_SSL_CERTS"