From a54b73c6432529f83a46ac3b1d5e48ba41e41987 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Mon, 28 Jul 2025 09:53:27 +0200 Subject: [PATCH 01/19] Kubernetes: refactor local cluster deployment Changes * Use calico CNI for networking as we use it in master deployments * Move local-reated targets to a seperate Makefile (keep clean main Makefile used in master/stag/prod) * Add delete (local) cluster target --- charts/Makefile | 25 +----------------------- charts/README.md | 5 +---- charts/local-k8s.Makefile | 28 +++++++++++++++++++++++++++ scripts/create_local_k8s_cluster.bash | 23 ++++++++++++++++++++-- scripts/kind_config.yaml | 6 ++++++ 5 files changed, 57 insertions(+), 30 deletions(-) create mode 100644 charts/local-k8s.Makefile diff --git a/charts/Makefile b/charts/Makefile index 5ca4a3b5..39844abb 100644 --- a/charts/Makefile +++ b/charts/Makefile @@ -1,3 +1,4 @@ +# to be executed on kubernetes control nodes REPO_BASE_DIR := $(shell git rev-parse --show-toplevel) include ${REPO_BASE_DIR}/scripts/common.Makefile @@ -23,40 +24,16 @@ helmfile-lint: .check-helmfile-installed helmfile.yaml ## Lints the helmfile set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile lint -.PHONY: .helmfile-local-post-install -.helmfile-local-post-install: ## Post install steps for local helmfile deployment - @$(MAKE) -s configure-local-hosts - @echo ""; - @echo "Cluster has been deployed locally: https://$(MACHINE_FQDN)"; - @echo " For secure connections self-signed certificates are used."; - @echo ""; - .PHONY: helmfile-apply helmfile-apply: .check-helmfile-installed helmfile.yaml ## Applies the helmfile configuration set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml apply - @if [ "$(MACHINE_FQDN)" = "osparc.local" ]; then \ - $(MAKE) -s .helmfile-local-post-install; \ - fi - .PHONY: helmfile-sync helmfile-sync: .check-helmfile-installed helmfile.yaml ## Syncs the helmfile configuration (use `helmfile-apply` to deploy the app) set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml sync - @if [ "$(MACHINE_FQDN)" = "osparc.local" ]; then \ - $(MAKE) -s .helmfile-local-post-install; \ - fi - -.PHONY: configure-local-hosts -configure-local-hosts: $(REPO_CONFIG_LOCATION) ## Adds local hosts entries for the machine - # "Updating /etc/hosts with k8s $(MACHINE_FQDN) hosts ..." - @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ - grep -q "127.0.0.1 $$K8S_MONITORING_FQDN" /etc/hosts || echo "127.0.0.1 $$K8S_MONITORING_FQDN" | sudo tee -a /etc/hosts - @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ - grep -q "127.0.0.1 $$K8S_PRIVATE_FQDN" /etc/hosts || echo "127.0.0.1 $$K8S_PRIVATE_FQDN" | sudo tee -a /etc/hosts - .PHONY: helmfile-diff helmfile-diff: .check-helmfile-installed helmfile.yaml ## Shows the differences that would be applied by helmfile @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ diff --git a/charts/README.md b/charts/README.md index 45d83630..8ee43cc8 100644 --- a/charts/README.md +++ b/charts/README.md @@ -45,7 +45,4 @@ helmfile init ## Running k8s cluster locally -```bash -cd ./osparc-ops-environments -./scripts/create_local_k8s_cluster.bash -``` +Use `./local-k8s.Makefile` targets diff --git a/charts/local-k8s.Makefile b/charts/local-k8s.Makefile new file mode 100644 index 00000000..dd83b852 --- /dev/null +++ b/charts/local-k8s.Makefile @@ -0,0 +1,28 @@ +REPO_BASE_DIR := $(shell git rev-parse --show-toplevel) +K8S_CLUSTER_NAME := osparc-cluster +# Determine this makefile's path. +# Be sure to place this BEFORE `include` directives, if any. +THIS_MAKEFILE := $(lastword $(MAKEFILE_LIST)) + +include ${REPO_BASE_DIR}/scripts/common.Makefile +include $(REPO_CONFIG_LOCATION) + +create-cluster: ## Creates a local Kubernetes cluster + @$(REPO_BASE_DIR)/scripts/create_local_k8s_cluster.bash $(K8S_CLUSTER_NAME) + @$(MAKE) --no-print-directory --file $(THIS_MAKEFILE) configure-local-hosts + @echo ""; + @echo "Cluster has been deployed locally: https://$(MACHINE_FQDN)"; + @echo " For secure connections self-signed certificates are used."; + @echo ""; + +delete-cluster: ## Deletes the local Kubernetes cluster + @kind delete cluster --name $(K8S_CLUSTER_NAME) + @echo "Local Kubernetes cluster $(K8S_CLUSTER_NAME) has been deleted." + +.PHONY: configure-local-hosts +configure-local-hosts: $(REPO_CONFIG_LOCATION) ## Adds local hosts entries for the machine + # "Updating /etc/hosts with k8s $(MACHINE_FQDN) hosts ..." + @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ + grep -q "127.0.0.1 $$K8S_MONITORING_FQDN" /etc/hosts || echo "127.0.0.1 $$K8S_MONITORING_FQDN" | sudo tee -a /etc/hosts + @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ + grep -q "127.0.0.1 $$K8S_PRIVATE_FQDN" /etc/hosts || echo "127.0.0.1 $$K8S_PRIVATE_FQDN" | sudo tee -a /etc/hosts diff --git a/scripts/create_local_k8s_cluster.bash b/scripts/create_local_k8s_cluster.bash index 028f446a..049853c2 100755 --- a/scripts/create_local_k8s_cluster.bash +++ b/scripts/create_local_k8s_cluster.bash @@ -6,7 +6,7 @@ set -o pipefail THIS_SCRIPT_DIR=$(dirname "$0") KIND_CONFIG_FILE="$THIS_SCRIPT_DIR/kind_config.yaml" -KIND_CLUSTER_NAME="kind" +KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-osparc-cluster}" if ! command -v kind &> /dev/null; then echo "Error: kind is not installed. Please install kind and try again." @@ -19,7 +19,7 @@ if ! command -v kubectl &> /dev/null; then fi if kind get clusters | grep -q "^$KIND_CLUSTER_NAME$"; then - echo "A cluster is already up." + echo "A cluster '$KIND_CLUSTER_NAME' is already up." exit 0 fi @@ -28,5 +28,24 @@ if [[ ! -f "$KIND_CONFIG_FILE" ]]; then exit 1 fi +# # create a k8s cluster +# + +echo "Creating a local Kubernetes cluster named '$KIND_CLUSTER_NAME' using configuration from '$KIND_CONFIG_FILE'..." + kind create cluster --config "$KIND_CONFIG_FILE" --name "$KIND_CLUSTER_NAME" + +# +# install Calico network CNI +# + +# https://archive-os-3-26.netlify.app/calico/3.26/getting-started/kubernetes/kind/ + +echo "Installing Calico network CNI ..." + +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/tigera-operator.yaml +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/custom-resources.yaml + +echo "Waiting for Calico pods to start..." +while ! kubectl get pods -A -l k8s-app=calico-node 2>/dev/null | grep -q "Running"; do sleep 1; done diff --git a/scripts/kind_config.yaml b/scripts/kind_config.yaml index a5c9dae2..bee02c23 100644 --- a/scripts/kind_config.yaml +++ b/scripts/kind_config.yaml @@ -13,3 +13,9 @@ nodes: labels: ops: "true" simcore: "true" + +# https://archive-os-3-26.netlify.app/calico/3.26/getting-started/kubernetes/kind/ +networking: + disableDefaultCNI: true + # must match with cidr in https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/custom-resources.yaml + podSubnet: 192.168.0.0/16 From 3b2a5c95910a43306696edd9e9a760ba71e25880 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 29 Jul 2025 08:15:54 +0200 Subject: [PATCH 02/19] Use newer version --- scripts/create_local_k8s_cluster.bash | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/create_local_k8s_cluster.bash b/scripts/create_local_k8s_cluster.bash index 049853c2..5f6cd529 100755 --- a/scripts/create_local_k8s_cluster.bash +++ b/scripts/create_local_k8s_cluster.bash @@ -44,8 +44,10 @@ kind create cluster --config "$KIND_CONFIG_FILE" --name "$KIND_CLUSTER_NAME" echo "Installing Calico network CNI ..." -kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/tigera-operator.yaml -kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/custom-resources.yaml +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.30.2/manifests/operator-crds.yaml +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.30.2/manifests/tigera-operator.yaml + +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.30.2/manifests/custom-resources.yaml echo "Waiting for Calico pods to start..." while ! kubectl get pods -A -l k8s-app=calico-node 2>/dev/null | grep -q "Running"; do sleep 1; done From 2b9923662e2bf6e1dc327f2ec1f75c8ebdf9a48d Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 29 Jul 2025 10:28:19 +0200 Subject: [PATCH 03/19] Update install calico link --- scripts/create_local_k8s_cluster.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_local_k8s_cluster.bash b/scripts/create_local_k8s_cluster.bash index 5f6cd529..30bdb7bb 100755 --- a/scripts/create_local_k8s_cluster.bash +++ b/scripts/create_local_k8s_cluster.bash @@ -40,7 +40,7 @@ kind create cluster --config "$KIND_CONFIG_FILE" --name "$KIND_CLUSTER_NAME" # install Calico network CNI # -# https://archive-os-3-26.netlify.app/calico/3.26/getting-started/kubernetes/kind/ +# https://docs.tigera.io/calico/3.30/getting-started/kubernetes/kind echo "Installing Calico network CNI ..." From eadd4a7355adb95ef09334bb886ed9896ba3c12a Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 29 Jul 2025 14:04:34 +0200 Subject: [PATCH 04/19] Add default global network policy --- charts/.gitignore | 2 + charts/adminer/templates/networkpolicy.yaml | 20 ++++ charts/calico-configuration/.helmignore | 23 ++++ charts/calico-configuration/Chart.yaml | 24 ++++ charts/calico-configuration/README.md | 4 + .../calico-configuration/templates/NOTES.txt | 1 + .../templates/globalpolicy.yaml | 26 ++++ charts/portainer/Chart.lock | 6 + charts/portainer/Chart.yaml | 29 +++++ charts/portainer/templates/NOTES.txt | 1 + charts/portainer/templates/networkpolicy.yaml | 34 ++++++ charts/portainer/values.yaml.gotmpl | 113 +++++++++--------- 12 files changed, 226 insertions(+), 57 deletions(-) create mode 100644 charts/adminer/templates/networkpolicy.yaml create mode 100644 charts/calico-configuration/.helmignore create mode 100644 charts/calico-configuration/Chart.yaml create mode 100644 charts/calico-configuration/README.md create mode 100644 charts/calico-configuration/templates/NOTES.txt create mode 100644 charts/calico-configuration/templates/globalpolicy.yaml create mode 100644 charts/portainer/Chart.lock create mode 100644 charts/portainer/Chart.yaml create mode 100644 charts/portainer/templates/NOTES.txt create mode 100644 charts/portainer/templates/networkpolicy.yaml diff --git a/charts/.gitignore b/charts/.gitignore index 264930c8..8f77c54f 100644 --- a/charts/.gitignore +++ b/charts/.gitignore @@ -2,3 +2,5 @@ values.yaml values.*.yaml k8s_hosts.ini helmfile.y?ml + +*.tgz diff --git a/charts/adminer/templates/networkpolicy.yaml b/charts/adminer/templates/networkpolicy.yaml new file mode 100644 index 00000000..c4cd7da7 --- /dev/null +++ b/charts/adminer/templates/networkpolicy.yaml @@ -0,0 +1,20 @@ +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: adminer-network-policy + labels: + {{- include "adminer.labels" . | nindent 4 }} +spec: + selector: app.kubernetes.io/instance == "{{ .Release.Name }}" + ingress: + - action: Allow + protocol: TCP + destination: + ports: + - {{ .Values.service.port }} + egress: + - action: Allow + protocol: TCP + destination: + ports: + - 5432 diff --git a/charts/calico-configuration/.helmignore b/charts/calico-configuration/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/charts/calico-configuration/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/calico-configuration/Chart.yaml b/charts/calico-configuration/Chart.yaml new file mode 100644 index 00000000..81560549 --- /dev/null +++ b/charts/calico-configuration/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: calico-configuration +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "3.26.4" diff --git a/charts/calico-configuration/README.md b/charts/calico-configuration/README.md new file mode 100644 index 00000000..9b03884d --- /dev/null +++ b/charts/calico-configuration/README.md @@ -0,0 +1,4 @@ +## Observe network traffic + +https://docs.tigera.io/calico/latest/observability/enable-whisker +https://docs.tigera.io/calico/3.30/observability/view-flow-logs diff --git a/charts/calico-configuration/templates/NOTES.txt b/charts/calico-configuration/templates/NOTES.txt new file mode 100644 index 00000000..f03af9f5 --- /dev/null +++ b/charts/calico-configuration/templates/NOTES.txt @@ -0,0 +1 @@ +This chart configures Calico but does not deploy Calico itself. This is done via Kubespray during Kubernetes Cluster provisioning. diff --git a/charts/calico-configuration/templates/globalpolicy.yaml b/charts/calico-configuration/templates/globalpolicy.yaml new file mode 100644 index 00000000..e9254d96 --- /dev/null +++ b/charts/calico-configuration/templates/globalpolicy.yaml @@ -0,0 +1,26 @@ +# Source: https://docs.tigera.io/calico-enterprise/latest/network-policy/default-deny#best-practice-2-keep-the-scope-to-non-system-pods +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: default-global-deny-network-policy +spec: + # on local deployment, calico is installed in the `calico-system` namespace + # and the operator is in the `tigera-operator` namespace + namespaceSelector: kubernetes.io/metadata.name in {"adminer"} + types: + - Ingress + - Egress + egress: + # allow all namespaces to communicate to DNS pods + - action: Allow + protocol: UDP + destination: + selector: 'k8s-app == "kube-dns"' + ports: + - 53 + - action: Allow + protocol: TCP + destination: + selector: 'k8s-app == "kube-dns"' + ports: + - 53 diff --git a/charts/portainer/Chart.lock b/charts/portainer/Chart.lock new file mode 100644 index 00000000..dc3e362d --- /dev/null +++ b/charts/portainer/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: portainer + repository: https://portainer.github.io/k8s/ + version: 1.0.54 +digest: sha256:bafe4182881aee8c6df3d3c6f8c523a1bd7577bed04942ad3d9b857a5437d96f +generated: "2025-07-29T11:07:15.39037387+02:00" diff --git a/charts/portainer/Chart.yaml b/charts/portainer/Chart.yaml new file mode 100644 index 00000000..7a7b4f48 --- /dev/null +++ b/charts/portainer/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: portainer +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.0.54 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: 2.21.2 + +dependencies: + - name: portainer + version: 1.0.54 + repository: "https://portainer.github.io/k8s/" diff --git a/charts/portainer/templates/NOTES.txt b/charts/portainer/templates/NOTES.txt new file mode 100644 index 00000000..48340702 --- /dev/null +++ b/charts/portainer/templates/NOTES.txt @@ -0,0 +1 @@ +Wrapper around portainer helm chart https://github.com/portainer/k8s diff --git a/charts/portainer/templates/networkpolicy.yaml b/charts/portainer/templates/networkpolicy.yaml new file mode 100644 index 00000000..bbd864f0 --- /dev/null +++ b/charts/portainer/templates/networkpolicy.yaml @@ -0,0 +1,34 @@ +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: portainer-network-policy +spec: + selector: app.kubernetes.io/instance == "portainer" + types: + - Ingress + - Egress + egress: + - action: Allow + protocol: TCP + # connect to the Kubernetes API server + destination: + ports: + - 6443 + nets: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + # coredns 53 allow from kube system + - action: Allow + protocol: UDP + destination: + selector: 'k8s-app == "kube-dns"' + ports: + - 53 + ingress: + - action: Allow + # allow traffic to portainer GUI + protocol: TCP + destination: + ports: + - 9000 diff --git a/charts/portainer/values.yaml.gotmpl b/charts/portainer/values.yaml.gotmpl index 8cdd2dfc..7e5454b1 100644 --- a/charts/portainer/values.yaml.gotmpl +++ b/charts/portainer/values.yaml.gotmpl @@ -1,69 +1,68 @@ -# Default values for adminer. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. +servicePort: &servicePort 9000 -replicaCount: 1 +portainer: + replicaCount: 1 -image: - repository: portainer/portainer-ce - pullPolicy: IfNotPresent + image: + repository: portainer/portainer-ce + pullPolicy: IfNotPresent -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" -serviceAccount: - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: portainer-sa-clusteradmin + serviceAccount: + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: portainer-sa-clusteradmin -persistence: {} + persistence: {} -podAnnotations: {} -podLabels: {} + podAnnotations: {} + podLabels: {} -podSecurityContext: - {} + podSecurityContext: + {} -securityContext: - {} + securityContext: + {} -service: - type: "ClusterIP" - port: 9000 + service: + type: "ClusterIP" + port: *servicePort -ingress: - enabled: true - className: "" - annotations: - namespace: {{ .Release.Namespace }} - cert-manager.io/cluster-issuer: "cert-issuer" - traefik.ingress.kubernetes.io/router.entrypoints: websecure - traefik.ingress.kubernetes.io/router.middlewares: traefik-traefik-basic-auth@kubernetescrd,traefik-portainer-strip-prefix@kubernetescrd # namespace + middleware name - tls: - - hosts: - - {{ requiredEnv "K8S_MONITORING_FQDN" }} - secretName: monitoring-tls - hosts: - - host: {{ requiredEnv "K8S_MONITORING_FQDN" }} - paths: - - path: /portainer - pathType: Prefix - backend: - service: - name: portainer - port: - number: 9000 + ingress: + enabled: true + className: "" + annotations: + namespace: {{ .Release.Namespace }} + cert-manager.io/cluster-issuer: "cert-issuer" + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.middlewares: traefik-traefik-basic-auth@kubernetescrd,traefik-portainer-strip-prefix@kubernetescrd # namespace + middleware name + tls: + - hosts: + - {{ requiredEnv "K8S_MONITORING_FQDN" }} + secretName: monitoring-tls + hosts: + - host: {{ requiredEnv "K8S_MONITORING_FQDN" }} + paths: + - path: /portainer + pathType: Prefix + backend: + service: + name: portainer + port: + number: *servicePort -resources: - limits: - cpu: 2 - memory: 1024Mi - requests: - cpu: 0.1 - memory: 128Mi + resources: + limits: + cpu: 2 + memory: 1024Mi + requests: + cpu: 0.1 + memory: 128Mi -nodeSelector: - ops: "true" + nodeSelector: + ops: "true" From 79b333ae0947db9f3b2446684d11d1a99b29c85d Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Mon, 4 Aug 2025 15:43:19 +0200 Subject: [PATCH 05/19] Global deny network policy --- charts/calico-configuration/README.md | 21 ++++++++-- .../calico-configuration/templates/NOTES.txt | 2 + .../templates/globalpolicy.yaml | 42 ++++++++++--------- charts/portainer/templates/networkpolicy.yaml | 6 ++- 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/charts/calico-configuration/README.md b/charts/calico-configuration/README.md index 9b03884d..b0524333 100644 --- a/charts/calico-configuration/README.md +++ b/charts/calico-configuration/README.md @@ -1,4 +1,19 @@ -## Observe network traffic +## How to add network policy (local deployment) -https://docs.tigera.io/calico/latest/observability/enable-whisker -https://docs.tigera.io/calico/3.30/observability/view-flow-logs +How to discover ports / networks that are used by application +* enable and observer traffic via + - https://docs.tigera.io/calico/3.30/observability/enable-whisker + - https://docs.tigera.io/calico/3.30/observability/view-flow-logs +* add staged policies to make sure all cases are included https://docs.tigera.io/calico/3.30/network-policy/staged-network-policies + +Debug network policies: +* observe traffic and check `policies` field in whisker logs + - https://docs.tigera.io/calico/3.30/observability/enable-whisker + - https://docs.tigera.io/calico/3.30/observability/view-flow-logs + +Warning: make sure that calico version being used support Whisker (e.g. in v3.26 whisker is not documented at all) + + +## Known issues + +If network policy is created after pod, pod **MUST** be restarted for policy to take effect. Read more https://github.com/projectcalico/calico/issues/10753#issuecomment-3140717418 diff --git a/charts/calico-configuration/templates/NOTES.txt b/charts/calico-configuration/templates/NOTES.txt index f03af9f5..bba614e5 100644 --- a/charts/calico-configuration/templates/NOTES.txt +++ b/charts/calico-configuration/templates/NOTES.txt @@ -1 +1,3 @@ This chart configures Calico but does not deploy Calico itself. This is done via Kubespray during Kubernetes Cluster provisioning. + +Note: to make sure network policies are applied correctly, you may need to restart targeted application pods. diff --git a/charts/calico-configuration/templates/globalpolicy.yaml b/charts/calico-configuration/templates/globalpolicy.yaml index e9254d96..0c79841c 100644 --- a/charts/calico-configuration/templates/globalpolicy.yaml +++ b/charts/calico-configuration/templates/globalpolicy.yaml @@ -1,26 +1,30 @@ -# Source: https://docs.tigera.io/calico-enterprise/latest/network-policy/default-deny#best-practice-2-keep-the-scope-to-non-system-pods +# Source: https://docs.tigera.io/calico/3.30/network-policy/get-started/kubernetes-default-deny apiVersion: projectcalico.org/v3 kind: GlobalNetworkPolicy metadata: name: default-global-deny-network-policy spec: - # on local deployment, calico is installed in the `calico-system` namespace - # and the operator is in the `tigera-operator` namespace - namespaceSelector: kubernetes.io/metadata.name in {"adminer"} + # "kube-public", "kube-system", "kube-node-lease" -- system namespaces + # "calico-system", "calico-apiserver", "tigera-operator" -- calico namespaces (when installed via scripts [local deployment]) + # TODO: other namespaces are to be removed from this list (once appropriate network policies are created) + namespaceSelector: + kubernetes.io/metadata.name not in {"kube-public", "kube-system", "kube-node-lease", "calico-system", "calico-apiserver", "tigera-operator", "simcore", "cert-manager", "reflector", "traefik", "victoria-logs", "csi-s3", "portainer", "topolvm", "local-path-storage"} types: - - Ingress - - Egress + - Ingress + - Egress egress: - # allow all namespaces to communicate to DNS pods - - action: Allow - protocol: UDP - destination: - selector: 'k8s-app == "kube-dns"' - ports: - - 53 - - action: Allow - protocol: TCP - destination: - selector: 'k8s-app == "kube-dns"' - ports: - - 53 + # allow all namespaces to communicate to DNS pods + # this will also apply to pods that have network policy defined + # so that we don't need to define DNS policy for each pod + - action: Allow + protocol: UDP + destination: + selector: 'k8s-app == "kube-dns"' + ports: + - 53 + - action: Allow + protocol: TCP + destination: + selector: 'k8s-app == "kube-dns"' + ports: + - 53 diff --git a/charts/portainer/templates/networkpolicy.yaml b/charts/portainer/templates/networkpolicy.yaml index bbd864f0..6b21b510 100644 --- a/charts/portainer/templates/networkpolicy.yaml +++ b/charts/portainer/templates/networkpolicy.yaml @@ -22,7 +22,9 @@ spec: - action: Allow protocol: UDP destination: - selector: 'k8s-app == "kube-dns"' + # `selector: 'k8s-app == "kube-dns"'` does not work (so global policy default dns allow does not work) + # manually allow dns and use different selector that works. + selectorNamespace: kubernetes.io/metadata.name == "kube-system" ports: - 53 ingress: @@ -31,4 +33,4 @@ spec: protocol: TCP destination: ports: - - 9000 + - {{ .Values.servicePort }} From f4f980d0f49f0a4882fd79af05b36cf1359f9b9b Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Mon, 4 Aug 2025 15:47:41 +0200 Subject: [PATCH 06/19] Report progress on waiting for calico to start --- scripts/create_local_k8s_cluster.bash | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/create_local_k8s_cluster.bash b/scripts/create_local_k8s_cluster.bash index 30bdb7bb..b15c6a4e 100755 --- a/scripts/create_local_k8s_cluster.bash +++ b/scripts/create_local_k8s_cluster.bash @@ -49,5 +49,4 @@ kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.30.2 kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.30.2/manifests/custom-resources.yaml -echo "Waiting for Calico pods to start..." -while ! kubectl get pods -A -l k8s-app=calico-node 2>/dev/null | grep -q "Running"; do sleep 1; done +while ! kubectl get pods -A -l k8s-app=calico-node 2>/dev/null | grep -q "Running"; do echo "Waiting for Calico pods to start..."; sleep 1; done From cab67869bd71eaf1b9fafdb0f4474d6c87fd159e Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 5 Aug 2025 08:23:40 +0200 Subject: [PATCH 07/19] Update notes for calico configuration helm chart --- charts/calico-configuration/templates/NOTES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/calico-configuration/templates/NOTES.txt b/charts/calico-configuration/templates/NOTES.txt index bba614e5..d08265bf 100644 --- a/charts/calico-configuration/templates/NOTES.txt +++ b/charts/calico-configuration/templates/NOTES.txt @@ -1,3 +1,3 @@ -This chart configures Calico but does not deploy Calico itself. This is done via Kubespray during Kubernetes Cluster provisioning. +This chart configures Calico but does not deploy Calico itself. Calico is deployed during the Kubernetes cluster creation. Note: to make sure network policies are applied correctly, you may need to restart targeted application pods. From 0130ae34b3fdf7e6cb387f1787ea0591e5b46dc3 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 5 Aug 2025 08:25:35 +0200 Subject: [PATCH 08/19] Update calico configuration readme --- charts/calico-configuration/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/calico-configuration/README.md b/charts/calico-configuration/README.md index b0524333..e17b1466 100644 --- a/charts/calico-configuration/README.md +++ b/charts/calico-configuration/README.md @@ -1,19 +1,19 @@ ## How to add network policy (local deployment) How to discover ports / networks that are used by application -* enable and observer traffic via +* enable and observe traffic via - https://docs.tigera.io/calico/3.30/observability/enable-whisker - https://docs.tigera.io/calico/3.30/observability/view-flow-logs * add staged policies to make sure all cases are included https://docs.tigera.io/calico/3.30/network-policy/staged-network-policies +* transform staged policies to "normal" policies -Debug network policies: +## Debug network policies: * observe traffic and check `policies` field in whisker logs - https://docs.tigera.io/calico/3.30/observability/enable-whisker - https://docs.tigera.io/calico/3.30/observability/view-flow-logs Warning: make sure that calico version being used support Whisker (e.g. in v3.26 whisker is not documented at all) - ## Known issues If network policy is created after pod, pod **MUST** be restarted for policy to take effect. Read more https://github.com/projectcalico/calico/issues/10753#issuecomment-3140717418 From 28815f522ab5235fef2f9787d6cd654172cf0f63 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 5 Aug 2025 08:25:56 +0200 Subject: [PATCH 09/19] Fix typo --- charts/calico-configuration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/calico-configuration/README.md b/charts/calico-configuration/README.md index e17b1466..4e35cef9 100644 --- a/charts/calico-configuration/README.md +++ b/charts/calico-configuration/README.md @@ -7,7 +7,7 @@ How to discover ports / networks that are used by application * add staged policies to make sure all cases are included https://docs.tigera.io/calico/3.30/network-policy/staged-network-policies * transform staged policies to "normal" policies -## Debug network policies: +## Debug network policies * observe traffic and check `policies` field in whisker logs - https://docs.tigera.io/calico/3.30/observability/enable-whisker - https://docs.tigera.io/calico/3.30/observability/view-flow-logs From dbd7fccb6bf7066c993632f9192a05ef4e25484a Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 5 Aug 2025 09:13:35 +0200 Subject: [PATCH 10/19] Fix readme --- charts/calico-configuration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/calico-configuration/README.md b/charts/calico-configuration/README.md index 4e35cef9..13b8a0e1 100644 --- a/charts/calico-configuration/README.md +++ b/charts/calico-configuration/README.md @@ -12,7 +12,7 @@ How to discover ports / networks that are used by application - https://docs.tigera.io/calico/3.30/observability/enable-whisker - https://docs.tigera.io/calico/3.30/observability/view-flow-logs -Warning: make sure that calico version being used support Whisker (e.g. in v3.26 whisker is not documented at all) +Warning: make sure that calico version being used support Whisker (first introduced in v3.30) ## Known issues From df6f591d348ad9430631e42f68708b0fcaf0a9fc Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 5 Aug 2025 16:18:36 +0200 Subject: [PATCH 11/19] Add missing longhorn ns --- charts/calico-configuration/templates/globalpolicy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/calico-configuration/templates/globalpolicy.yaml b/charts/calico-configuration/templates/globalpolicy.yaml index 0c79841c..8c8973e0 100644 --- a/charts/calico-configuration/templates/globalpolicy.yaml +++ b/charts/calico-configuration/templates/globalpolicy.yaml @@ -8,7 +8,7 @@ spec: # "calico-system", "calico-apiserver", "tigera-operator" -- calico namespaces (when installed via scripts [local deployment]) # TODO: other namespaces are to be removed from this list (once appropriate network policies are created) namespaceSelector: - kubernetes.io/metadata.name not in {"kube-public", "kube-system", "kube-node-lease", "calico-system", "calico-apiserver", "tigera-operator", "simcore", "cert-manager", "reflector", "traefik", "victoria-logs", "csi-s3", "portainer", "topolvm", "local-path-storage"} + kubernetes.io/metadata.name not in {"kube-public", "kube-system", "kube-node-lease", "calico-system", "calico-apiserver", "tigera-operator", "simcore", "cert-manager", "reflector", "traefik", "victoria-logs", "csi-s3", "portainer", "topolvm", "local-path-storage", "longhorn"} types: - Ingress - Egress From 952b976cde157c522ad79ca9634f68b282ff5e78 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 5 Aug 2025 16:20:29 +0200 Subject: [PATCH 12/19] Fix portainer values --- charts/portainer/values.ebs-pv.yaml.gotmpl | 9 +++++---- charts/portainer/values.longhorn-pv.yaml.gotmpl | 9 +++++---- charts/portainer/values.s3-pv.yaml.gotmpl | 9 +++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/charts/portainer/values.ebs-pv.yaml.gotmpl b/charts/portainer/values.ebs-pv.yaml.gotmpl index b31010f6..bae9c97b 100644 --- a/charts/portainer/values.ebs-pv.yaml.gotmpl +++ b/charts/portainer/values.ebs-pv.yaml.gotmpl @@ -1,4 +1,5 @@ -persistence: - enabled: true - size: "1Gi" # minimal size for gp3 is 1Gi - storageClass: "{{ .Values.ebsStorageClassName }}" +portainer: + persistence: + enabled: true + size: "1Gi" # minimal size for gp3 is 1Gi + storageClass: "{{ .Values.ebsStorageClassName }}" diff --git a/charts/portainer/values.longhorn-pv.yaml.gotmpl b/charts/portainer/values.longhorn-pv.yaml.gotmpl index 4eae1b7b..f5d9f717 100644 --- a/charts/portainer/values.longhorn-pv.yaml.gotmpl +++ b/charts/portainer/values.longhorn-pv.yaml.gotmpl @@ -1,4 +1,5 @@ -persistence: - enabled: true - size: "300Mi" # cannot be lower https://github.com/longhorn/longhorn/issues/8488 - storageClass: "{{ .Values.longhornStorageClassName }}" +portainer: + persistence: + enabled: true + size: "300Mi" # cannot be lower https://github.com/longhorn/longhorn/issues/8488 + storageClass: "{{ .Values.longhornStorageClassName }}" diff --git a/charts/portainer/values.s3-pv.yaml.gotmpl b/charts/portainer/values.s3-pv.yaml.gotmpl index e7b6b460..11d51e48 100644 --- a/charts/portainer/values.s3-pv.yaml.gotmpl +++ b/charts/portainer/values.s3-pv.yaml.gotmpl @@ -1,4 +1,5 @@ -persistence: - enabled: true - size: "1Gi" - storageClass: "csi-s3" +portainer: + persistence: + enabled: true + size: "1Gi" + storageClass: "csi-s3" From ea116a7f4beefa2a7db64ff68e55bf2d3398e45e Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 6 Aug 2025 08:17:14 +0200 Subject: [PATCH 13/19] Allow public dns requests and imrpove calico config readme --- charts/adminer/templates/networkpolicy.yaml | 6 ++++++ charts/calico-configuration/README.md | 23 ++++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/charts/adminer/templates/networkpolicy.yaml b/charts/adminer/templates/networkpolicy.yaml index c4cd7da7..d39ed2ec 100644 --- a/charts/adminer/templates/networkpolicy.yaml +++ b/charts/adminer/templates/networkpolicy.yaml @@ -18,3 +18,9 @@ spec: destination: ports: - 5432 + # allow dns requests to public dns servers + - action: Allow + protocol: UDP + destination: + ports: + - 53 diff --git a/charts/calico-configuration/README.md b/charts/calico-configuration/README.md index 13b8a0e1..ab35651f 100644 --- a/charts/calico-configuration/README.md +++ b/charts/calico-configuration/README.md @@ -1,18 +1,31 @@ ## How to add network policy (local deployment) How to discover ports / networks that are used by application -* enable and observe traffic via - - https://docs.tigera.io/calico/3.30/observability/enable-whisker - - https://docs.tigera.io/calico/3.30/observability/view-flow-logs +* observe existing traffic (see `Debug network policies` below) * add staged policies to make sure all cases are included https://docs.tigera.io/calico/3.30/network-policy/staged-network-policies -* transform staged policies to "normal" policies + - make sure deployed calico version supports it +* based on observations, create a needed network policy ## Debug network policies + +if calico version 3.30+ is installed * observe traffic and check `policies` field in whisker logs - https://docs.tigera.io/calico/3.30/observability/enable-whisker - https://docs.tigera.io/calico/3.30/observability/view-flow-logs -Warning: make sure that calico version being used support Whisker (first introduced in v3.30) +if calico version <= 3.29 +* create network policy with action log + ```yaml + apiVersion: projectcalico.org/v3 + kind: NetworkPolicy + metadata: + name: log ingress requests + spec: + selector: app == 'db' + ingress: + - action: Log + ``` +* apply policy and see logs via journalctl (you can grep with `calico-packet`) ## Known issues From 574c5bb4fcac370bd0600c887b02bd3744143623 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 6 Aug 2025 08:23:02 +0200 Subject: [PATCH 14/19] Document how to view network policies --- charts/calico-configuration/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/charts/calico-configuration/README.md b/charts/calico-configuration/README.md index ab35651f..acec3e6b 100644 --- a/charts/calico-configuration/README.md +++ b/charts/calico-configuration/README.md @@ -30,3 +30,16 @@ if calico version <= 3.29 ## Known issues If network policy is created after pod, pod **MUST** be restarted for policy to take effect. Read more https://github.com/projectcalico/calico/issues/10753#issuecomment-3140717418 + +## How to view existing policies + +via kubectl: +* `kubectl get networkpolicies.crd.projectcalico.org -n adminer` +* `kubectl describe networkpolicies.crd.projectcalico.org -n adminer default.adminer-network-policy` + +via calicoctl: +* `calicoctl get networkpolicy -n adminer -o yaml` + +Note: +* global network policies and network policies are separate resources for calico +* To see all resources execute `kubectl get crd | grep calico` or `calicoctl get --help` From 05adb57649b58d6513bb27e6475c560df6723e27 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 6 Aug 2025 08:24:38 +0200 Subject: [PATCH 15/19] Warn to restart pods to apply network policies --- charts/calico-configuration/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/calico-configuration/README.md b/charts/calico-configuration/README.md index acec3e6b..a53de4e5 100644 --- a/charts/calico-configuration/README.md +++ b/charts/calico-configuration/README.md @@ -43,3 +43,6 @@ via calicoctl: Note: * global network policies and network policies are separate resources for calico * To see all resources execute `kubectl get crd | grep calico` or `calicoctl get --help` + +Warning: +* Network policies update are only applied to "new connections". To make them act, one may need to restart affected applications (pods) From b64c8829ee5e4b86d6d8ef6087bf040c39e74558 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 6 Aug 2025 09:43:17 +0200 Subject: [PATCH 16/19] Automaticalla restart adminer pods on network policy change --- charts/adminer/templates/deployment.yaml | 5 +++-- charts/adminer/values.yaml.gotmpl | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/charts/adminer/templates/deployment.yaml b/charts/adminer/templates/deployment.yaml index c687ad41..ca561057 100644 --- a/charts/adminer/templates/deployment.yaml +++ b/charts/adminer/templates/deployment.yaml @@ -13,9 +13,10 @@ spec: {{- include "adminer.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} + {{- if .Values.podAnnotations }} annotations: - {{- toYaml . | nindent 8 }} + # allow dynamic values in pod annotations + {{- tpl (toYaml .Values.podAnnotations) . | nindent 8 }} {{- end }} labels: {{- include "adminer.labels" . | nindent 8 }} diff --git a/charts/adminer/values.yaml.gotmpl b/charts/adminer/values.yaml.gotmpl index fb419e49..4be1bc89 100644 --- a/charts/adminer/values.yaml.gotmpl +++ b/charts/adminer/values.yaml.gotmpl @@ -25,7 +25,9 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: "" -podAnnotations: {} +podAnnotations: + # automatically restart pod on network policy change (to be sure new rules are applied) + checksum/networkpolicy: '{{`{{ include (print $.Template.BasePath "/networkpolicy.yaml") . | sha256sum }}`}}' podLabels: {} podSecurityContext: From 15336ba6f6c54710cf0a6f3af867f20b4aee8973 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 6 Aug 2025 09:44:48 +0200 Subject: [PATCH 17/19] Remove comment. It renders in final chart --- charts/adminer/templates/deployment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/adminer/templates/deployment.yaml b/charts/adminer/templates/deployment.yaml index ca561057..a084cd8f 100644 --- a/charts/adminer/templates/deployment.yaml +++ b/charts/adminer/templates/deployment.yaml @@ -15,7 +15,6 @@ spec: metadata: {{- if .Values.podAnnotations }} annotations: - # allow dynamic values in pod annotations {{- tpl (toYaml .Values.podAnnotations) . | nindent 8 }} {{- end }} labels: From 6c41bc4747836ab4cd12220516d05f8fbb2c96be Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 6 Aug 2025 09:46:57 +0200 Subject: [PATCH 18/19] Portainer: document lacking pod annotations and link PR --- charts/portainer/values.yaml.gotmpl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/portainer/values.yaml.gotmpl b/charts/portainer/values.yaml.gotmpl index 7e5454b1..303d084e 100644 --- a/charts/portainer/values.yaml.gotmpl +++ b/charts/portainer/values.yaml.gotmpl @@ -20,7 +20,10 @@ portainer: persistence: {} - podAnnotations: {} + # podAnnotations: {} + # Not implemented in portainer chart (see https://github.com/portainer/k8s/pull/183) + # Once implemented, we can use it to add checksum of network policy like in adminer + podLabels: {} podSecurityContext: From 4839a75405a914d22769099058ffd768cc46d1be Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 6 Aug 2025 09:50:36 +0200 Subject: [PATCH 19/19] Document pod annotation checksum trick --- charts/calico-configuration/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/calico-configuration/README.md b/charts/calico-configuration/README.md index a53de4e5..c722ade6 100644 --- a/charts/calico-configuration/README.md +++ b/charts/calico-configuration/README.md @@ -30,6 +30,7 @@ if calico version <= 3.29 ## Known issues If network policy is created after pod, pod **MUST** be restarted for policy to take effect. Read more https://github.com/projectcalico/calico/issues/10753#issuecomment-3140717418 +* To automate this, we can add annotations with network policy checksum to pods (see https://stackoverflow.com/questions/58602311/will-helm-upgrade-restart-pods-even-if-they-are-not-affected-by-upgrade) ## How to view existing policies