Merge branch 'dev' into audittrail-pcs

Author: zaklawrencea

cluster/cluster.yaml

@@ -714,7 +714,7 @@ Resources:
         - !Sub "arn:aws:iam::${AWS::AccountId}:policy/ZalandoCloud-AllowPowerUser"
       RoleName: "{{.Cluster.LocalID}}-deployment"
     Type: 'AWS::IAM::Role'
-{{- if eq .Cluster.ConfigItems.deployment_secret_key_managed "true" }}
+{{- if and (eq .Cluster.ConfigItems.deployment_secret_key_managed "true") (ne .Cluster.Environment "e2e") }}
   DeploymentSecretKey:
     Properties:
       Description: Key used by deployment pipeline for secret encryption/decryption
@@ -876,14 +876,16 @@ Resources:
               - Action:
                   - kms:Decrypt
                 Effect: Allow
-{{- if eq .Cluster.ConfigItems.deployment_secret_key_managed "true" }}
+{{- if and (eq .Cluster.ConfigItems.deployment_secret_key_managed "true") (ne .Cluster.Environment "e2e") }}
                 Resource:
                   - !GetAtt DeploymentSecretKey.Arn
 {{- else }}
                 Resource: "*"
+{{- if and (eq .Cluster.ConfigItems.deployment_secret_decrypt_any "false") (ne .Cluster.Environment "e2e") }}
                 Condition:
                   StringLike:
                     "kms:RequestAlias": "alias/deployment-secret"
+{{- end }}
 {{- end }}
               - Action:
                   - 'sts:AssumeRole'
@@ -1468,6 +1470,9 @@ Resources:
               - Action: 'cloudwatch:List*'
                 Effect: Allow
                 Resource: '*'
+              - Action: 'dax:DescribeClusters'
+                Effect: Allow
+                Resource: '*'
               - Action: 'ec2:Describe*'
                 Effect: Allow
                 Resource: '*'
@@ -1976,13 +1981,6 @@ Resources:
           PolicyName: root
       RoleName: "{{.Cluster.LocalID}}-audittrail-adapter"
     Type: 'AWS::IAM::Role'
-{{- if eq .Cluster.ConfigItems.dynamodb_service_link_enabled "true" }}
-  ServiceLinkedRoleAutoScalingDynamoDB:
-    Properties:
-      AWSServiceName: "dynamodb.application-autoscaling.amazonaws.com"
-      Description: "AWS service role for application autoscaling DynamoDB"
-    Type: "AWS::IAM::ServiceLinkedRole"
-{{- end }}
   RemoteFilesEncryptionKey:
     Type: "AWS::KMS::Key"
     Properties:

cluster/config-defaults.yaml

@@ -60,13 +60,14 @@ nlb_switch: "exec"
 # skipper ingress settings
 skipper_ingress_target_average_utilization_cpu: "60"
 skipper_ingress_target_average_utilization_memory: "80"
-skipper_ingress_max_replicas: "50"
 skipper_ingress_hpa_scale_down_wait: "600"
 skipper_ingress_hpa_scale_up_max_perc: "100"
 {{if eq .Cluster.Environment "production"}}
 skipper_ingress_min_replicas: "3"
+skipper_ingress_max_replicas: "300"
 {{else}}
 skipper_ingress_min_replicas: "2"
+skipper_ingress_max_replicas: "50"
 {{end}}
 skipper_ingress_cpu: "1000m"
 skipper_ingress_memory: "1500Mi"
@@ -230,7 +231,7 @@ skipper_max_tcp_listener_concurrency: "-1"
 skipper_max_tcp_listener_queue: "-1"
 
 # opentracing
-skipper_ingress_opentracing_excluded_proxy_tags: "skipper.route"
+skipper_ingress_opentracing_excluded_proxy_tags: ""
 skipper_ingress_opentracing_backend_name_tag: "true"
 skipper_opentracing_disable_filter_spans: "true"
 # lightstep
@@ -316,7 +317,7 @@ skipper_open_policy_agent_styra_token: ""
 #   - production: runs the controller
 #
 fabric_gateway_controller_mode: "disabled"
-fabric_gateway_controller_version: "master-231"
+fabric_gateway_controller_version: "master-234"
 fabric_gateway_controller_cpu: "50m"
 fabric_gateway_controller_memory: "150Mi"
 fabric_gateway_crd_v1_enabled: "false"
@@ -374,7 +375,7 @@ journald_reader_cpu: "1m"
 journald_reader_memory: "30Mi"
 
 # Logging settings
-logging_s3_bucket: "zalando-logging-{{.InfrastructureAccount | getAWSAccountID}}-{{.Region}}"
+logging_s3_bucket: "zalando-logging-{{ .Cluster.InfrastructureAccount | getAWSAccountID}}-{{ .Cluster.Region }}"
 scalyr_team_token: ""
 log_destination_infra: "scalyr/stups"
 log_destination_both: "scalyr/main+stups"
@@ -404,6 +405,9 @@ prometheus_remote_min_backoff: "3s"
 # Maximum retry delay.
 prometheus_remote_max_backoff: "10s"
 
+# Comma-separated list of user ids allowed to access Prometheus UI
+prometheus_ui_users: ""
+
 # dashboard metrics scraper resource limits
 dashboard_metrics_scraper_cpu_min: "50m"
 dashboard_metrics_scraper_mem_min: "200Mi"
@@ -551,7 +555,7 @@ teapot_admission_controller_namespace_delete_protection_enabled: "false"
 teapot_admission_controller_resolve_vanity_images: "true"
 
 {{if eq .Cluster.Environment "e2e"}}
-teapot_admission_controller_ignore_namespaces: "^kube-system|((downward-api|kubectl|projected|statefulset|pod-network|scope-selectors|resourcequota|limitrange)-.*)$"
+teapot_admission_controller_ignore_namespaces: "^kube-system|((downward-api|kubectl|projected|statefulset|pod-network|scope-selectors|resourcequota|limitrange|sysctl|node-tests|e2e-kubelet-etc-hosts)-.*)$"
 teapot_admission_controller_crd_ensure_no_resources_on_delete: "false"
 {{else}}
 teapot_admission_controller_ignore_namespaces: "^kube-system$"
@@ -607,6 +611,21 @@ teapot_admission_controller_configmap_deletion_protection_enabled: "false"
 teapot_admission_controller_configmap_deletion_protection_enabled: "true"
 {{end}}
 
+# Enable and configure Pod Security Policy rules implemented in admission-controller.
+teapot_admission_controller_pod_security_policy_enabled: "true"
+
+# comma separated list of service accounts that are allowed to use privileged
+# pod security policy rules. Format: `<namespace>_<service-account-name>`
+{{ if eq .Cluster.Environment "e2e" }}
+teapot_admission_controller_pod_security_policy_privileged_service_accounts: "psp-privileged-zalando_privileged-sa,psp-privileged-deployment-zalando_privileged-sa"
+{{ else }}
+teapot_admission_controller_pod_security_policy_privileged_service_accounts: ""
+{{ end }}
+teapot_admission_controller_pod_security_policy_privileged_allow_privilege_escalation: "false"
+
+# Optionally disable PodSecurityPolicy. Make sure `teapot_admission_controller_pod_security_policy_enabled` is true if this is disabled, otherwise there are no Pod security Policy enforcement in the cluster.
+pod_security_policy_enabled: "false"
+
 # Prevent the use of a particular AZ as much as possible
 blocked_availability_zone: ""
 
@@ -622,9 +641,8 @@ etcd_instance_type: "t3.medium"
 {{end}}
 
 etcd_scalyr_key: ""
-etcd_ami: {{ amiID "zalando-ubuntu-etcd-production-v3.5.9-amd64-main-21" "861068367966"}}
 
-dynamodb_service_link_enabled: "false"
+etcd_ami: {{ amiID "zalando-ubuntu-etcd-production-v3.5.9-amd64-main-24" "861068367966"}}
 
 cluster_dns: "coredns"
 coredns_log_svc_names: "true"
@@ -874,11 +892,20 @@ stackset_routegroup_support_enabled: "true"
 stackset_ingress_source_switch_ttl: "5m"
 
 # enable/disable configmap support for stackset
+{{if eq .Cluster.Environment "e2e"}}
+stackset_configmap_support_enabled: "true"
+{{else}}
 stackset_configmap_support_enabled: "false"
+{{end}}
 
 # enable/disable traffic segment support for stackset
+{{if eq .Cluster.Environment "e2e"}}
+stackset_enable_traffic_segments: "true"
+stackset_annotated_traffic_segments: "true"
+{{else}}
 stackset_enable_traffic_segments: "false"
 stackset_annotated_traffic_segments: "false"
+{{end}}
 
 # Enable/Disable profiling for Kubernetes components
 enable_control_plane_profiling: "false"
@@ -1010,3 +1037,4 @@ cronjob_time_zone_enabled: "true"
 # or not. When set to a value != "true" the key will be removed from the stack.
 # TODO: remove after migrating out of all cluster stacks.
 deployment_secret_key_managed: "true"
+deployment_secret_decrypt_any: "true"

cluster/etcd/files.yaml

@@ -11,3 +11,7 @@ files:
     data: "{{ .Cluster.ConfigItems.etcd_client_server_key }}"
     permissions: 0400
     encrypted: true
+  - path: /etc/scalyr-agent-2/userdata.yaml
+    data: {{ printf "scalyr_api_key: %s\ncluster_alias: %s\n" .Cluster.ConfigItems.etcd_scalyr_key .Cluster.Alias | base64 }}
+    permissions: 0400
+    encrypted: true

cluster/etcd/stack.yaml

@@ -51,11 +51,6 @@ Resources:
           Fn::Base64: !Sub |
             #cloud-config
             write_files:
-              - path: /etc/scalyr-agent-2/userdata.yaml
-                permissions: 0644
-                content: |
-                  scalyr_api_key: "{{.Values.etcd_scalyr_key}}"
-                  stack_name: ${AWS::StackName}
               - path: /etc/default/etcd
                 permissions: 0644
                 content: |
@@ -64,8 +59,8 @@ Resources:
                   ETCD_TRUSTED_CA_FILE=/etc/etcd/ssl/ca.cert
                   ETCD_LOG_LEVEL=info
                   HOSTED_ZONE="{{.Values.hosted_zone}}"
-                  S3_CERTS_BUCKET="{{ .S3GeneratedFilesPath }}"
-                  AWS_DEFAULT_REGION=eu-central-1
+                  S3_CERTS_BUCKET="{{ .Values.S3GeneratedFilesPath }}"
+                  AWS_DEFAULT_REGION="{{ .Cluster.Region }}"
             runcmd:
               - [ cfn-signal, --success, 'true', --stack, ${AWS::StackName}, --resource, AppServer, --region, ${AWS::Region} ]
               - [ complete-asg-lifecycle.py, 'etcd-server-lifecycle-hook' ]

cluster/manifests/01-admission-control/config.yaml

@@ -82,6 +82,33 @@ data:
   # This setting enables and disables replacement of vanity images with ECR images during pod admission (during create and update)
   pod.vanity-image-replacement.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_resolve_vanity_images }}"
 
+  # Pod Security Policy
+  pod.pod-security-policy.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_pod_security_policy_enabled }}"
+
+  # service accounts that need privileged PSP should be defined here as `<namespace>_<sa-name>`
+  pod.pod-security-policy.privileged-service-accounts.kube-system_kube-proxy: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_skipper-ingress: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_node-monitor: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_nvidia: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_audittrail-adapter: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_kube-aws-iam-controller: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_kube2iam: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_ebs-csi-node-sa: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_flannel: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_etcd-backup: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_coredns: ""
+  pod.pod-security-policy.privileged-service-accounts.kube-system_efs-provisioner: ""
+  pod.pod-security-policy.privileged-service-accounts.visibility_logging-agent: ""
+{{- range $sa := split .Cluster.ConfigItems.teapot_admission_controller_pod_security_policy_privileged_service_accounts "," }}
+  pod.pod-security-policy.privileged-service-accounts.{{ $sa }}: ""
+{{- end}}
+
+{{- range $sysctl := split .Cluster.ConfigItems.allowed_unsafe_sysctls "," }}
+  pod.pod-security-policy.allowed-unsafe-sysctls.{{ $sysctl }}: ""
+{{- end}}
+
+  pod.pod-security-policy.allow-privilege-escalation: "{{ .Cluster.ConfigItems.teapot_admission_controller_pod_security_policy_privileged_allow_privilege_escalation }}"
+
   deployment.default.rolling-update-max-surge: "{{ .Cluster.ConfigItems.teapot_admission_controller_deployment_default_max_surge }}"
   deployment.default.rolling-update-max-unavailable: "{{ .Cluster.ConfigItems.teapot_admission_controller_deployment_default_max_unavailable }}"
 

cluster/manifests/01-admission-control/skipper-webhook.yaml

@@ -15,7 +15,7 @@ webhooks:
         resources: ["routegroups"]
     clientConfig:
       url: "https://localhost:9085/routegroups"
-      caBundle: "{{ .ConfigItems.ca_cert_decompressed }}"
+      caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}"
     admissionReviewVersions: ["v1"]
     sideEffects: None
     timeoutSeconds: 5
@@ -29,7 +29,7 @@ webhooks:
         resources: ["ingresses"]
     clientConfig:
       url: "https://localhost:9085/ingresses"
-      caBundle: "{{ .ConfigItems.ca_cert_decompressed }}"
+      caBundle: "{{ .Cluster.ConfigItems.ca_cert_decompressed }}"
     admissionReviewVersions: ["v1"]
     sideEffects: None
     timeoutSeconds: 5