Merge branch 'stackhpc/yoga' into upstream/yoga-2023-04-03

Author: mnasiadka

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @stackhpc/openstack

.github/workflows/tag-and-release.yml

@@ -0,0 +1,12 @@
+---
+name: Tag & Release
+'on':
+  push:
+    branches:
+      - stackhpc/yoga
+permissions:
+  actions: read
+  contents: write
+jobs:
+  tag-and-release:
+    uses: stackhpc/.github/.github/workflows/tag-and-release.yml@main

.github/workflows/tox.yml

@@ -0,0 +1,7 @@
+---
+name: Tox Continuous Integration
+'on':
+  pull_request:
+jobs:
+  tox:
+    uses: stackhpc/.github/.github/workflows/tox.yml@main

doc/source/user/index.rst

@@ -389,6 +389,14 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `boot_volume_type`_                   | see below          | see below     |
 +---------------------------------------+--------------------+---------------+
+| `master_volume_size`_                 | see below          | see below     |
++---------------------------------------+--------------------+---------------+
+| `master_volume_type`_                 | see below          | see below     |
++---------------------------------------+--------------------+---------------+
+| `worker_volume_size`_                 | see below          | see below     |
++---------------------------------------+--------------------+---------------+
+| `worker_volume_type`_                 | see below          | see below     |
++---------------------------------------+--------------------+---------------+
 | `etcd_volume_size`_                   | etcd storage       | 0             |
 |                                       | volume size        |               |
 +---------------------------------------+--------------------+---------------+
@@ -502,6 +510,18 @@ the table are linked to more details elsewhere in the user guide.
 +---------------------------------------+--------------------+---------------+
 | `fixed_subnet_cidr`_                  | see below          | ""            |
 +---------------------------------------+--------------------+---------------+
+| `extra_network`_                      | see below          | ""            |
++---------------------------------------+--------------------+---------------+
+| `extra_subnet`_                       | see below          | ""            |
++---------------------------------------+--------------------+---------------+
+| `extra_security_group`_               | see below          | see below     |
++---------------------------------------+--------------------+---------------+
+| `octavia_provider`_                   | see below          | amphora       |
++---------------------------------------+--------------------+---------------+
+| `octavia_lb_algorithm`_               | see bellow         | ROUND_ROBIN   |
++---------------------------------------+--------------------+---------------+
+| `octavia_lb_healthcheck`_             | see bellow         | true          |
++---------------------------------------+--------------------+---------------+
 
 .. _cluster:
 
@@ -1200,13 +1220,42 @@ _`admission_control_list`
 _`boot_volume_size`
   This label overrides the default_boot_volume_size of instances which is
   useful if your flavors are boot from volume only. The default value is 0,
-  meaning that cluster instances will not boot from volume.
+  meaning that cluster instances will not boot from volume unless
+  master_volume_size or worker_volume_size is defined. This label has
+  lower priority than abovementioned and can be overridden by them.
+  Current precedence is master/worker_volume_size, boot_volume_size,
+  default_boot_volume_size.
+
 
 _`boot_volume_type`
   This label overrides the default_boot_volume_type of instances which is
   useful if your flavors are boot from volume only. The default value is '',
   meaning that Magnum will randomly select a Cinder volume type from all
-  available options.
+  available options unless master_volume_type or worker_volume_type are set.
+  Current precedence is master/worker_volume_type, boot_volume_type,
+  default_boot_volume_type.
+
+_`master_volume_size`
+  This labed can be used to define different volume size for master nodes than
+  set in boot_volume_size. Master_volume_size will be set to boot_volume_size
+  if not defined, fallback to default_boot_volume_size if the latter is
+  missing. If neither is defined master nodes will not boot
+  from volume.
+
+_`master_volume_type`
+  This label can be used to override volume type of master nodes if defined.
+  Otherwise boot_volume_type value will be used.
+
+_`worker_volume_size`
+  This labed can be used to define different volume size for worker nodes than
+  set in boot_volume_size. worker_volume_size will be set to boot_volume_size
+  if not defined, fallback to default_boot_volume_size if the latter is
+  missing. If neither is defined worker nodes will not boot
+  from volume.
+
+_`worker_volume_type`
+  This label can be used to override volume type of worker nodes if defined.
+  Otherwise boot_volume_type value will be used.
 
 _`etcd_volume_size`
   This label sets the size of a volume holding the etcd storage data.
@@ -1264,13 +1313,14 @@ _`container_infra_prefix`
 
   Images that might be needed if 'monitoring_enabled' is 'true':
 
-  * quay.io/prometheus/alertmanager:v0.20.0
-  * docker.io/squareup/ghostunnel:v1.5.2
-  * docker.io/jettech/kube-webhook-certgen:v1.0.0
-  * quay.io/coreos/prometheus-operator:v0.37.0
-  * quay.io/coreos/configmap-reload:v0.0.1
-  * quay.io/coreos/prometheus-config-reloader:v0.37.0
-  * quay.io/prometheus/prometheus:v2.15.2
+  * quay.io/prometheus/alertmanager:v0.21.0
+  * docker.io/jettech/kube-webhook-certgen:v1.5.0
+  * quay.io/prometheus-operator/prometheus-operator:v0.44.0
+  * docker.io/jimmidyson/configmap-reload:v0.4.0
+  * quay.io/prometheus-operator/prometheus-config-reloader:v0.44.0
+  * quay.io/prometheus/prometheus:v2.22.1
+  * quay.io/prometheus/node-exporter:v1.0.1
+  * docker.io/directxman12/k8s-prometheus-adapter:v0.8.2
 
   Images that might be needed if 'cinder_csi_enabled' is 'true':
 
@@ -1434,37 +1484,48 @@ _`cinder_csi_plugin_tag`
   <https://hub.docker.com/r/k8scloudprovider/cinder-csi-plugin/tags>`_.
   Train default: v1.16.0
   Ussuri default: v1.18.0
+  Yoga default: v1.23.0
 
 _`csi_attacher_tag`
   This label allows users to override the default container tag for CSI attacher.
   For additional tags, `refer to CSI attacher page
   <https://quay.io/repository/k8scsi/csi-attacher?tab=tags>`_.
   Ussuri-default: v2.0.0
+  Yoga-default: v3.3.0
 
 _`csi_provisioner_tag`
   This label allows users to override the default container tag for CSI provisioner.
   For additional tags, `refer to CSI provisioner page
   <https://quay.io/repository/k8scsi/csi-provisioner?tab=tags>`_.
   Ussuri-default: v1.4.0
+  Yoga-default: v3.0.0
 
 _`csi_snapshotter_tag`
   This label allows users to override the default container tag for CSI snapshotter.
   For additional tags, `refer to CSI snapshotter page
   <https://quay.io/repository/k8scsi/csi-snapshotter?tab=tags>`_.
   Ussuri-default: v1.2.2
+  Yoga-default: v4.2.1
 
 _`csi_resizer_tag`
   This label allows users to override the default container tag for CSI resizer.
   For additional tags, `refer to CSI resizer page
   <https://quay.io/repository/k8scsi/csi-resizer?tab=tags>`_.
   Ussuri-default: v0.3.0
+  Yoga-default: v1.3.0
 
 _`csi_node_driver_registrar_tag`
   This label allows users to override the default container tag for CSI node
   driver registrar. For additional tags, `refer to CSI node driver registrar
   page
   <https://quay.io/repository/k8scsi/csi-node-driver-registrar?tab=tags>`_.
   Ussuri-default: v1.1.0
+  Yoga-default: v2.4.0
+
+-`csi_liveness_probe_tag`
+  This label allows users to override the default container tag for CSI
+  liveness probe.
+  Yoga-default: v2.5.0
 
 _`keystone_auth_enabled`
   If this label is set to True, Kubernetes will support use Keystone for
@@ -1635,6 +1696,33 @@ _`fixed_subnet_cidr`
   specified an existing fixed_subnet during cluster creation.
   Ussuri default: 10.0.0.0/24
 
+_`extra_network`
+  Optional additional network name or UUID to add to cluster nodes.
+  When not specified, additional networks are not added. Optionally specify
+  'extra_subnet' if you wish to use a specific subnet on the network.
+  Default: ""
+
+_`extra_subnet`
+  Optional additional subnet name or UUID to add to cluster nodes.
+  Only used when 'extra_network' is defined.
+  Default: ""
+
+_`extra_security_group`
+  Optional additional group name or UUID to add to network port.
+  Only used when 'extra_network' is defined.
+  Default: cluster node default security group.
+
+_`octavia_provider`
+  Octavia provider driver to be used for creating load balancers.
+
+_`octavia_lb_algorithm`
+  Octavia Octavia lb algorithm to use for LoadBalancer type service
+  Default: ROUND_ROBIN
+
+_`octavia_lb_healthcheck`
+  If true, enable Octavia load balancer healthcheck
+  Default: true
+
 External load balancer for services
 -----------------------------------
 
@@ -2723,7 +2811,6 @@ _`calico_tag`
   Victoria default: v3.13.1
   Wallaby default: v3.13.1
 
-
 Besides, the Calico network driver needs kube_tag with v1.9.3 or later, because
 Calico needs extra mounts for the kubelet container. See `commit
 <https://github.com/projectatomic/atomic-system-containers/commit/54ab8abc7fa1bfb6fa674f55cd0c2fa0c812fd36>`_
@@ -3035,6 +3122,39 @@ for the COE types is summarized as follows:
 | rexray | unsupported | supported   | supported   |
 +--------+-------------+-------------+-------------+
 
+Labels can be used to customize nodes boot volume at creation time:
+
+- boot_volume_type
+- boot_volume_size
+
+These define volume type and size used for boot media of node vm, and they can
+be further overriden by:
+
+- master_volume_type
+- master_volume_size
+- worker_volume_type
+- worker_volume_size
+
+Current precedence is:
+- master_volume_size / worker_volume_size
+- boot_volume_size
+- default_boot_volume_size
+
+Labels shown above allow user to use different storage types and sizes for
+master and worker nodes. They can be used independently of each other
+for ex. boot_volume_type to define type of storage to use for both master
+and worker vms, along with master_volume_size and worker_volume_size
+setting size of their boot volumes. Another example would be usage of
+boot_volume_size to define size of both master and worker, with different
+storage types for them set by master_volume_type and worker_volume_type.
+It's possible to use any combination of the above.
+
+If either master_volume_type or worker_volume_type is missing,
+boot_volume_type will be used instead. A random volume type from Cinder will
+be used if none of those options is set. In case of master_volume_size or
+worker_volume_size missing value for boot_volume_size is used. If neither
+is defined instances will not be volume based.
+
 Following are some examples for using Cinder as persistent storage.
 
 Using Cinder in Kubernetes

doc/source/user/monitoring.rst

@@ -35,15 +35,15 @@ _`metrics_server_enabled`
 
 _`monitoring_enabled`
   Enable installation of cluster monitoring solution provided by the
-  stable/prometheus-operator helm chart.
+  prometheus-community/kube-prometheus-stack helm chart.
   To use this service tiller_enabled must be true when using
   helm_client_tag<v3.0.0.
   Default: false
 
 _`prometheus_adapter_enabled`
   Enable installation of cluster custom metrics provided by the
-  stable/prometheus-adapter helm chart. This service depends on
-  monitoring_enabled.
+  prometheus-community/prometheus-adapter helm chart.
+  This service depends on monitoring_enabled.
   Default: true
 
 To control deployed versions, extra labels are available:
@@ -56,14 +56,17 @@ _`metrics_server_chart_tag`
 
 _`prometheus_operator_chart_tag`
   Add prometheus_operator_chart_tag to select version of the
-  stable/prometheus-operator chart to install. When installing the chart,
-  helm will use the default values of the tag defined and overwrite them based
-  on the prometheus-operator-config ConfigMap currently defined. You must
-  certify that the versions are compatible.
+  prometheus-community/kube-prometheus-stack chart to install.
+  When installing the chart, helm will use the default values of the tag
+  defined and overwrite them based on the prometheus-operator-config
+  ConfigMap currently defined.
+  You must certify that the versions are compatible.
+  Wallaby-default: 17.2.0
 
 _`prometheus_adapter_chart_tag`
-  The stable/prometheus-adapter helm chart version to use.
+  The prometheus-community/prometheus-adapter helm chart version to use.
   Train-default: 1.4.0
+  Wallaby-default: 2.12.1
 
 Full fledged cluster monitoring
 +++++++++++++++++++++++++++++++

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -269,7 +269,7 @@ CERT_DIR=/etc/kubernetes/certs
 
 # kube-proxy config
 PROXY_KUBECONFIG=/etc/kubernetes/proxy-kubeconfig.yaml
-KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR} --hostname-override=${INSTANCE_NAME}"
+KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR} --hostname-override=${INSTANCE_NAME} --metrics-bind-address=0.0.0.0"
 cat > /etc/kubernetes/proxy << EOF
 KUBE_PROXY_ARGS="${KUBE_PROXY_ARGS} ${KUBEPROXY_OPTIONS}"
 EOF
@@ -404,6 +404,8 @@ KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true --kubeconfig=/etc/kubernetes/a
 KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}"
 KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true"
 KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${PODS_NETWORK_CIDR}"
+KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --secure-port=10257"
+KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics"
 KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS"
 if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
     KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt"
@@ -426,12 +428,13 @@ sed -i '
     /^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"#
 ' /etc/kubernetes/controller-manager
 
-sed -i '/^KUBE_SCHEDULER_ARGS=/ s#=.*#="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf"#' /etc/kubernetes/scheduler
+sed -i '/^KUBE_SCHEDULER_ARGS=/ s#=.*#="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics "#' /etc/kubernetes/scheduler
 
 $ssh_cmd mkdir -p /etc/kubernetes/manifests
 KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${INSTANCE_NAME}"
 KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.1"
 KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
+KUBELET_ARGS="${KUBELET_ARGS} --resolv-conf=/run/systemd/resolve/resolv.conf"
 KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
 KUBELET_ARGS="${KUBELET_ARGS} ${KUBELET_OPTIONS}"
 

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh

@@ -246,6 +246,7 @@ mkdir -p /etc/kubernetes/manifests
 KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests --kubeconfig ${KUBELET_KUBECONFIG} --hostname-override=${INSTANCE_NAME}"
 KUBELET_ARGS="${KUBELET_ARGS} --address=${KUBE_NODE_IP} --port=10250 --read-only-port=0 --anonymous-auth=false --authorization-mode=Webhook --authentication-token-webhook=true"
 KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=${DNS_CLUSTER_DOMAIN}"
+KUBELET_ARGS="${KUBELET_ARGS} --resolv-conf=/run/systemd/resolve/resolv.conf"
 KUBELET_ARGS="${KUBELET_ARGS} --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
 KUBELET_ARGS="${KUBELET_ARGS} --node-labels=magnum.openstack.org/role=${NODEGROUP_ROLE}"
 KUBELET_ARGS="${KUBELET_ARGS} --node-labels=magnum.openstack.org/nodegroup=${NODEGROUP_NAME}"