Merge pull request #125 from scrungus/stackhpc/yoga

Upstream changes to magnum to enable cluster-api

Author: scrungus

devstack/contrib/new-devstack.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+#
+# These instructions assume an Ubuntu-based host or VM for running devstack.
+# Please note that if you are running this in a VM, it is vitally important
+# that the underlying hardware have nested virtualization enabled or you will
+# experience very poor amphora performance.
+#
+# Heavily based on:
+# https://opendev.org/openstack/octavia/src/branch/master/devstack/contrib/new-octavia-devstack.sh
+
+set -ex
+
+# Set up the packages we need. Ubuntu package manager is assumed.
+sudo apt-get update
+sudo apt-get install git vim apparmor apparmor-utils -y
+
+# Clone the devstack repo
+sudo mkdir -p /opt/stack
+if [ ! -f /opt/stack/stack.sh ]; then
+    sudo chown -R ${USER}. /opt/stack
+    git clone https://git.openstack.org/openstack-dev/devstack /opt/stack
+fi
+
+cat <<EOF > /opt/stack/local.conf
+[[local|localrc]]
+enable_plugin barbican https://opendev.org/openstack/barbican
+enable_plugin heat https://opendev.org/openstack/heat
+enable_plugin neutron https://opendev.org/openstack/neutron
+# NOTE: you can reference your gerrit patch here
+# enable_plugin magnum https://review.opendev.org/openstack/magnum refs/<etc>
+enable_plugin magnum https://opendev.org/openstack/magnum
+enable_plugin magnum-ui https://opendev.org/openstack/magnum-ui
+enable_plugin octavia https://opendev.org/openstack/octavia
+enable_plugin octavia-dashboard https://opendev.org/openstack/octavia-dashboard
+LIBS_FROM_GIT+=python-octaviaclient
+DATABASE_PASSWORD=secretdatabase
+RABBIT_PASSWORD=secretrabbit
+ADMIN_PASSWORD=secretadmin
+HOST_IP=$(hostname -i)
+SERVICE_PASSWORD=secretservice
+SERVICE_TOKEN=111222333444
+# Enable Logging
+LOGFILE=/opt/stack/logs/stack.sh.log
+VERBOSE=True
+LOG_COLOR=True
+# Octavia services
+enable_service octavia o-api o-cw o-da o-hk o-hm
+enable_service tempest
+GLANCE_LIMIT_IMAGE_SIZE_TOTAL=10000
+LIBVIRT_TYPE=kvm
+
+[[post-config|/etc/neutron/neutron.conf]]
+[DEFAULT]
+advertise_mtu = True
+EOF
+
+# Fix permissions on current tty so screens can attach
+sudo chmod go+rw `tty`
+
+# Stack that stack!
+/opt/stack/stack.sh
+
+#
+# Install this checkout and restart the Magnum services
+#
+SELF_PATH="$(realpath "${BASH_SOURCE[0]:-${(%):-%x}}")"
+REPO_PATH="$(dirname "$(dirname "$(dirname "$SELF_PATH")")")"
+python3 -m pip install -e "$REPO_PATH"
+sudo systemctl restart devstack@magnum-api devstack@magnum-cond
+
+source /opt/stack/openrc admin admin
+
+pip install python-magnumclient
+
+# Add a k8s image
+curl -O https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images-prerelease/ubuntu-focal-kube-v1.26.3-230411-1504.qcow2
+openstack image create ubuntu-focal-kube-v1.26.3 \
+  --file ubuntu-focal-kube-v1.26.3-230411-1504.qcow2 \
+  --disk-format qcow2 \
+  --container-format bare \
+  --public
+openstack image set ubuntu-focal-kube-v1.26.3 --os-distro ubuntu --os-version 20.04
+openstack image set ubuntu-focal-kube-v1.26.3 --property kube_version=v1.26.3
+
+curl -O https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images-prerelease/ubuntu-focal-kube-v1.27.0-230418-0937.qcow2
+openstack image create ubuntu-focal-kube-v1.27.0 \
+  --file ubuntu-focal-kube-v1.27.0-230418-0937.qcow2 \
+  --disk-format qcow2 \
+  --container-format bare \
+  --public
+openstack image set ubuntu-focal-kube-v1.27.0 --os-distro ubuntu --os-version 20.04
+openstack image set ubuntu-focal-kube-v1.27.0 --property kube_version=v1.27.0
+
+# Register template for cluster api driver
+openstack coe cluster template create new_driver \
+  --coe kubernetes \
+  --image $(openstack image show ubuntu-focal-kube-v1.26.3 -c id -f value) \
+  --external-network public \
+  --master-flavor ds2G20 \
+  --flavor ds2G20 \
+  --public \
+  --master-lb-enabled
+
+openstack coe cluster template create new_driver_upgrade \
+  --coe kubernetes \
+  --image $(openstack image show ubuntu-focal-kube-v1.27.0 -c id -f value) \
+  --external-network public \
+  --master-flavor ds2G20 \
+  --flavor ds2G20 \
+  --public \
+  --master-lb-enabled
+
+# You can test it like this:
+#  openstack coe cluster create devstacktest \
+#   --cluster-template new_driver \
+#   --master-count 1 \
+#   --node-count 2
+#  openstack coe cluster list
+#  openstack coe cluster config devstacktest

devstack/lib/magnum

@@ -311,11 +311,22 @@ function magnum_register_image {
         echo "Unknown image extension in $image_filename, supported extensions: tgz, img, qcow2, iso, vhd, vhdx, tar.gz, img.gz, img.bz2, vhd.gz, vhdx.gz, qcow2.xz"; false
     fi
 
+    # Cluster API driver sets kube_version on the image
+    # as the image includes the k8s binaries
+    if [ ! -z "$MAGNUM_IMAGE_KUBE_VERSION" ]; then
+        magnum_image_property=$magnum_image_property" --property kube_version=$MAGNUM_IMAGE_KUBE_VERSION"
+    fi
+
     openstack image set $image_name $magnum_image_property
+    openstack image set --public $image_name
+    openstack image show -f yaml $image_name
 }
 
 #magnum_configure_flavor - set hw_rng property for flavor to address the potential entropy issue
 function magnum_configure_flavor {
+    # add a new flavor with two vcpus and just enough RAM
+    openstack flavor create ds2G20 --ram 2048 --disk 20 --id d5 --vcpus 2 --public
+
     local magnum_flavor_property="--property hw_rng:allowed=True --property hw_rng:rate_bytes=1024 --property hw_rng:rate_period=1"
 
     local FLAVOR_LIST=$(openstack flavor list -c Name -f value)
@@ -333,12 +344,122 @@ function install_magnumclient {
     fi
 }
 
+function setup_capi_management_cluster {
+
+    # # Install `kubectl` CLI
+    curl -fsLo /tmp/kubectl "https://dl.k8s.io/release/$(curl -fsL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+    sudo install -o root -g root -m 0755 /tmp/kubectl /usr/local/bin/kubectl
+
+    # K3s has issues without apparmor, so we add it here
+    sudo apt install -y apparmor apparmor-utils
+
+    # Install k3s
+    curl -fsL https://get.k3s.io | sudo bash -s - --disable traefik
+
+    # copy kubeconfig file into standard location
+    mkdir -p $HOME/.kube
+    sudo cp /etc/rancher/k3s/k3s.yaml $HOME/.kube/config
+    sudo chown $USER $HOME/.kube/config
+
+    # Install helm
+    curl -fsL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+
+    {
+        # Install cert manager
+        helm upgrade cert-manager cert-manager \
+        --install \
+        --namespace cert-manager \
+        --create-namespace \
+        --repo https://charts.jetstack.io \
+        --version v1.10.1 \
+        --set installCRDs=true \
+        --wait \
+        --timeout 10m
+    } || {
+        kubectl -n cert-manager get pods |  awk '$1 && $1!="NAME" { print $1 }' | xargs -n1 kubectl -n cert-manager logs
+        exit
+    }
+
+    # Install Cluster API resources
+    mkdir -p capi
+    cat <<EOF > capi/kustomization.yaml
+---
+resources:
+  - >-
+    https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.2/cluster-api-components.yaml
+  - >-
+    https://github.com/kubernetes-sigs/cluster-api-provider-openstack/releases/download/v0.7.2/infrastructure-components.yaml
+patches:
+  - patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/args
+        value:
+          - --leader-elect
+          - --metrics-bind-addr=localhost:8080
+    target:
+      kind: Deployment
+      namespace: capi-system
+      name: capi-controller-manager
+  - patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/args
+        value:
+          - --leader-elect
+          - --metrics-bind-addr=localhost:8080
+    target:
+      kind: Deployment
+      namespace: capi-kubeadm-bootstrap-system
+      name: capi-kubeadm-bootstrap-controller-manager
+  - patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/args
+        value:
+          - --leader-elect
+          - --metrics-bind-addr=localhost:8080
+    target:
+      kind: Deployment
+      namespace: capi-kubeadm-control-plane-system
+      name: capi-kubeadm-control-plane-controller-manager
+EOF
+
+    kubectl apply -k capi
+
+    kubectl rollout status deployment/capi-controller-manager \
+        --namespace capi-system \
+        --timeout 5m \
+    && \
+    kubectl rollout status deployment/capi-kubeadm-bootstrap-controller-manager \
+        --namespace capi-kubeadm-bootstrap-system \
+        --timeout 5m \
+    && \
+    kubectl rollout status deployment/capi-kubeadm-control-plane-controller-manager \
+        --namespace capi-kubeadm-control-plane-system \
+        --timeout 5m \
+    && \
+    kubectl rollout status deployment/capo-controller-manager \
+        --namespace capo-system \
+        --timeout 10m
+
+    # Install addon manager
+    helm upgrade cluster-api-addon-provider cluster-api-addon-provider \
+    --install \
+    --repo https://stackhpc.github.io/cluster-api-addon-provider \
+    --version 0.1.0-dev.0.main.26 \
+    --namespace capi-addon-system \
+    --create-namespace \
+    --wait \
+    --timeout 10m
+}
+
 # install_magnum() - Collect source and prepare
 function install_magnum {
     install_apache_uwsgi
 
     git_clone $MAGNUM_REPO $MAGNUM_DIR $MAGNUM_BRANCH
     setup_develop $MAGNUM_DIR
+
+    # get ready for capi driver
+    setup_capi_management_cluster
 }
 
 # start_magnum_api() - Start the API process ahead of other things

magnum/common/clients.py

@@ -12,7 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from barbicanclient.v1 import client as barbicanclient
+from barbicanclient import client as barbicanclient
 from cinderclient.v3 import client as cinder_client
 from glanceclient import client as glanceclient
 from heatclient import client as heatclient

magnum/common/exception.py

@@ -301,6 +301,10 @@ class OSDistroFieldNotFound(ResourceNotFound):
     code = 400
 
 
+class KubeVersionPropertyNotFound(Invalid):
+    message = _("Image %(image_id)s does not have a kube_version property.")
+
+
 class X509KeyPairNotFound(ResourceNotFound):
     message = _("A key pair %(x509keypair)s could not be found.")
 

magnum/common/urlfetch.py

@@ -34,8 +34,6 @@ def get(url, allowed_schemes=('http', 'https')):
     """Get the data at the specified URL.
 
     The URL must use the http: or https: schemes.
-    The file: scheme is also supported if you override
-    the allowed_schemes argument.
     Raise an IOError if getting the data fails.
     """
     LOG.info('Fetching data from %s', url)
@@ -45,14 +43,8 @@ def get(url, allowed_schemes=('http', 'https')):
     if components.scheme not in allowed_schemes:
         raise URLFetchError(_('Invalid URL scheme %s') % components.scheme)
 
-    if components.scheme == 'file':  # nosec
-        try:
-            return urllib.request.urlopen(url).read()
-        except urllib.error.URLError as uex:
-            raise URLFetchError(_('Failed to retrieve manifest: %s') % uex)
-
     try:
-        resp = requests.get(url, stream=True)
+        resp = requests.get(url, stream=True, timeout=60)
         resp.raise_for_status()
 
         # We cannot use resp.text here because it would download the

magnum/conf/__init__.py

@@ -17,6 +17,7 @@
 
 from magnum.conf import api
 from magnum.conf import barbican
+from magnum.conf import capi_driver
 from magnum.conf import certificates
 from magnum.conf import cinder
 from magnum.conf import cluster
@@ -48,6 +49,7 @@
 
 api.register_opts(CONF)
 barbican.register_opts(CONF)
+capi_driver.register_opts(CONF)
 cluster.register_opts(CONF)
 cluster_templates.register_opts(CONF)
 cluster_heat.register_opts(CONF)