Merge pull request #118 from aishwaryabk/hyp-het

To add heterogeneous hypershift agent-based installation

Author: ppc64le-cloud-bot

examples/all.yaml

@@ -427,6 +427,7 @@ logging_e2e_testcases: "" #Set to either all or critical
 hypershift_agent_install: false
 hypershift_agent_destroy: false #Enable only to destroy the hosted cluster
 acm_mce_install: false
+heterogeneous_install: false
 hosted_cluster_name: "hypershift-hosted"
 acm_catalogsource_name: "redhat-operators-acm"
 acm_catalogsource_image: ""
@@ -437,16 +438,20 @@ mce_upgrade_channel: "stable-2.7"
 base_domain: "qe-ppc64le.cis.ibm.net"
 pull_secret_file: "~/.pullSecret"
 ocp_release: ""
-arch: "ppc64le"
+ocp_release_image: ""
+arch_ppc: "ppc64le"
+arch_x86: "x86_64"
 hypershift_dir: "/root/hypershift"
 hypershift_git_repo: "https://github.com/openshift/hypershift"
 hypershift_git_branch: "main"
 cluster_namespace: "clusters"
 ssh_pub_key: ""
 db_volume_size: "10Gi"
 fs_volume_size: "100Gi"
-iso_url: ""
-root_fs_url: ""
+iso_url_ppc: ""
+root_fs_url_ppc: ""
+iso_url_x86: ""
+root_fs_url_x86: ""
 
 # ocp-lso vars
 lso_enabled: false            #set to true for lso operator installation

examples/hypershift_agent.yaml

@@ -4,6 +4,7 @@
 hypershift_agent_install: false
 hypershift_agent_destroy: false #Enable only to destroy the hosted cluster
 acm_mce_install: false
+heterogeneous_install: false
 hosted_cluster_name: "hypershift-hosted"
 acm_catalogsource_name: "redhat-operators-acm"
 acm_catalogsource_image: ""
@@ -14,13 +15,17 @@ mce_upgrade_channel: "stable-2.7"
 base_domain: "qe-ppc64le.cis.ibm.net"
 pull_secret_file: "~/.pullSecret"
 ocp_release: ""
-arch: "ppc64le"
+ocp_release_image: ""
+arch_ppc: "ppc64le"
+arch_x86: "x86_64"
 hypershift_dir: "/root/hypershift"
 hypershift_git_repo: "https://github.com/openshift/hypershift"
 hypershift_git_branch: "main"
 cluster_namespace: "clusters"
 ssh_pub_key: ""
 db_volume_size: "10Gi"
 fs_volume_size: "100Gi"
-iso_url: ""
-root_fs_url: ""
+iso_url_ppc: ""
+root_fs_url_ppc: ""
+iso_url_x86: ""
+root_fs_url_x86: ""

playbooks/roles/hypershift-agent/README.md

@@ -15,6 +15,7 @@ Common variables used for both hypershift deployment and destroy operations:
 | hypershift_agent_install  | no | false | Enable it to install the hosted cluster |
 | hypershift_agent_destroy  | no | false | Enable it to destroy the hosted cluster |
 | acm_mce_install | no | false | Enable it to install ACM and MCE operator |
+| heterogeneous_install | no | false | Enable it to install heterogeneous deployment |
 | hosted_cluster_name | no | hypershift-hosted | Name of the hypershift cluster |
 | acm_catalogsource_name | no | redhat-operators-acm | Name of the ACM CatalogSource |
 | acm_catalogsource_image | yes | | Custom catalog source index image for ACM Operator (Ex: brew.registry.redhat.io/rh-osbs/iib:860542 )|
@@ -25,16 +26,20 @@ Common variables used for both hypershift deployment and destroy operations:
 | base_domain | no | qe-ppc64le.cis.ibm.net | The base domain of the cluster |
 | pull_secret_file | no | ~/.pullSecret | Pull secret file |
 | ocp_release | yes | | ocp release version of the cluster to deploy |
-| arch | no | ppc64le | cluster architecture |
+| ocp_release_image | yes | | release image used to deploy cluster |
+| arch_ppc | no | ppc64le | power cluster architecture |
+| arch_x86 | no | x86_64 | x86 cluster architecture |
 | hypershift_dir | no | /root/hypershift | folder that will contain kubeconfig, etc |
 | hypershift_git_repo | no | https://github.com/openshift/hypershift | git repository to install hypershift |
 | hypershift_git_branch | no | main | Hypershift git repository branch |
 | cluster_namespace | no | clusters |  |
 | ssh_pub_key | no | ~/.ssh/id_rsa.pub | |
 | db_volume_size | no | 10Gi | Database Volume Size |
 | fs_volume_size | no | 100Gi | FileSystem Volume Size |
-| iso_url | yes | | ISO url |
-| root_fs_url | yes | | Root fs url |
+| iso_url_ppc | yes | | ppc64le ISO url |
+| root_fs_url_ppc | yes | | ppc64le root fs url |
+| iso_url_x86 | yes | | x86 ISO url |
+| root_fs_url_x86 | yes | | x86 root fs url |
 
 Dependencies
 ------------

playbooks/roles/hypershift-agent/defaults/main.yml

@@ -3,6 +3,7 @@
 hypershift_agent_install: false
 hypershift_agent_destroy: false #Enable only to destroy the hosted cluster
 acm_mce_install: false
+heterogeneous_install: false
 hosted_cluster_name: "hypershift-hosted"
 acm_catalogsource_name: "redhat-operators-acm"
 acm_catalogsource_image: ""
@@ -13,13 +14,17 @@ mce_upgrade_channel: "stable-2.7"
 base_domain: "qe-ppc64le.cis.ibm.net"
 pull_secret_file: "~/.pullSecret"
 ocp_release: ""
-arch: "ppc64le"
+ocp_release_image: ""
+arch_ppc: "ppc64le"
+arch_x86: "x86_64"
 hypershift_dir: "/root/hypershift"
 hypershift_git_repo: "https://github.com/openshift/hypershift"
 hypershift_git_branch: "main"
 cluster_namespace: "clusters"
 ssh_pub_key: "~/.ssh/id_rsa.pub"
 db_volume_size: "10Gi"
 fs_volume_size: "100Gi"
-iso_url: ""
-root_fs_url: ""
+iso_url_ppc: ""
+root_fs_url_ppc: ""
+iso_url_x86: ""
+root_fs_url_x86: ""

playbooks/roles/hypershift-agent/tasks/install.yml

@@ -4,7 +4,7 @@
   block:
     - name: Download baremetalhosts manifest
       get_url:
-        url: https://raw.githubusercontent.com/openshift/assisted-service/master/hack/crds/metal3.io_baremetalhosts.yaml
+        url: https://raw.githubusercontent.com/openshift/assisted-service/refs/heads/master/hack/crds/metal3/metal3.io_baremetalhosts.yaml
         dest: ./metal3.io_baremetalhosts.yaml
         mode: 0664
 
@@ -15,7 +15,7 @@
 
     - name: Download preprovisioningimages manifest
       get_url:
-        url: https://raw.githubusercontent.com/openshift/assisted-service/refs/heads/master/hack/crds/metal3.io_preprovisioningimages.yaml
+        url: https://raw.githubusercontent.com/openshift/assisted-service/refs/heads/master/hack/crds/metal3/metal3.io_preprovisioningimages.yaml
         dest: ./metal3.io_preprovisioningimages.yaml
         mode: 0664
 
@@ -216,9 +216,23 @@
     src: "{{ role_path }}/templates/agent-service-config.yml"
   when: ocp_release_version.stdout != "" and ocp_release_min.stdout != ""
 
+- name: Update AgentServiceConfig for heterogeneous
+  block:
+    - name: Update AgentServiceConfig
+      template:
+        src: "{{ role_path }}/templates/agent-service-config-heterogeneous.yaml.j2"
+        dest: "{{ role_path }}/templates/agent-service-config-heterogeneous.yml"
+
+    - name: Deploy AgentServiceConfig
+      kubernetes.core.k8s:
+        state: present
+        src: "{{ role_path }}/templates/agent-service-config-heterogeneous.yml"
+      when: ocp_release_version.stdout != "" and ocp_release_min.stdout != ""
+  when: heterogeneous_install
+
 - name: Hypershift deployment
   block:
-    - name: Create namespace for hosted control plane namespace
+    - name: Create namespace for hosted control plane
       kubernetes.core.k8s:
         state: present
         definition:
@@ -233,9 +247,14 @@
             finalizers:
             - kubernetes
 
+    - name: Set value for infra_availability_policy
+      set_fact:
+        infra_availability_policy: "{{ 'HighlyAvailable' if heterogeneous_install else 'SingleReplica' }}"
+        cacheable: yes
+
     - name: Create Hypershift Cluster Agent
       shell: |
-        hcp create cluster agent --name={{ hosted_cluster_name }} --pull-secret={{ pull_secret_file }} --infra-availability-policy SingleReplica --control-plane-availability-policy=SingleReplica --agent-namespace={{ hosted_control_plane_namespace }} --ssh-key {{ ssh_pub_key_value.stdout }} --base-domain={{ base_domain }} --api-server-address=api.{{ hosted_cluster_name }}.{{ base_domain }} --release-image=quay.io/openshift-release-dev/ocp-release:{{ ocp_release }}-multi --render-sensitive --render > {{ hypershift_dir }}/render-{{ hosted_cluster_name }}.yaml
+        hcp create cluster agent --name={{ hosted_cluster_name }} --pull-secret={{ pull_secret_file }} --infra-availability-policy {{ infra_availability_policy }} --control-plane-availability-policy=SingleReplica --agent-namespace={{ hosted_control_plane_namespace }} --ssh-key {{ ssh_pub_key }} --base-domain={{ base_domain }} --api-server-address=api.{{ hosted_cluster_name }}.{{ base_domain }} --release-image={{ ocp_release_image }} --render-sensitive --render --node-pool-replicas -1 > {{ hypershift_dir }}/render-{{ hosted_cluster_name }}.yaml
       ignore_errors: true
       environment:
         PATH: "{{ hypershift_dir }}/bin:{{ ansible_env.PATH }}"
@@ -285,39 +304,100 @@
   shell: "cat {{ ssh_pub_key }}"
   register: ssh_pub_key_value
 
-- name: Create InfraEnv
+- name: Create NodePool for ppc64le nodes
+  template:
+    src: "{{ role_path }}/templates/Nodepool-ppc.yaml.j2"
+    dest: "{{ role_path }}/templates/Nodepool-ppc.yml"
+  delegate_to: localhost
+
+- name: Deploy NodePool for ppc64le nodes
+  kubernetes.core.k8s:
+    state: present
+    src: "{{ role_path }}/templates/Nodepool-ppc.yml"
+
+- name: Create InfraEnv for ppc64le
   kubernetes.core.k8s:
     state: present
     definition:
       apiVersion: agent-install.openshift.io/v1beta1
       kind: InfraEnv
       metadata:
-        name: "{{ hosted_cluster_name }}"
+        name: "{{ hosted_cluster_name }}-ppc"
         namespace: "{{ hosted_control_plane_namespace }}"
       spec:
-        cpuArchitecture: ppc64le
+        cpuArchitecture: "{{ arch_ppc }}"
         pullSecretRef:
           name: pull-secret
         sshAuthorizedKey: "{{ ssh_pub_key_value.stdout }}"
 
+- name: Create NodePool and InfraEnv for heterogeneous
+  block:
+    - name: Create NodePool for x86 nodes
+      template:
+        src: "{{ role_path }}/templates/Nodepool-x86.yaml.j2"
+        dest: "{{ role_path }}/templates/Nodepool-x86.yml"
+
+    - name: Deploy NodePool for x86 nodes
+      kubernetes.core.k8s:
+        state: present
+        src: "{{ role_path }}/templates/Nodepool-x86.yml"
+
+    - name: Create InfraEnv for x86
+      kubernetes.core.k8s:
+        state: present
+        definition:
+          apiVersion: agent-install.openshift.io/v1beta1
+          kind: InfraEnv
+          metadata:
+            name: "{{ hosted_cluster_name }}-x86"
+            namespace: "{{ hosted_control_plane_namespace }}"
+          spec:
+            cpuArchitecture: "{{ arch_x86 }}"
+            pullSecretRef:
+              name: pull-secret
+            sshAuthorizedKey: "{{ ssh_pub_key_value.stdout }}"
+  when: heterogeneous_install
+
 - name: Check if the ISO is generated
   shell: "oc get infraenv -n {{ hosted_control_plane_namespace }} --no-headers | awk '{ print $2 }'"
   register: infraenv_iso
   until: infraenv_iso.stdout|length > 0
   retries: 15
   delay: 60
 
-- name: Get the url of the generated ISO
-  shell: "oc get -n {{ hosted_control_plane_namespace }} InfraEnv {{ hosted_cluster_name }} -ojsonpath='{.status.isoDownloadURL}'"
+- name: Get the url of the generated power ISO
+  shell: "oc get -n {{ hosted_control_plane_namespace }} InfraEnv {{ hosted_cluster_name }}-ppc -ojsonpath='{.status.isoDownloadURL}'"
   register: infraenv_iso
 
-- name: Output the ISO URL link for download
+- name: Output the power ISO URL link for download
   debug:
     msg: "{{ infraenv_iso.stdout }}"
 
+- name: Get the x86 ISO details
+  block:
+    - name: Get initrd_url
+      shell: (oc get -n {{ hosted_control_plane_namespace }} InfraEnv {{ hosted_cluster_name }}-x86 -o json | jq -r '."status"."bootArtifacts"."initrd"')
+      register: initrd_url
+
+    - name: Get kernel_url
+      shell: (oc get -n {{ hosted_control_plane_namespace }} InfraEnv {{ hosted_cluster_name }}-x86 -o json | jq -r '."status"."bootArtifacts"."kernel"')
+      register: kernel_url
+
+    - name: Get rootfs_url
+      shell: (oc get -n {{ hosted_control_plane_namespace }} InfraEnv {{ hosted_cluster_name }}-x86 -o json | jq -r '."status"."bootArtifacts"."rootfs"')
+      register: rootfs_url
+
+    - name: Output values of initrd_url, kernel_url and rootfs_url
+      debug:
+        msg:
+          - "initrd_url: {{ initrd_url.stdout }}"
+          - "kernel_url: {{ kernel_url.stdout }}"
+          - "rootfs_url: {{ rootfs_url.stdout }}"
+  when: heterogeneous_install
+
 - name: Output the Kubernetes api service to create DNS entries
   debug:
-   msg: "{{ dns_entry.stdout }}"
+    msg: "{{ dns_entry.stdout }}"
 
 - name: Output the location of Kubeconfig of the hosted cluster
   debug:

playbooks/roles/hypershift-agent/templates/Nodepool-ppc.yaml.j2

@@ -0,0 +1,22 @@
+apiVersion: hypershift.openshift.io/v1beta1
+kind: NodePool
+metadata:
+  name: {{ hosted_cluster_name }}-ppc
+  namespace: {{ cluster_namespace }}
+spec:
+  arch: {{ arch_ppc }}
+  clusterName: {{ hosted_cluster_name }}
+  management:
+    autoRepair: false
+    upgradeType: InPlace
+  nodeDrainTimeout: 0s
+  nodeVolumeDetachTimeout: 0s
+  platform:
+    agent:
+      agentLabelSelector:
+        matchLabels:
+          inventory.agent-install.openshift.io/cpu-architecture: {{ arch_ppc }}
+    type: Agent
+  release:
+    image: {{ ocp_release_image }}
+  replicas: 0

playbooks/roles/hypershift-agent/templates/Nodepool-x86.yaml.j2

@@ -0,0 +1,22 @@
+apiVersion: hypershift.openshift.io/v1beta1
+kind: NodePool
+metadata:
+  name: {{ hosted_cluster_name }}-x86
+  namespace: {{ cluster_namespace }}
+spec:
+  arch: amd64
+  clusterName: {{ hosted_cluster_name }}
+  management:
+    autoRepair: false
+    upgradeType: InPlace
+  nodeDrainTimeout: 0s
+  nodeVolumeDetachTimeout: 0s
+  platform:
+    agent:
+      agentLabelSelector:
+        matchLabels:
+          inventory.agent-install.openshift.io/cpu-architecture: {{ arch_x86 }}
+    type: Agent
+  release:
+    image: {{ ocp_release_image }}
+  replicas: 0

playbooks/roles/hypershift-agent/templates/agent-service-config-heterogeneous.yaml.j2

@@ -0,0 +1,30 @@
+apiVersion: agent-install.openshift.io/v1beta1
+kind: AgentServiceConfig
+metadata:
+ name: agent
+spec:
+  mirrorRegistryRef:
+    name: mirror-config
+  databaseStorage:
+    accessModes:
+    - ReadWriteOnce
+    resources:
+      requests:
+        storage: "{{ db_volume_size }}"
+  filesystemStorage:
+    accessModes:
+    - ReadWriteOnce
+    resources:
+      requests:
+        storage: "{{ fs_volume_size }}"
+  osImages:
+    - openshiftVersion: "{{ ocp_release_min.stdout }}"
+      version: "{{ ocp_release_version.stdout }}"
+      url: "{{ iso_url_x86 }}"
+      rootFSUrl: "{{ root_fs_url_x86 }}"
+      cpuArchitecture: "{{ arch_x86 }}"
+    - openshiftVersion: "{{ ocp_release_min.stdout }}"
+      version: "{{ ocp_release_version.stdout }}"
+      url: "{{ iso_url_ppc }}"
+      rootFSUrl: "{{ root_fs_url_ppc }}"
+      cpuArchitecture: "{{ arch_ppc }}"

playbooks/roles/hypershift-agent/templates/agent-service-config.yaml.j2

@@ -20,6 +20,6 @@ spec:
   osImages:
     - openshiftVersion: "{{ ocp_release_min.stdout }}"
       version: "{{ ocp_release_version.stdout }}"
-      url: "{{ iso_url }}"
-      rootFSUrl: "{{ root_fs_url }}"
-      cpuArchitecture: "{{ arch }}"
+      url: "{{ iso_url_ppc }}"
+      rootFSUrl: "{{ root_fs_url_ppc }}"
+      cpuArchitecture: "{{ arch_ppc }}"