ci-group-login_node_aarch64.yaml.j2

- name: {{ functional_group_name }}
  description: "{{ functional_group_name }}"

  file:
    encoding: plain
    content: |
      ## template: jinja
      #cloud-config
      merge_how:
      - name: list
        settings: [append]
      - name: dict
        settings: [no_replace, recurse_list]
      users:
        - name: root
          ssh_authorized_keys: "{{ read_ssh_key.stdout }}"
          lock_passwd: false
          hashed_passwd: "{{ hashed_password_output.stdout }}"
        - name: {{ slurm_user }}
          uid: {{ slurm_uid }}
          system: true
          no_create_home: true
          shell: /sbin/nologin
      disable_root: false

      write_files:
        - path: /usr/local/bin/doca-install.sh
          owner: root:root
          permissions: '{{ file_mode_755 }}'
          content: |
            {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }}

        - path: /usr/local/bin/configure-ib-network.sh
          owner: root:root
          permissions: '{{ file_mode_755 }}'
          content: |
            {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }}

        - path: /usr/local/bin/set-ssh.sh
          permissions: '{{ file_mode_755 }}'
          content: |
            #!/bin/bash
            timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }}
            localectl set-locale LANG={{ hostvars['localhost']['language'] }}
            sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config
            sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config
            sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf
            systemctl restart sshd
            default_count=$(ip route | grep -c "^default")
            if [ "$default_count" -le 1 ]; then
                echo "Only one or no default route found. No action needed."
            else
                private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}')
                # Get all default routes
                ip route | grep '^default' | while read -r line; do
                    nmcli con del "Wired Connection"
                    # Extract NIC name
                    nic=$(echo "$line" | awk '{print $5}')

                    # Add the default route to the connection
                    if [ -n "$nic" ]; then
                        echo "Adding nmcli device $nic"
                        nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto
                        if [ "$nic" = "$private_nic" ]; then
                          nmcli con modify "$nic" ipv4.never-default yes
                          nmcli con delete "cloud-init $nic"
                        fi
                        nmcli con up "$nic"
                    else
                        echo "No connection found for device $nic"
                    fi
                done
            fi

        - path: /root/.ssh/config
          permissions: '0600'
          content: |
            Host {{ slurm_control_ssh_patterns }}
                IdentityFile {{ client_mount_path }}/slurm/ssh/oim_rsa
                IdentitiesOnly yes

{% if hostvars['localhost']['openldap_support'] %}
        - path: /etc/sssd/sssd.conf
          owner: root:root
          permissions: '{{ file_mode_600 }}'
          content: |
            {{ lookup('template', 'templates/openldap/sssd.conf.j2') | indent(6) }}

        - path: /usr/local/bin/update_ldap_conf.sh
          owner: root:root
          permissions: '{{ file_mode_755 }}'
          content: |
            {{ lookup('template', 'templates/openldap/update_ldap_conf.sh.j2') | indent(12) }}
{% endif %}

{% if ldms_support %}
        - path: /root/ldms_sampler.sh
          owner: root:root
          permissions: '0755'
          content: |
            {{ lookup('template', 'templates/ldms/ldms_sampler.sh.j2') | indent(12) }}
{% endif %}

        - path: /etc/hosts
          append: true
          content: |
{% for key in ip_name_map | sort %}
            {{ ip_name_map[key] }} {{ key }}
{% endfor %}

        - path: /etc/sysconfig/slurmd
          owner: root:root
          permissions: '0644'
          content: |
            SLURMD_OPTIONS="{{ conf_server }}"

        - path: /usr/local/bin/check_slurm_controller_status.sh
          owner: root:root
          permissions: '{{ file_mode_755 }}'
          content: |
            {{ lookup('template', 'templates/slurm/check_slurm_controller_status.sh.j2') | indent(12) }}

        - path: /usr/local/bin/configure_vast_installation.sh
          owner: root:root
          permissions: '{{ file_mode_755 }}'
          content: |
            {{ lookup('template', 'templates/vast/configure_vast_installation.sh.j2') | indent(12) }}

        - path: /tmp/apptainer_mirror.conf
          permissions: '0644'
          content: |
            {{ lookup('template', 'templates/nodes/apptainer_mirror.conf.j2') | indent(12) }}
{% for pv_entry in cloud_init_groups_dict[functional_group_name].powervault_scripts | default([]) %}
        - path: /usr/local/bin/setup_iscsi_storage_{{ pv_entry.name }}.sh
          permissions: '{{ file_mode_755 }}'
          content: |
            {{ pv_entry.content | indent(12) }}
{% endfor %}

      runcmd:
        - /usr/local/bin/set-ssh.sh
        - /usr/local/bin/configure_vast_installation.sh
        # DOCA prerequisites - mount /cert and prepare for DOCA installation
        - mkdir -p {{ client_mount_path }}/slurm/ssh
        - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools
        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
        - mount -av
        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
        # DOCA and IB configuration - now ready before vendor_data mounts
        - bash /usr/local/bin/doca-install.sh || true
        - bash /usr/local/bin/configure-ib-network.sh
{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
{%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
{% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
        - {{ cmd }}
{% endfor %}
{% endif %}
{% raw %}
      {% if ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map %}
      {% set mymounts = ds.meta_data.instance_data.v1.vendor_data.groups.ssh.host_mount_map.get(local_hostname, {}) %}
      {% if mymounts %}
      {% for mount in mymounts.get("mounts", []) %}
        - mkdir -pv {{ mount[1] }}
        - echo "{{ mount | join(' ') }}" >> /etc/fstab
      {% endfor %}
        - mount -av
      {% for decoded_cmd in mymounts.get("runcmd", []) %}
        - {{ decoded_cmd }}
      {% endfor %}
      {% else %}
         - echo "No mount entries found for this host"
      {% endif %}
      {% endif %}
{% endraw %}
{% set fg_swap = cloud_init_groups_dict.get(functional_group_name, {}).get('swap', {}) %}
{% include 'configure_swap.yaml.j2' %}
        - mount -av
{% for pv_entry in cloud_init_groups_dict[functional_group_name].powervault_scripts | default([]) %}
        - /usr/local/bin/setup_iscsi_storage_{{ pv_entry.name }}.sh
{% endfor %}
        # Ensure Slurm NFS root is mounted at client_mount_path (e.g. /share_omnia)
        - mkdir -p {{ client_mount_path }}/slurm/ssh
        - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools
        # Additional Slurm login node mounts (excluding cert which was mounted earlier)
        - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/log/slurm  {{ slurm_slurmd_log_dir_effective }}   nfs defaults,_netdev 0 0" >> /etc/fstab
        - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool/slurmd      {{ slurm_slurmd_spool_dir_effective }}       nfs defaults,_netdev 0 0" >> /etc/fstab
        - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs defaults,_netdev 0 0" >> /etc/fstab
        - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/munge      /etc/munge       nfs defaults,_netdev 0 0" >> /etc/fstab
        - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
        - echo "{{ cloud_init_nfs_path}}/hpc_tools  /hpc_tools   nfs defaults,_netdev 0 0" >> /etc/fstab
        - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab
        - echo "{{ cloud_init_nfs_path }}/packages  /var/lib/packages   nfs defaults,_netdev 0 0" >> /etc/fstab
        - chmod {{ file_mode }} /etc/fstab
        - mount -av
        - /usr/local/bin/check_slurm_controller_status.sh
        - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_log_dir_effective }}
        - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_pid_dir_effective }}
        - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_spool_dir_effective }}
        - chown -R {{ munge_user }}:{{ munge_group }} /etc/munge/munge.key
        - chmod {{ file_mode_755 }} {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }}
        - chmod {{ file_mode_400 }} /etc/munge/munge.key
        - chmod {{ file_mode_755 }} /etc/slurm/epilog.d/
{% for epath in slurm_epilog_custom_paths %}
        - bash -c 'if [ ! -f "{{ epath }}" ]; then mkdir -p "$(dirname "{{ epath }}")"; printf "#!/bin/bash\n# Custom epilog script placeholder\n# Add your epilog commands here\n" > "{{ epath }}"; chown {{ slurm_user }}:{{ slurm_user }} "{{ epath }}"; chmod {{ file_mode_755 }} "{{ epath }}"; fi'
{% endfor %}

        - mkdir -p {{ slurm_slurmd_spool_dir_effective }}
        - chmod {{ file_mode_755 }} {{ slurm_slurmd_spool_dir_effective }}
        - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_spool_dir_effective }}
        - setenforce 0
        - systemctl enable firewalld
        - systemctl start firewalld
        - firewall-cmd --permanent --add-service=ssh
        - firewall-cmd --permanent --add-port={{ slurm_conf_dict.SrunPortRange }}/tcp
        - firewall-cmd --permanent --add-port={{ slurm_conf_dict.SrunPortRange }}/udp
        - firewall-cmd --permanent --add-port={{  slurm_conf_dict.SlurmdPort }}/tcp
        - firewall-cmd --permanent --add-port={{ slurm_conf_dict.SlurmdPort }}/udp
        
        # Add PXE network to trusted zone for ORTE communication
        - echo "[INFO] Adding PXE network to trusted zone for ORTE communication"
        - |
          bash -c '
          ADMIN_IP="{{ hostvars['localhost']['admin_nic_ip'] }}"
          NETMASK_BITS="{{ hostvars['localhost']['admin_netmask_bits'] }}"
          
          # Convert IP to integer and calculate network address
          ip_to_int() {
            local IFS=.
            read -r a b c d <<< "$1"
            echo $(( (a << 24) + (b << 16) + (c << 8) + d ))
          }
          
          int_to_ip() {
            local ip=$1
            echo "$(( (ip >> 24) & 255 )).$(( (ip >> 16) & 255 )).$(( (ip >> 8) & 255 )).$(( ip & 255 ))"  
          }
          
          ADMIN_IP_INT=$(ip_to_int "$ADMIN_IP")
          HOST_BITS=$(( 32 - NETMASK_BITS ))
          HOST_MASK=$(( (1 << HOST_BITS) - 1 ))
          NETWORK_MASK=$(( ~HOST_MASK & 0xFFFFFFFF ))
          NETWORK_INT=$(( ADMIN_IP_INT & NETWORK_MASK ))
          NETWORK_IP=$(int_to_ip "$NETWORK_INT")
          
          PXE_SUBNET="$NETWORK_IP/$NETMASK_BITS"
          echo "[INFO] Admin IP: $ADMIN_IP, Netmask: /$NETMASK_BITS, PXE Subnet: $PXE_SUBNET"
          firewall-cmd --zone=trusted --add-source="$PXE_SUBNET" --permanent
          '
        
        - firewall-cmd --reload
        - systemctl enable sshd
        - systemctl start sshd
        - systemctl enable munge
        - systemctl start munge
        - systemctl enable slurmd
        - systemctl start slurmd
        - systemctl daemon-reexec
        - systemctl restart sshd
        - mkdir -p /etc/containers/registries.conf.d
        - mv /tmp/apptainer_mirror.conf /etc/containers/registries.conf.d/apptainer_mirror.conf

{% if hostvars['localhost']['openldap_support'] %}
        - /usr/local/bin/update_ldap_conf.sh
        - mkdir /ldapcerts
        - echo "{{ cloud_init_nfs_path_openldap }}/certs                /ldapcerts       nfs defaults,_netdev 0 0" >> /etc/fstab
        - if ! mountpoint -q /home; then echo "{{ cloud_init_nfs_path_openldap }}/ldapuser /home nfs defaults,_netdev 0 0" >> /etc/fstab; fi
        - chmod {{ file_mode }} /etc/fstab
        - mount -a
        - yes | cp /ldapcerts/* /etc/openldap/certs
        - umount /ldapcerts

        - firewall-cmd --permanent --add-port={{ ldap_starttls_port }}/tcp
        - firewall-cmd --permanent --add-port={{ ldap_ssl_port }}/tcp
        - firewall-cmd --reload

        - setenforce 0
        - authselect select sssd with-mkhomedir --force
        - sudo systemctl enable --now oddjobd.service
        - sudo systemctl enable --now sssd
        - setsebool -P authlogin_nsswitch_use_ldap on
        - setsebool -P authlogin_yubikey on
        - sudo systemctl restart sssd
        - systemctl restart sshd
{% endif %}

{% if ldms_support %}
        - echo " Starting LDMS setup " | tee -a /var/log/ldms-cloudinit.log
        # Add NFS entry and mount
        - mkdir -p {{ client_mount_path }}
        - echo "{{ cloud_init_slurm_nfs_path }} {{ client_mount_path }} nfs defaults,_netdev 0 0" >> /etc/fstab
        - mount -av
        - /root/ldms_sampler.sh
{% endif %}
        - echo "Cloud-Init has completed successfully."