Merge pull request #24 from oci-hpc/dev

Pull request for marketplace release.

Author: MarcinZablocki

autoscaling/cleanup.sh

@@ -7,5 +7,10 @@ playbooks_path=$folder/../playbooks/
 inventory_path=$folder/clusters/$1
 
 ssh_options="-i ~/.ssh/id_rsa -o StrictHostKeyChecking=no"
-
-ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook $playbooks_path/destroy.yml -i $inventory_path/inventory
+if [[ "$2" == "FORCE" ]];
+then
+   echo Force Deletion 
+   ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook $playbooks_path/destroy.yml -i $inventory_path/inventory -e "force=yes"
+else
+    ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook $playbooks_path/destroy.yml -i $inventory_path/inventory
+fi

autoscaling/configure.sh

@@ -13,6 +13,8 @@ execution=1
 playbooks_path=$folder/../playbooks/
 inventory_path=$folder/clusters/$1
 
+ssh_options="-i ~/.ssh/cluster.key -o StrictHostKeyChecking=no"
+
 #
 # A little waiter function to make sure all the nodes are up before we start configure 
 #

autoscaling/create_cluster.sh

@@ -13,6 +13,9 @@ cd $folder/clusters/$2
 if [[ $3 == VM.Standard.E3.* ]]
 then  
   sed "s/##NODES##/$1/g;s/##NAME##/$2/g;s/##SHAPE##/VM.Standard.E3.Flex/g;s/##CN##/$4/g;s/##OCPU##/${3:15}/g" $folder/tf_init/variables.tf > variables.tf
+elif [[ $3 == VM.Standard.E4.* ]]
+then  
+  sed "s/##NODES##/$1/g;s/##NAME##/$2/g;s/##SHAPE##/VM.Standard.E4.Flex/g;s/##CN##/$4/g;s/##OCPU##/${3:15}/g" $folder/tf_init/variables.tf > variables.tf
 else
   sed "s/##NODES##/$1/g;s/##NAME##/$2/g;s/##SHAPE##/$3/g;s/##CN##/$4/g" $folder/tf_init/variables.tf > variables.tf
 fi
@@ -31,5 +34,5 @@ if [ $status -eq 0 ]
   else
     echo "Could not create $2 with $1 nodes in $runtime seconds"
     rm currently_building
-    $folder/delete_cluster.sh $2
+    $folder/delete_cluster.sh $2 FORCE
 fi

autoscaling/crontab/autoscale_slurm.sh

@@ -57,8 +57,11 @@ try:
     shapes['BM.Standard.E2.64']='amd264'
     for i in range(1,65):
         shapes['VM.Standard.E3.'+str(i)]='amd3'+str(i)
+        shapes['VM.Standard.E4.'+str(i)]='amd4'+str(i)
     shapes['BM.Standard.E3.128']='amd3128'
+    shapes['BM.Standard.E3.128']='amd4128'
     shapes['BM.HPC2.36']='hpc'
+    shapes['BM.Optimized3.36']='hpc2'
 
     def getstatus_slurm():
         cluster_to_build=[]
@@ -71,7 +74,7 @@ try:
                         if feature.startswith('VM') or feature.startswith('BM'):
                             shape=feature
                             break
-                    if shape == "BM.HPC2.36" or shape ==  "BM.GPU4.8":
+                    if shape == "BM.HPC2.36" or shape ==  "BM.GPU4.8" or shape ==  "BM.Optimized3.36":
                         CN = "true"
                     else:
                         CN = "false"
@@ -135,6 +138,8 @@ try:
         for shape in shapes.keys():
             available_names[shapes[shape]] = ["cluster-"+str(i) for i in range(1,cluster_names_number+1)]
         available_names['hpc']=["cluster-"+str(i) for i in range(1,hpc_cluster_names_number+1)]
+        available_names['hpc2']=["cluster-"+str(i) for i in range(1,hpc_cluster_names_number+1)]
+
         for clusterName in os.listdir(clusters_path):
             clusterType=clusterName.split('-')[-1]
             clusterNumber='-'.join(clusterName.split('-')[:2])

autoscaling/delete_cluster.sh

@@ -15,17 +15,26 @@ then
     echo "The cluster is already being destroyed"
 else
   echo $1 >> currently_destroying
-  terraform destroy -auto-approve >> $folder/logs/delete_$1_${date}.log 2>&1
-status=$?
-end=`date +%s`
-runtime=$((end-start))
-  if [ $status -eq 0 ]
+  $folder/cleanup.sh $1 >> $folder/logs/delete_$1_${date}.log 2>&1
+  status_initial_deletion=$?
+  if [ $status_initial_deletion -eq 0 ] || [[ $2 == FORCE ]]
   then
-    echo "Successfully deleted cluster $1 in $runtime seconds"
-    cd
-    rm -rf $folder/clusters/$1
+    $folder/cleanup.sh $1 FORCE >> $folder/logs/delete_$1_${date}.log 2>&1
+    terraform destroy -auto-approve >> $folder/logs/delete_$1_${date}.log 2>&1
+    status=$?
+    end=`date +%s`
+    runtime=$((end-start))
+    if [ $status -eq 0 ]
+    then
+      echo "Successfully deleted cluster $1 in $runtime seconds"
+      cd
+      rm -rf $folder/clusters/$1
+    else
+      echo "Could not delete cluster $1 (Time: $runtime seconds)"
+      rm currently_destroying
+    fi
   else
-    echo "Could not delete cluster $1 (Time: $runtime seconds)"
+    echo "Could not delete cluster $1 (Use FORCE to delete anyway)"
     rm currently_destroying
   fi
 fi

autoscaling/tf_init/bastion_update.tf

@@ -55,17 +55,3 @@ resource "null_resource" "configure" {
     command = "timeout 30m ${var.scripts_folder}/configure.sh ${local.cluster_name}"  
   }
 }
-
-resource "null_resource" "on_destroy" {
-  depends_on = [local_file.inventory]
-  triggers = {
-    scripts_folder = var.scripts_folder
-    cluster_name = local.cluster_name
-    bastion_path = local.bastion_path
-  }
-  provisioner "local-exec" {
-    command = "${self.triggers.scripts_folder}/cleanup.sh ${self.triggers.cluster_name}"
-    when    = destroy
-    on_failure = fail
-  }
-}

autoscaling/tf_init/cluster-network-configuration.tf

@@ -17,6 +17,9 @@ resource "oci_core_instance_configuration" "cluster-network-instance_configurati
         ssh_authorized_keys = file("/home/opc/.ssh/id_rsa.pub")
         user_data           = base64encode(data.template_file.config.rendered)
       }
+      agent_config {
+        is_management_disabled = true
+        }
       shape = var.cluster_network_shape
       source_details {
         source_type             = "image"

autoscaling/tf_init/instance-pool-configuration.tf

@@ -17,6 +17,9 @@ resource "oci_core_instance_configuration" "instance_pool_configuration" {
         ssh_authorized_keys = file("/home/opc/.ssh/id_rsa.pub")
         user_data           = base64encode(data.template_file.config.rendered)
       }
+      agent_config {
+        is_management_disabled = true
+        }
       shape = var.instance_pool_shape
 
       dynamic "shape_config" {

autoscaling/tf_init/inventory.tpl

@@ -5,9 +5,7 @@ ${bastion_name} ansible_host=${bastion_ip} ansible_user=opc role=bastion
 ${host} ansible_host=${ip} ansible_user=opc role=compute
 %{ endfor ~}
 [nfs]
-%{ if scratch_nfs ~}
 ${nfs}
-%{ endif ~}
 [all:children]
 bastion
 compute

autoscaling/tf_init/locals.tf

@@ -21,8 +21,8 @@ locals {
 //  image = (var.cluster_network && var.use_marketplace_image == true) || (var.cluster_network == false && var.use_marketplace_image == false) ? var.image : data.oci_core_images.linux.images.0.id
 
 //  is_bastion_flex_shape = var.bastion_shape == "VM.Standard.E3.Flex" ? [var.bastion_ocpus]:[]
-  is_instance_pool_flex_shape = var.instance_pool_shape == "VM.Standard.E3.Flex" ? [var.instance_pool_ocpus]:[]
-
+  is_instance_pool_flex_shape = length(regexall(".*VM.*E[3-4].*Flex$", var.instance_pool_shape)) > 0 ? [var.instance_pool_ocpus]:[]
+  
 //  bastion_mount_ip = var.bastion_block ? element(concat(oci_core_volume_attachment.bastion_volume_attachment.*.ipv4, [""]), 0) : "none"
 
   scratch_nfs_type = var.cluster_network ? var.scratch_nfs_type_cluster : var.scratch_nfs_type_pool