Update llama3 70b and removes the neuron-top in update_neuron_sdk.sh script (#530)

* Update the Neuron SDK to 2.21.0

* Update the Llama3-70B pretraining with the Neuron SDK 2.21

* Fix a typo

* Add --hw_backend trn1 in the convert_checkpoint command

* More update

* Update the update_neuron_sdk.sh by removing the neuron-top check

* Keep enable_update_neuron_sdk as Flase by default

* Update automate-eks-cluster-creation.sh (#529)

Minor bug fix

* Update according to the review comments.

* minor updates in doc

---------

Co-authored-by: Aman Shanbhag <[email protected]>
Co-authored-by: Keita Watanabe <[email protected]>

Author: 3 people

1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/update_neuron_sdk.sh

@@ -1,33 +1,28 @@
 #!/bin/bash
 set -euxo pipefail
-if neuron-top -v &> /dev/null
-then 
-    echo "Update to Neuron SDK Release 2.21.0"
-    # Configure Linux for Neuron repository updates
-    . /etc/os-release
 
-    sudo echo "deb https://apt.repos.neuron.amazonaws.com ${VERSION_CODENAME} main" | sudo tee /etc/apt/sources.list.d/neuron.list 
-    wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | sudo apt-key add -
+echo "Update to Neuron SDK Release 2.21.0"
+# Configure Linux for Neuron repository updates
+. /etc/os-release
 
-    # Update OS packages
-    sudo apt-get update -y
+sudo echo "deb https://apt.repos.neuron.amazonaws.com ${VERSION_CODENAME} main" | sudo tee /etc/apt/sources.list.d/neuron.list 
+wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | sudo apt-key add -
 
-    # Install git
-    sudo apt-get install git -y
+# Update OS packages
+sudo apt-get update -y
 
-    # Remove preinstalled packages and Install Neuron Driver and Runtime
-    sudo apt-get remove aws-neuron-dkms -y
-    sudo apt-get remove aws-neuronx-dkms -y
-    sudo apt-get remove aws-neuronx-oci-hook -y
-    sudo apt-get remove aws-neuronx-runtime-lib -y
-    sudo apt-get remove aws-neuronx-collectives -y
-    sudo apt-get install aws-neuronx-dkms=2.19.64.0 -y
-    sudo apt-get install aws-neuronx-oci-hook=2.6.36.0 -y
-    sudo apt-get install aws-neuronx-runtime-lib=2.23.110.0* -y
-    sudo apt-get install aws-neuronx-collectives=2.23.133.0* -y
+# Remove preinstalled packages and Install Neuron Driver and Runtime
+sudo apt-get remove aws-neuron-dkms -y
+sudo apt-get remove aws-neuronx-dkms -y
+sudo apt-get remove aws-neuronx-oci-hook -y
+sudo apt-get remove aws-neuronx-runtime-lib -y
+sudo apt-get remove aws-neuronx-collectives -y
+sudo apt-get install aws-neuronx-dkms=2.19.64.0 -y
+sudo apt-get install aws-neuronx-oci-hook=2.6.36.0 -y
+sudo apt-get install aws-neuronx-runtime-lib=2.23.110.0* -y
+sudo apt-get install aws-neuronx-collectives=2.23.133.0* -y
 
-    # Remove pre-installed package and Install Neuron Tools
-    sudo apt-get remove aws-neuron-tools  -y
-    sudo apt-get remove aws-neuronx-tools  -y
-    sudo apt-get install aws-neuronx-tools=2.19.0.0 -y
-fi
+# Remove pre-installed package and Install Neuron Tools
+sudo apt-get remove aws-neuron-tools  -y
+sudo apt-get remove aws-neuronx-tools  -y
+sudo apt-get install aws-neuronx-tools=2.20.204.0 -y