Skip to content

Commit 405b571

Browse files
arnaudfroidmontarnaudfroidmont
committed
Fix work request issue in case of inst-pool fail
1 parent e5ee197 commit 405b571

File tree

1 file changed

+59
-52
lines changed

1 file changed

+59
-52
lines changed

‎bin/create_cluster.sh‎

Lines changed: 59 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -96,63 +96,70 @@ do
9696
end_timestamp=`date -u +'%F %T'`
9797
runtime=$((end-start))
9898
if [ $status -eq 0 ]
99+
then
100+
echo "Successfully created $2 in $runtime seconds"
101+
rm currently_building
102+
if [ -f $monitoring_folder/activated ]
99103
then
100-
echo "Successfully created $2 in $runtime seconds"
101-
rm currently_building
102-
if [ -f $monitoring_folder/activated ]
103-
then
104-
ocid=`tail $logs_folder/create_$2_${date}.log | grep "cluster_ocid =" | awk '{print $3}'`
105-
ips=`tail $logs_folder/create_$2_${date}.log | grep "private_ips =" | awk '{print $3}'`
106-
hostnames=`tail $logs_folder/create_$2_${date}.log | grep "hostnames =" | awk '{print $3}'`
107-
ocids=`tail $logs_folder/create_$2_${date}.log | grep "ocids =" | awk '{print $3}'`
108-
mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; UPDATE cluster_log.clusters SET cluster_OCID='${ocid:1:-1}',created='$end_timestamp',state='running',creation_time=SEC_TO_TIME($runtime) WHERE id='$2_${date}';" >> $logs_folder/create_$2_${date}.log 2>&1
109-
export IFS=","
110-
for ip in ${ips:1:-5}; do
111-
ip_array+=( $ip )
112-
done
113-
for ocid in ${ocids:1:-5}; do
114-
ocid_array+=( $ocid )
115-
done
116-
for hostname in ${hostnames:1:-1}; do
117-
hostname_array+=( $hostname )
118-
done
119-
for index in "${!ip_array[@]}"; do
120-
mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; UPDATE nodes SET created='$end_timestamp',state='running',hostname='${hostname_array[$index]}',ip='${ip_array[$index]}',node_OCID='${ocid_array[$index]}' WHERE cluster_id='$2_${date}' AND cluster_index=$(($index+1));" >> $logs_folder/create_$2_${date}.log 2>&1
121-
done
122-
fi
123-
break
124-
else
125-
ERROR_MSG=`cat $logs_folder/create_$2_${date}.log | grep Error: | grep -o 'Output.*'`
126-
if [ "$ERROR_MSG" == "" ]
127-
then
128-
ERROR_MSG=`cat $logs_folder/create_$2_${date}.log | grep Error:`
129-
fi
130-
comp_tmp=`curl -sH "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/ | jq .compartmentId`
131-
compartment_ocid=${comp_tmp:1:-1}
104+
ocid=`tail $logs_folder/create_$2_${date}.log | grep "cluster_ocid =" | awk '{print $3}'`
105+
ips=`tail $logs_folder/create_$2_${date}.log | grep "private_ips =" | awk '{print $3}'`
106+
hostnames=`tail $logs_folder/create_$2_${date}.log | grep "hostnames =" | awk '{print $3}'`
107+
ocids=`tail $logs_folder/create_$2_${date}.log | grep "ocids =" | awk '{print $3}'`
108+
mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; UPDATE cluster_log.clusters SET cluster_OCID='${ocid:1:-1}',created='$end_timestamp',state='running',creation_time=SEC_TO_TIME($runtime) WHERE id='$2_${date}';" >> $logs_folder/create_$2_${date}.log 2>&1
109+
export IFS=","
110+
for ip in ${ips:1:-5}; do
111+
ip_array+=( $ip )
112+
done
113+
for ocid in ${ocids:1:-5}; do
114+
ocid_array+=( $ocid )
115+
done
116+
for hostname in ${hostnames:1:-1}; do
117+
hostname_array+=( $hostname )
118+
done
119+
for index in "${!ip_array[@]}"; do
120+
mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; UPDATE nodes SET created='$end_timestamp',state='running',hostname='${hostname_array[$index]}',ip='${ip_array[$index]}',node_OCID='${ocid_array[$index]}' WHERE cluster_id='$2_${date}' AND cluster_index=$(($index+1));" >> $logs_folder/create_$2_${date}.log 2>&1
121+
done
122+
fi
123+
break
124+
else
125+
ERROR_MSG=`cat $logs_folder/create_$2_${date}.log | grep Error: | grep -o 'Output.*'`
126+
if [ "$ERROR_MSG" == "" ]
127+
then
128+
ERROR_MSG=`cat $logs_folder/create_$2_${date}.log | grep Error:`
129+
fi
130+
comp_tmp=`curl -sH "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/ | jq .compartmentId`
131+
compartment_ocid=${comp_tmp:1:-1}
132132

133-
inst_pool_ocid=`oci compute-management instance-pool list --compartment-id $compartment_ocid --auth instance_principal --region $region --all --display-name $2 | jq '.data | sort_by(."time-created" | split(".") | .[0] | strptime("%Y-%m-%dT%H:%M:%S")) |.[-1] .id'` >> $logs_folder/create_$2_${date}.log 2>&1
134-
if [ "$inst_pool_ocid" == "" ]
135-
then
136-
inst_pool_work_request_error_messages=""
137-
else
138-
requestID=`oci work-requests work-request list --compartment-id $compartment_ocid --auth instance_principal --region $region --all --resource-id ${inst_pool_ocid:1:-1} | jq '.data | .[] | select(."operation-type"=="LaunchInstancesInPool") | .id'` >> $logs_folder/create_$2_${date}.log 2>&1
139-
inst_pool_work_request_error_messages=`oci work-requests work-request-error list --work-request-id ${requestID:1:-1} --auth instance_principal --region $region --all | jq '.data | .[] | .message '` >> $logs_folder/create_$2_${date}.log 2>&1
140-
fi
141-
if [ "$inst_pool_work_request_error_messages" == "" ]
142-
then
143-
cn_ocid=`oci compute-management cluster-network list --compartment-id $compartment_ocid --auth instance_principal --region $region --all --display-name $2 | jq '.data | sort_by(."time-created" | split(".") | .[0] | strptime("%Y-%m-%dT%H:%M:%S")) |.[-1] .id'` >> $logs_folder/create_$2_${date}.log 2>&1
133+
inst_pool_ocid=`oci compute-management instance-pool list --compartment-id $compartment_ocid --auth instance_principal --region $region --all --display-name $2 | jq '.data | sort_by(."time-created" | split(".") | .[0] | strptime("%Y-%m-%dT%H:%M:%S")) |.[-1] .id'` >> $logs_folder/create_$2_${date}.log 2>&1
134+
if [ "$inst_pool_ocid" == "" ]
135+
then
136+
inst_pool_work_request_error_messages=""
137+
else
138+
requestID=`oci work-requests work-request list --compartment-id $compartment_ocid --auth instance_principal --region $region --all --resource-id ${inst_pool_ocid:1:-1} | jq '.data | .[] | select(."operation-type"=="LaunchInstancesInPool") | .id'` >> $logs_folder/create_$2_${date}.log 2>&1
139+
inst_pool_work_request_error_messages=`oci work-requests work-request-error list --work-request-id ${requestID:1:-1} --auth instance_principal --region $region --all | jq '.data | .[] | .message '` >> $logs_folder/create_$2_${date}.log 2>&1
140+
fi
141+
if [ "$inst_pool_work_request_error_messages" == "" ] && [ "$cluster_network" == "true" ]
142+
then
143+
cn_ocid=`oci compute-management cluster-network list --compartment-id $compartment_ocid --auth instance_principal --region $region --all --display-name $2 | jq '.data | sort_by(."time-created" | split(".") | .[0] | strptime("%Y-%m-%dT%H:%M:%S")) |.[-1] .id'` >> $logs_folder/create_$2_${date}.log 2>&1
144+
if [ "$cn_ocid" == "" ]
145+
then
146+
cn_work_request_error_messages=""
147+
else
144148
requestID=`oci work-requests work-request list --compartment-id $compartment_ocid --auth instance_principal --region $region --all --resource-id ${cn_ocid:1:-1} | jq '.data | .[] | select(."operation-type"=="CreateClusterNetworkReservation") | .id'` >> $logs_folder/create_$2_${date}.log 2>&1
145149
cn_work_request_error_messages=`oci work-requests work-request-log-entry list --work-request-id ${requestID:1:-1} --auth instance_principal --region $region --all | jq '.data | .[] | .message '` >> $logs_folder/create_$2_${date}.log 2>&1
146-
fi
147-
echo "Could not create $2 with $1 nodes in $runtime seconds"
148-
echo "$ERROR_MSG $inst_pool_work_request_error_messages $cn_work_request_error_messages" | tee -a $logs_folder/create_$2_${date}.log 2>&1
150+
fi
151+
else
152+
cn_work_request_error_messages=""
153+
fi
154+
echo "Could not create $2 with $1 nodes in $runtime seconds"
155+
echo "$ERROR_MSG $inst_pool_work_request_error_messages $cn_work_request_error_messages" | tee -a $logs_folder/create_$2_${date}.log 2>&1
149156

150-
if [ -f $monitoring_folder/activated ]
151-
then
152-
mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; INSERT INTO cluster_log.errors_timeserie (cluster_id,state,error_log,error_type,nodes,created_on_m,class_name) VALUES ('$2_${date}','creation','$logs_folder/create_$2_${date}.log','$ERROR_MSG $inst_pool_work_request_error_messages $cn_work_request_error_messages','$1','$end_timestamp','$4');" >> $logs_folder/create_$2_${date}.log 2>&1
153-
mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; UPDATE cluster_log.clusters SET state='deleting',creation_error='`tail $logs_folder/create_$2_${date}.log | grep Error`' WHERE id='$2_${date}';" >> $logs_folder/create_$2_${date}.log 2>&1
154-
fi
155-
rm currently_building
157+
if [ -f $monitoring_folder/activated ]
158+
then
159+
mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; INSERT INTO cluster_log.errors_timeserie (cluster_id,state,error_log,error_type,nodes,created_on_m,class_name) VALUES ('$2_${date}','creation','$logs_folder/create_$2_${date}.log','$ERROR_MSG $inst_pool_work_request_error_messages $cn_work_request_error_messages','$1','$end_timestamp','$4');" >> $logs_folder/create_$2_${date}.log 2>&1
160+
mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; UPDATE cluster_log.clusters SET state='deleting',creation_error='`tail $logs_folder/create_$2_${date}.log | grep Error`' WHERE id='$2_${date}';" >> $logs_folder/create_$2_${date}.log 2>&1
161+
fi
162+
rm currently_building
156163
fi
157164
done
158165

0 commit comments

Comments
 (0)