improve retry script

Author: yf225

benchmarks/run_input_shard.sh

@@ -9,9 +9,13 @@ OUTPUT_FILE="benchmarks_autotune_${TIMESTAMP}_input_shard_$((SHARD+1))_of_${WORL
 # Retry until success
 attempt=0
 while true; do
+# while (( attempt < 10 )); do
     attempt=$((attempt + 1))
     echo "Attempt $attempt: Running benchmark for shard $((SHARD+1))/${WORLD_SIZE}..."
 
+    # TIMESTAMP=$(date +%s)
+    # OUTPUT_FILE="benchmarks_autotune_${TIMESTAMP}_input_shard_$((SHARD+1))_of_${WORLD_SIZE}.txt"
+
     CUDA_VISIBLE_DEVICES=$((RANK_OFFSET+SHARD)) python benchmarks/run.py --input-shard $((SHARD+1))/${WORLD_SIZE} --metrics tflops,gbps,speedup >"$OUTPUT_FILE" 2>&1
 
     exit_code=$?
@@ -20,7 +24,7 @@ while true; do
         break
     else
         echo "Failed with exit code $exit_code. Retrying..."
-        sleep 5  # Optional: wait 5 seconds before retrying
+        sleep 10  # wait a few seconds before retrying
     fi
 done