[VOQ][saidump] Modify generate_dump: replace save_saidump with save_saidump_by_route_size (#2972)

* * [saidump]
•	Saidump for DNX-SAI sonic-net/sonic-buildimage#13561

Solution and modification:
To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it.

(1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host.
(2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format.
(3) Add a new script file: files/scripts/saidump.sh, to do the below steps
  For each ASIC0, such as ASIC0,

  #1. Save the Redis data.
  sudo sonic-db-cli -n asic$1 SAVE > /dev/null

  #2. Move dump files to /var/run/redisX/
  docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/"

  #3. Run rdb command to convert the dump files into JSON files
  sudo python /usr/local/bin/rdb --command json  /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null

  #4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output.
  docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json"

  #5. clear
  sudo rm -f /var/run/redis$1/dump.rdb
  sudo rm -f /var/run/redis$1/dump.json

(4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh
* * [saidump]
•	Saidump for DNX-SAI sonic-net/sonic-buildimage#13561

Author: JunhongMao

scripts/generate_dump

@@ -50,6 +50,7 @@ SKIP_BCMCMD=0
 SAVE_STDERR=true
 RETURN_CODE=$EXT_SUCCESS
 DEBUG_DUMP=false
+ROUTE_TAB_LIMIT_DIRECT_ITERATION=24000
 
 # lock dirs/files
 LOCKDIR="/tmp/techsupport-lock"
@@ -863,24 +864,114 @@ save_redis() {
 }
 
 ###############################################################################
-# SAI DUMP from syncd
+# GET ROUTE table size by ASIC id and ip version
+# Globals:
+#  TIMEOUT_MIN
+#  TIMEOUT_EXIT_CODE
+# Arguments:
+#  asic id
+#  IP version
+# Returns:
+#  Status: 0 success, otherwise failure
+###############################################################################
+get_route_table_size_by_asic_id_and_ipver() {
+    local asic_id="$1"
+    local ip_ver="$2"
+    local filepath="/tmp/route_summary.txt"
+    local ns=""
+    RC=0
+
+    if [[ $NUM_ASICS -gt 1 ]] ; then
+        ns="-n ${asic_id}"
+    fi
+
+    if [ $ip_ver = "ipv4" ]; then
+        cmd="vtysh ${ns} -c 'show ip route summary json'"
+    elif [ $ip_ver = "ipv6" ]; then
+        cmd="vtysh ${ns} -c 'show ipv6 route summary json'"
+    else
+        echo "Wrong argument $ip_ver."
+        return 255
+    fi
+
+    local timeout_cmd="timeout --foreground ${TIMEOUT_MIN}m"
+    local cmds="$cmd > '$filepath'"
+
+    eval "${timeout_cmd} bash -c \"${cmds}\"" || RC=$?
+
+    if [ $RC -eq $TIMEOUT_EXIT_CODE ]; then
+        echo "Command: $cmds timedout after ${TIMEOUT_MIN} minutes."
+        return $RC
+    elif [ $RC -ne 0 ]; then
+        echo "Command: $cmds failed with RC $RC"
+        return $RC
+    fi
+
+    local route_tab_size=$(python3 -c "\
+import json
+with open('$filepath') as json_file:
+    data = json.load(json_file)
+    print(data['routesTotal'])")
+    rm $filepath
+    echo "$route_tab_size"
+}
+
+###############################################################################
+# SAI DUMP based on the route table size
+# if the route table has more than ROUTE_TAB_LIMIT_DIRECT_ITERATION
+# then dump by Redis Save command,
+# otherwize, dump it by directly iteration the Redis
+#
 # Globals:
 #  NUM_ASICS
+#  ROUTE_TAB_LIMIT_DIRECT_ITERATION
 # Arguments:
 #  None
 # Returns:
 #  None
 ###############################################################################
-save_saidump() {
+save_saidump_by_route_size() {
     trap 'handle_error $? $LINENO' ERR
-    if [[ ( "$NUM_ASICS" == 1 ) ]] ; then
-        save_cmd "docker exec syncd saidump" "saidump"
-    else
-        for (( i=0; i<$NUM_ASICS; i++ ))
-        do
-            save_cmd "docker exec syncd$i saidump" "saidump$i"
-        done
-    fi
+
+    for (( i=0; i<$NUM_ASICS; i++ ))
+    do
+        route_size_ipv4=`get_route_table_size_by_asic_id_and_ipver $i ipv4`
+        ret=$?
+
+        if [ $ret -ne 0 ]; then
+            echo "Get route table's size by asicid $i and ipv4 failed."
+            return $ret
+        fi
+
+        route_size_ipv6=`get_route_table_size_by_asic_id_and_ipver $i ipv6`
+        ret=$?
+
+        if [ $ret -ne 0 ]; then
+            echo "Get route table's size by asicid $i and ipv6 failed."
+            return $ret
+        fi
+
+        route_size=`expr $route_size_ipv4 + $route_size_ipv6`
+        echo "The route table's size is $route_size(ipv4 $route_size_ipv4, ipv6 $route_size_ipv6)"
+
+        if [[ $route_size -gt $ROUTE_TAB_LIMIT_DIRECT_ITERATION ]]; then
+            echo "Dump by using Redis SAVE."
+
+            if [[ ( "$NUM_ASICS" == 1 ) ]] ; then
+                save_cmd "docker exec syncd saidump.sh" "saidump"
+            else
+                save_cmd "docker exec syncd$i saidump.sh" "saidump$i"
+            fi
+        else
+            echo "Dump by using direct iteration of Redis DB."
+
+            if [[ ( "$NUM_ASICS" == 1 ) ]] ; then
+                save_cmd "docker exec syncd saidump" "saidump"
+            else
+                save_cmd "docker exec syncd$i saidump" "saidump$i"
+            fi
+        fi
+    done
 }
 
 ###############################################################################
@@ -1807,9 +1898,7 @@ main() {
     save_cmd "ps -AwwL -o user,pid,lwp,ppid,nlwp,pcpu,pri,nice,vsize,rss,tty,stat,wchan:12,start,bsdtime,command" "ps.extended" &
     wait
 
-    if [[ "$device_type" != "SpineRouter" ]]; then
-        save_saidump
-    fi
+    save_saidump_by_route_size
 
     if [ "$asic" = "barefoot" ]; then
         collect_barefoot