fix: recoverdata and log print (#3545)

* fix: ops log print

* fix log and try recover all table

Author: vagetablechicken

tools/openmldb_ops.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-
+# for Python 2, don't use f-string
 log = logging.getLogger(__name__)
 import os
 import sys
@@ -118,8 +118,8 @@ def RecoverPartition(executor, db, partitions, endpoint_status):
             db=db, table_name=table_name, pid=pid, leader_endpoint=leader_endpoint))
         status = executor.LoadTable(leader_endpoint, table_name, tid, pid)
         if not status.OK():
-            log.error("load table failed. db {db} name {table_name} tid {tid} pid {pid} endpoint {leader_endpoint} msg {status.GetMsg()}".format(
-                db=db, table_name=table_name, tid=tid, pid=pid, leader_endpoint=leader_endpoint, status=status))
+            log.error("load table failed. db {db} name {table_name} tid {tid} pid {pid} endpoint {leader_endpoint} msg {status}".format(
+                db=db, table_name=table_name, tid=tid, pid=pid, leader_endpoint=leader_endpoint, status=status.GetMsg()))
             return Status(-1, "recover partition failed")
     if not partitions[leader_pos].IsAlive():
         status =  executor.UpdateTableAlive(db, table_name, pid, leader_endpoint, "yes")
@@ -204,8 +204,9 @@ def RecoverData(executor):
             log.error("get all table failed")
             return
         for name in tables:
+            # if recover failed, continue to recover next table
             if not RecoverTable(executor, db, name).OK():
-                return
+                log.error("recover table failed. db {db} name {name}, check log for detail".format(db=db, name=name))
 
 def ChangeLeader(db, partition, src_endpoint, desc_endpoint, one_replica, restore = True):
     log.info(

tools/tool.py

@@ -16,6 +16,7 @@
 import subprocess
 import sys
 import time
+# for Python 2, don't use f-string
 log = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO, format = '%(levelname)s: %(message)s')
 
@@ -276,6 +277,7 @@ def LoadTable(self, endpoint, name, tid, pid, sync = True):
         cmd = list(self.tablet_base_cmd)
         cmd.append("--endpoint=" + self.endpoint_map[endpoint])
         cmd.append("--cmd=loadtable {} {} {} 0 8".format(name, tid, pid))
+        log.info("run {cmd}".format(cmd = cmd))
         status, output = self.RunWithRetuncode(cmd)
         time.sleep(1)
         if status.OK() and output.find("LoadTable ok") != -1:
@@ -289,12 +291,12 @@ def LoadTable(self, endpoint, name, tid, pid, sync = True):
                     if table_stat == "kTableNormal":
                         return Status()
                     elif table_stat == "kTableLoading" or table_stat == "kTableUndefined":
-                        log.info("table is loading... tid {tid} pid {pid}".format(tid, pid))
+                        log.info("table is loading... tid {tid} pid {pid}".format(tid = tid, pid = pid))
                     else:
-                        return Status(-1, "table stat is {table_stat}".format(table_stat))
+                        return Status(-1, "table stat is {table_stat}".format(table_stat = table_stat))
                 time.sleep(2)
 
-        return Status(-1, "execute load table failed")
+        return Status(-1, "execute load table failed, status {msg}, output {output}".format(msg = status.GetMsg(), output = output))
 
     def GetLeaderFollowerOffset(self, endpoint, tid, pid):
         cmd = list(self.tablet_base_cmd)