V3.5.0

Author: gys-git

_cmd.py

@@ -749,9 +749,15 @@ def init(self, cmd, args, need_deploy_name=True):
         return self
 
     def get_obd_namespaces_data(self, obd):
-        data = {}
+        data = {
+            "component": {},
+            "error_messages": ''
+        }
+        comp_data = data['component']
         for component, _ in obd.namespaces.items():
-            data[component] = _.get_variable('run_result')
+            comp_data[component] = _.get_variable('run_result')
+        error = obd.stdio.get_error_buffer().read()
+        data['error_messages'] = error
         return data
 
 
@@ -1378,6 +1384,7 @@ class ClusterTenantOptimizeCommand(ClusterMirrorCommand):
     def __init__(self):
         super(ClusterTenantOptimizeCommand, self).__init__('optimize','Optimizing existing tenant scenarios')
         self.parser.add_option('-o', '--optimize', type='string', help='Optimize scenarios,the default is consistent with the cluster dimension.\n{express_oltp, complex_oltp, olap, htap, kv}\nSupported since version 4.3.')
+        self.parser.add_option('-p', '--tenant-root-password', type='string', help="tenant root password.")
 
     def init(self, cmd, args):
         super(ClusterTenantOptimizeCommand, self).init(cmd, args)
@@ -2602,16 +2609,27 @@ def __init__(self):
         super(HostCommand, self).__init__('host', 'Host tools')
         self.register_command(HostPrecheckCommand())
         self.register_command(HostInitCommand())
+        self.register_command(HostUserCommand())
+
+
+class HostUserCommand(MajorCommand):
+
+    def __init__(self):
+        super(HostUserCommand, self).__init__('user', 'Host user tools')
+        self.register_command(HostUserInitCommand())
 
 
 class HostPrecheckCommand(ObdCommand):
 
     def __init__(self):
         super(HostPrecheckCommand, self).__init__('precheck', 'Pre check host system parameters')
         self.parser.add_option('-p', '--password', type='str', help="Password of username, default is empty.")
+        self.parser.add_option('-u', '--username', type='str', help="Users to be checked (default: ssh user).")
+        self.parser.add_option('--ssh-key-file', type='string', help="ssh key file")
+
     def init(self, cmd, args):
         super(HostPrecheckCommand, self).init(cmd, args)
-        self.parser.set_usage('%s <username>  <server ip>' % self.prev_cmd)
+        self.parser.set_usage('%s <ssh username>  <server ip>' % self.prev_cmd)
         return self
 
     def _do_command(self, obd):
@@ -2632,10 +2650,12 @@ class HostInitCommand(ObdCommand):
     def __init__(self):
         super(HostInitCommand, self).__init__('init', 'Init server environment.')
         self.parser.add_option('-p', '--password', type='str', help="Password of username, default is empty.")
+        self.parser.add_option('-u', '--username', type='str', help="Users to be checked (default: ssh user).")
+        self.parser.add_option('--ssh-key-file', type='string', help="ssh key file")
 
     def init(self, cmd, args):
         super(HostInitCommand, self).init(cmd, args)
-        self.parser.set_usage('%s <username>  <server ip>' % self.prev_cmd)
+        self.parser.set_usage('%s <ssh username>  <server ip>' % self.prev_cmd)
         return self
 
     def _do_command(self, obd):
@@ -2651,6 +2671,18 @@ def _do_command(self, obd):
             return self._show_help()
 
 
+class HostUserInitCommand(ObdCommand):
+
+    def __init__(self):
+        super(HostUserInitCommand, self).__init__('init', 'Init host user.')
+        self.parser.add_option('-u', '--username', type='str', help="username, default is admin.")
+        self.parser.add_option('-p', '--password', type='str', help="Password of username.")
+        self.parser.add_option('--host', type='str', help="host ip, default is 127.0.0.1.")
+
+    def _do_command(self, obd):
+        return obd.host_user_init()
+
+
 class DevHostInitCommand(ObdCommand):
 
     def __init__(self):

_environ.py

@@ -59,7 +59,7 @@
 # obdeploy install path. default /usr/obd/
 ENV_OBD_INSTALL_PATH = "OBD_INSTALL_PATH"
 
-# obd web idle minite time before shutdown. default 30
+# obd web idle minite time before shutdown. default 120
 # if you do not want to set idle time, you can set it to "infinity"
 ENV_IDLE_TIME_BEFORE_SHUTDOWN_MINITES = "IDLE_TIME_BEFORE_SHUTDOWN_MINITES"
 

_stdio.py

@@ -403,6 +403,7 @@ def __init__(self,
         self._output_is_tty = False
         self._input_is_tty = False
         self._exit_buffer = SetBufferIO()
+        self._error_buffer = SetBufferIO()
         self.set_input_stream(input_stream)
         self.set_output_stream(output_stream)
 
@@ -551,6 +552,11 @@ def get_exit_buffer(self):
             return self._root_io.get_exit_buffer()
         return self._exit_buffer
 
+    def get_error_buffer(self):
+        if self._root_io:
+            return self._root_io.get_error_buffer()
+        return self._error_buffer
+
     def _start_buffer_io(self):
         if self._root_io:
             return False
@@ -811,6 +817,9 @@ def warn(self, msg, *args, **kwargs):
         self._print(MsgLevel.WARN, msg, prev_msg=self.WARNING_PREV.format(self.isatty()), *args, **kwargs)
 
     def error(self, msg, *args, **kwargs):
+        error_buffer = self.get_error_buffer()
+        error_msg = self.log_masking(str(msg), ip_masking=True)
+        error_buffer.write(error_msg)
         self._print(MsgLevel.ERROR, msg, prev_msg=self.ERROR_PREV.format(self.isatty()), *args, **kwargs)
 
     def critical(self, msg, *args, **kwargs):
@@ -824,18 +833,18 @@ def critical(self, msg, *args, **kwargs):
             self.exit(code)
 
     def contains_keys(self, msg):
-        keywords = ["IDENTIFIED", "PASSWORD", "CONNECT", "EXECUTER", "CLIENT", "PASSWD", "_PASSKEY", "SUDO", "ACCESS_"]
+        keywords = ["IDENTIFIED", "PASSWORD", "CONNECT", "EXECUTER", "CLIENT", "PASSWD", "_PASSKEY", "SUDO", "ACCESS_", "HOST INIT", "CHPASSWD"]
         return any(keyword in msg.upper() for keyword in keywords)
 
     def table_log_masking(self, msg):
-        regex = r'(\|\s*(http://[^\s]+)\s*\|\s*admin\s*\|\s*)(\S+)(\s*\|\s*active\s*\|)'
+        regex = r"(\|\s*http://[^\s]+\s*\|\s*\S+\s*\|\s*')([^']*)('\s*\|\s*active\s*\|)"
         pattern = re.compile(regex)
         str_msg = str(msg)
         if 'active' in str_msg:
             match = re.search(pattern, str_msg)
             if match:
-                masked_password = "*"*len(match.group(3))
-                str_msg = pattern.sub(rf"\1{masked_password}\4", str_msg)
+                masked_password = "*"*len(match.group(2))
+                str_msg = pattern.sub(rf"\1{masked_password}\3", str_msg)
         elif 'access_' in str_msg:
             access_regex = r"(access_id=)[^&]*|(access_key=)[^&,| ]*"
             access_pattern = re.compile(access_regex)
@@ -853,7 +862,7 @@ def replace_with_stars(match):
     def log_masking_static(msg):
         def contains_keys(msg):
             keywords = ["IDENTIFIED", "PASSWORD", "CONNECT", "EXECUTER", "CLIENT", "PASSWD", "_PASSKEY", "SUDO",
-                        "ACCESS_"]
+                        "ACCESS_", "HOST INIT", "CHPASSWD"]
             return any(keyword in msg.upper() for keyword in keywords)
 
         log_regex = [
@@ -891,6 +900,18 @@ def contains_keys(msg):
             if "IDENTIFIED BY" in msg:
                 return desensitize_sql_pwd(msg)
 
+            if "host init" in msg:
+                host_reg = r'(obd host init\s+.*?)(-p\s+)([^\s\'\"\`]+)'
+                host_pattern = re.compile(host_reg)
+                if host_pattern.search(msg):
+                    return host_pattern.sub(r'\1\2******', msg)
+
+            if "chpasswd" in msg:
+                chpasswd_reg = r'(echo\s*"[^:]+:)([^"]+)(".*chpasswd)'
+                chpasswd_pattern = re.compile(chpasswd_reg)
+                if chpasswd_pattern.search(msg):
+                    return chpasswd_pattern.sub(r'\1******\3', msg)
+
             pwd_args_regex = r"(_password \S+.*args:\s*\[['\"]?)([^\s'\"']+)(['\"]*)"
             arg_pattern = re.compile(pwd_args_regex)
             if arg_pattern.search(msg):
@@ -910,9 +931,15 @@ def contains_keys(msg):
             pwd_pattern = re.compile(echo_pwd_regex)
             if pwd_pattern.search(msg):
                 return pwd_pattern.sub(r"\1******\3", msg)
+
+            opts_regex = r'(\'password\':\s*\')([^\']+)(\')'
+            opts_pattern = re.compile(opts_regex)
+            if opts_pattern.search(msg):
+                return opts_pattern.sub(r"\1******\3", msg)
+
         return msg
 
-    def log_masking(self, msg):
+    def log_masking(self, msg, ip_masking=False):
         log_regex = [
             r"((-P\s*\S+\s+.*?)-p\s*['\"]?)([^\s'\"']+)(['\"]*)",
             r"(_PASSWORD\s*(=|to)\s*['\"]*)([^\s'\"']+)(['\"]*)",
@@ -947,7 +974,20 @@ def log_masking(self, msg):
 
             if "IDENTIFIED BY" in msg:
                 return desensitize_sql_pwd(msg)
-            
+
+            if "host init" in msg:
+                host_reg = r'(obd host init\s+.*?)(-p\s+)([^\s\'\"\`]+)'
+                host_pattern = re.compile(host_reg)
+                if host_pattern.search(msg):
+                    return host_pattern.sub(r'\1\2******', msg)
+
+            if "chpasswd" in msg:
+                chpasswd_reg = r'(echo\s*"[^:]+:)([^"]+)(".*chpasswd)'
+                chpasswd_pattern = re.compile(chpasswd_reg)
+                if chpasswd_pattern.search(msg):
+                    return chpasswd_pattern.sub(r'\1******\3', msg)
+
+
             pwd_args_regex = r"(_password \S+.*args:\s*\[['\"]?)([^\s'\"']+)(['\"]*)"
             arg_pattern = re.compile(pwd_args_regex)
             if arg_pattern.search(msg):
@@ -967,6 +1007,16 @@ def log_masking(self, msg):
             pwd_pattern = re.compile(echo_pwd_regex)
             if pwd_pattern.search(msg):
                 return pwd_pattern.sub(r"\1******\3", msg)
+
+            opts_regex = r'(\'password\':\s*\')([^\']+)(\')'
+            opts_pattern = re.compile(opts_regex)
+            if opts_pattern.search(msg):
+                return opts_pattern.sub(r"\1******\3", msg)
+
+        if ip_masking:
+            ipv4_pattern = r'\b(?:\d{1,3}\.){2}\d{1,3}\.\d{1,3}\b'
+            msg = re.sub(ipv4_pattern, "***.***.***.***", msg)
+
         return msg
 
     def verbose(self, msg, *args, **kwargs):

const.py

@@ -125,7 +125,7 @@
 STAGE_NINTH = 90
 STAGE_TENTH = 100
 
-IDLE_TIME_BEFORE_SHUTDOWN_MINITES = 30
+IDLE_TIME_BEFORE_SHUTDOWN_MINITES = 120
 
 #obshell task type
 TENANT_BACKUP = 'backup'

core.py

@@ -2435,6 +2435,7 @@ def create_standby_tenant(self, standby_deploy_name, primary_deploy_name, primar
             return False
 
         self.set_repositories(standby_repositories)
+        self.search_param_plugin_and_apply(standby_repositories, standby_deploy.deploy_config)
         standby_type = getattr(self.options, 'type')
         if standby_type not in [SERVICE_MODE, LOCATION_MODE]:
             self._call_stdio('error', "The value of type must be 'SERVICE' or 'LOCATION' and cannot be other values")
@@ -6140,6 +6141,7 @@ def precheck_host(self, username, host, dev=False):
                 host,
                 username,
                 password,
+                key_filename=getattr(self.options, 'ssh_key_file', None)
             ),
             self.stdio
         )
@@ -6154,17 +6156,6 @@ def precheck_host(self, username, host, dev=False):
         workflows = self.get_workflows('precheck')
         if not self.run_workflow(workflows, **{'host_tool': {'host_clients': clients}}):
             return False
-
-        need_change_servers_vars = self.get_namespace('host_tool').get_return('precheck').get_return('need_change_servers_vars')
-        print_data = []
-        for v in need_change_servers_vars.values():
-            print_data.extend(v)
-        if not print_data:
-            self._call_stdio('print', FormatText.success('No need to change system parameters'))
-            return True
-        self._call_stdio('print_list', print_data, ['ip', 'need_change_var', 'current_value', 'target_value'],
-                         lambda x: [x['server'], x['var'], x['current_value'], x['value']],
-                         title='System Parameter Change List')
         if dev:
             return 100
         return True
@@ -6179,6 +6170,7 @@ def init_host(self, username, host, dev=False):
                 host,
                 username,
                 password,
+                key_filename=getattr(self.options, 'ssh_key_file', None)
             ),
             self.stdio
         )
@@ -6191,7 +6183,7 @@ def init_host(self, username, host, dev=False):
         self.set_repositories([host_tool_repository])
 
         workflows = self.get_workflows('init')
-        if not self.run_workflow(workflows, **{'host_tool': {'host_clients': clients}}):
+        if not self.run_workflow(workflows, **{'host_tool': {'host_clients': clients, 'print_recommend_cmd': False}}):
             return False
 
         need_reboot_ips = self.get_namespace('host_tool').get_return('init').get_return('need_reboot_ips')
@@ -6202,6 +6194,19 @@ def init_host(self, username, host, dev=False):
                 self._call_stdio('print', FormatText.warning('You must reboot the following servers to ensure the ulimit parameters take effect: （{servers}）.'.format(servers=','.join(list(need_reboot_ips)))))
         return True
 
+    def host_user_init(self):
+        host_tool_repository = self.repository_manager.get_repository_allow_shadow('host_tool', '1.0')
+        self.set_repositories([host_tool_repository])
+
+        workflows = self.get_workflows('init_user')
+        if not self.run_workflow(workflows):
+            return False
+
+        need_reboot_ips = self.get_namespace('host_tool').get_return('init').get_return('need_reboot_ips')
+        if need_reboot_ips:
+            self._call_stdio('print', FormatText.warning('You must reboot the following servers to ensure the ulimit parameters take effect: （{servers}）.'.format(servers=','.join(list(need_reboot_ips)))))
+        return True
+
     def interactive_deploy(self, name):
 
         def signal_handler(sig, frame):

plugins/general/0.1/db_connect.py

@@ -111,7 +111,14 @@ def connect():
                 if component in COMPS_OB:
                     password = global_conf.get("proxyro_password")
                 elif component in ["obproxy", "obproxy-ce"]:
-                    password = global_conf.get("observer_sys_password")
+                    for comp in COMPS_OB:
+                        if comp in cluster_config.depends:
+                            ob_servers = cluster_config.get_depend_servers(comp)
+                            ob_config = cluster_config.get_depend_config(comp, ob_servers[0])
+                            password = ob_config.get("proxyro_password")
+                            break
+                        else:
+                            password = global_conf.get("observer_sys_password")
             if password:
                 connected = test_connect()
         need_password = not connected

plugins/general/0.1/sync_cluster_config.py

@@ -36,7 +36,6 @@ def sync_cluster_config(plugin_context, *args, **kwargs):
             cluster_config.update_global_conf('rs_list', depend_rs_list, save=False)
 
             config_map = {
-                'observer_sys_password': 'proxyro_password',
                 'cluster_name': 'appname',
                 'observer_root_password': 'root_password'
             }

plugins/general/0.1/telemetry_info_collect.py

@@ -188,14 +188,20 @@ def obd_revision(*args, **kwargs):
 
 def init_telemetry_data(opt_data):
     data = telemetry_base_data()
-    for component, _ in json.loads(opt_data).items():
-        for plugin_name, _ in _.items():
-            plugin_data = {}
-            plugin_data['component'] = component
-            plugin_data['name'] = plugin_name
-            plugin_data['runTime'] = _['time']
-            plugin_data['runResult'] = _['result']
-            data['plugins'].append(plugin_data)
+    for key, value in json.loads(opt_data).items():
+        if key == 'error_messages':
+            data['error_messages'] = value
+            continue
+        if key == 'component':
+            comp_data = value
+            for component, _ in comp_data.items():
+                for plugin_name, _ in _.items():
+                    plugin_data = {}
+                    plugin_data['component'] = component
+                    plugin_data['name'] = plugin_name
+                    plugin_data['runTime'] = _['time']
+                    plugin_data['runResult'] = _['result']
+                    data['plugins'].append(plugin_data)
 
     return data
 
@@ -216,6 +222,7 @@ def telemetry_base_data():
     data['instances'] = []
 
     data['plugins'] = []
+    data['error_messages'] = ''
     return data