v1.6.1 (#146)

Author: frf12

_cmd.py

@@ -854,6 +854,7 @@ def __init__(self):
         self.parser.add_option('--max-session-num', type='int', help="Max session unit number. Not supported after version 4.0")
         self.parser.add_option('--unit-num', type='int', help="Pool unit number.")
         self.parser.add_option('-z', '--zone-list', type='string', help="Tenant zone list.")
+        self.parser.add_option('--mode', type='string', help='Tenant compatibility mode. {mysql,oracle} [mysql]', default='mysql')
         self.parser.add_option('--charset', type='string', help="Tenant charset.")
         self.parser.add_option('--collate', type='string', help="Tenant COLLATE.")
         self.parser.add_option('--replica-num', type='int', help="Tenant replica number.")
@@ -925,6 +926,8 @@ class MySQLTestCommand(TestMirrorCommand):
 
     def __init__(self):
         super(MySQLTestCommand, self).__init__('mysqltest', 'Run a mysqltest for a deployment.')
+        self.parser.add_option('--mode', type='string', help='Test mode. Available values are mysql, oracle, and both.', default='both')
+        # self.parser.add_option('--case-mode', type='string', help='case run mode [mysql,oracle]', default='mysql')
         self.parser.add_option('--component', type='string', help='Components for mysqltest.')
         self.parser.add_option('--test-server', type='string', help='The server for mysqltest. By default, the first root server in the component is the mysqltest server.')
         self.parser.add_option('--user', type='string', help='Username for a test. [admin]', default='admin')
@@ -1042,7 +1045,7 @@ def __init__(self):
         self.parser.add_option('--remote-tbl-dir', type='string', help='Directory for the tbl on target observers. Make sure that you have read and write access to the directory when you start observer.')
         self.parser.add_option('--disable-transfer', '--dt', action='store_true', help='Disable the transfer. When enabled, OBD will use the tbl files under remote-tbl-dir instead of transferring local tbl files to remote remote-tbl-dir.')
         self.parser.add_option('--dss-config', type='string', help='Directory for dists.dss. [/usr/tpc-h-tools/tpc-h-tools]', default='/usr/tpc-h-tools/tpc-h-tools/')
-        self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1}. [1]', default=1)
+        self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1/2}. [1] 0 - No optimization. 1 - Optimize some of the parameters which do not need to restart servers. 2 - Optimize all the parameters and maybe RESTART SERVERS for better performance.', default=1)
         self.parser.add_option('--test-only', action='store_true', help='Only testing SQLs are executed. No initialization is executed.')
         self.parser.add_option('-S', '--skip-cluster-status-check', action='store_true', help='Skip cluster status check', default=False)
 

_deploy.py

@@ -625,6 +625,8 @@ def _check_param(self, config):
         return error
 
     def set_global_conf(self, conf):
+        if not isinstance(conf, dict):
+            raise Exception('%s global config is not a dictionary. Please check the syntax of your configuration file.\n See https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md' % self.name)
         self._original_global_conf = deepcopy(conf)
         self._global_conf = None
         self._clear_cache_server()

_errno.py

@@ -62,6 +62,8 @@ class InitDirFailedErrorMessage(object):
 EC_ULIMIT_CHECK = OBDErrorCode(1007, '({server}) {key} must not be less than {need} (Current value: {now})')
 
 EC_OBSERVER_NOT_ENOUGH_MEMORY = OBDErrorCode(2000, '({ip}) not enough memory. (Free: {free}, Need: {need})')
+EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE = OBDErrorCode(2000, '({ip}) not enough memory. (Available: {available}, Need: {need})')
+EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED = OBDErrorCode(2000, '({ip}) not enough memory. (Free: {free}, Buff/Cache: {cached}, Need: {need})')
 EC_OBSERVER_CAN_NOT_MIGRATE_IN = OBDErrorCode(2001, 'server can not migrate in')
 EC_OBSERVER_FAIL_TO_START = OBDErrorCode(2002, 'Failed to start {server} observer')
 EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG = OBDErrorCode(2003, '({ip}) {path} not enough disk space for clog. Use redo_dir to set other disk for clog, or reduce the value of datafile_size')

_repository.py

@@ -172,19 +172,23 @@ def extract(worker):
 
 class ParallerExtractor(object):
 
-    MAX_PARALLER = cpu_count() if cpu_count() else 8
+    MAX_PARALLER = cpu_count() * 2 if cpu_count() else 8
+    MAX_SIZE = 100
+    MIN_SIZE = 20
 
     def __init__(self, pkg, files, stdio=None):
         self.pkg = pkg
         self.files = files
         self.stdio = stdio
 
     def extract(self):
+        if not self.files:
+            return
         workers = []
         file_num = len(self.files)
-        paraler = int(min(self.MAX_PARALLER, file_num))
-        size = min(100, int(file_num / paraler))
-        size = int(max(10, size))
+        paraller = int(min(self.MAX_PARALLER, file_num))
+        size = min(self.MAX_SIZE, int(file_num / paraller)) # 
+        size = int(max(self.MIN_SIZE, size))
         index = 0
         while index < file_num:
             p_index = index + size
@@ -195,7 +199,7 @@ def extract(self):
             ))
             index = p_index
 
-        pool = Pool(processes=paraler)
+        pool = Pool(processes=paraller)
         try:
             results = pool.map(ParallerExtractWorker.extract, workers)
             for r in results:

core.py

@@ -890,7 +890,7 @@ def genconfig(self, name, opt=Values()):
         self._call_stdio('verbose', 'Get deploy configuration')
         deploy_config = deploy.deploy_config
         if not deploy_config:
-            self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.')
+            self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md')
             return False
 
         # Check the best suitable mirror for the components and installation plugins. Install locally
@@ -1061,7 +1061,7 @@ def change_deploy_config_style(self, name, options=Values()):
             return False
         deploy_config = deploy.deploy_config
         if not deploy_config:
-            self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.')
+            self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md')
             return False
 
         style = getattr(options, 'style', '')
@@ -1211,11 +1211,11 @@ def deploy_cluster(self, name, opt=Values()):
         self._call_stdio('verbose', 'Get deploy configuration')
         deploy_config = deploy.deploy_config
         if not deploy_config:
-            self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.')
+            self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md')
             return False
 
         if not deploy_config.components:
-            self._call_stdio('error', 'Components not detected.\nPlease check the syntax of your configuration file.')
+            self._call_stdio('error', 'Components not detected.\nPlease check the syntax of your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md')
             return False
 
         for component_name in deploy_config.components:

docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md

@@ -0,0 +1,41 @@
+# Quick deployment command
+
+## obd demo
+
+You can use this command to deploy and start the specified component on the local server without loading the configuration file. The fixed name of the cluster deployed is `demo`. After the deployment, you can run the `obd cluster list` command to view the cluster in the cluster list. You can also run other cluster commands, such as `obd cluster display demo`, to manage the cluster.
+
+```bash
+obd demo [-c/--components]
+```
+
+The following table describes the parameters.
+
+| Parameter | Required | Data type | Default value | Description |
+|------------------|---------|------------|----------|--------------------------------------------------------------------|
+| -c/--components | No | String | oceanbase-ce,obproxy-ce,obagent,prometheus,grafana | The list of components that are separated with commas (`,`). You can use this parameter to specify the components to be deployed. |
+
+By default, this command deploys the minimum specifications in the home directory of the current user, and the latest versions are deployed by default. You can use this command to deploy OceanBase Community Edition, OBProxy Community Edition, OBAgent, Grafana, and Prometheus.
+
+You can select the version and specify the configurations of a component to be deployed.
+
+```bash
+# Deploy components of the specified version.
+obd demo -c oceanbase-ce,obproxy-ce --oceanbase-ce.version=3.1.3
+# Specify the components to be deployed and the package hash of OceanBase Community Edition.
+obd demo -c oceanbase-ce,obproxy-ce --oceanbase-ce.package_hash=f38723204d49057d3e062ffad778edc1552a7c114622bf2a86fea769fbd202ea
+# Specify the installation path for all components to be deployed.
+## Deploy OceanBase Community Edition and OBProxy Community Edition in the /data/demo directory and create corresponding working directories for them.
+obd demo -c oceanbase-ce,obproxy-ce --home_path=/data/demo
+# Specify the installation path for all components to be deployed.
+obd demo --home_path=/path
+# Specify the installation path for a specific component to be deployed.
+## Deploy OceanBase Community Edition in the home directory and create a working directory for it, and deploy OBProxy Community Edition in the /data/playground/obproxy-ce directory.
+obd demo -c oceanbase-ce,obproxy-ce --obproxy-ce.home_path=/data/demo/
+# Specify the configurations of a component to be deployed.
+## Specify the mysql_port parameter of OceanBase Community Edition.
+obd demo --oceanbase-ce.mysql_port=3881
+```
+
+> **Notice**
+>
+> This command supports only level-1 configurations under global that are specified by using options.

optimize/obproxy/4.0.0/sysbench.yaml

@@ -0,0 +1,24 @@
+test:
+  system_config:
+  - name: proxy_mem_limited
+    value: 2G
+  - name: slow_proxy_process_time_threshold
+    value: 500ms
+  - name: syslog_level
+    value: error
+    query_key: level
+  - name: enable_prometheus
+    value: false
+  - name: enable_compression_protocol
+    value: false
+    need_restart: true
+    value_type: BOOL
+  - name: enable_ob_protocol_v2
+    value: false
+    need_restart: true
+    value_type: BOOL
+  - name: work_thread_num
+    value: 128
+    need_restart: true
+  - name: enable_async_log
+    value: true

optimize/obproxy/4.0.0/tpcc.yaml

@@ -0,0 +1,24 @@
+build:
+  system_config:
+  - name: proxy_mem_limited
+    value: 2G
+  - name: slow_proxy_process_time_threshold
+    value: 500ms
+  - name: syslog_level
+    value: error
+    query_key: level
+  - name: enable_prometheus
+    value: false
+  - name: enable_compression_protocol
+    value: false
+    need_restart: true
+    value_type: BOOL
+  - name: enable_ob_protocol_v2
+    value: false
+    need_restart: true
+    value_type: BOOL
+  - name: work_thread_num
+    value: 128
+    need_restart: true
+  - name: enable_async_log
+    value: true

plugins/grafana/7.5.17/oceanbase-metrics_rev1.json

@@ -218,11 +218,11 @@
           "exemplar": true,
           "expr": "(sum(rate(ob_sysstat{stat_id=\"40003\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40005\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40009\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40009\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40001\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group))\n/\n(sum(rate(ob_sysstat{stat_id=\"40002\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40004\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40006\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40008\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40000\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group))",
           "interval": "",
-          "legendFormat": "qps rt {{$group}}",
+          "legendFormat": "sql latency {{$group}}",
           "refId": "A"
         }
       ],
-      "title": "QPS rt",
+      "title": "Sql Latency",
       "type": "timeseries"
     },
     {
@@ -2129,19 +2129,19 @@
           "exemplar": true,
           "expr": "(sum(rate(ob_sysstat{stat_id=\"10005\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"10006\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)) / sum(rate(ob_sysstat{stat_id=\"10000\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)",
           "interval": "",
-          "legendFormat": "roc in rt {{$group}}",
+          "legendFormat": "rpc in Latency {{$group}}",
           "refId": "A"
         },
         {
           "exemplar": true,
           "expr": "(sum(rate(ob_sysstat{stat_id=\"10005\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"10006\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)) / sum(rate(ob_sysstat{stat_id=\"10002\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)",
           "hide": false,
           "interval": "",
-          "legendFormat": "rpc out rt {{$group}}",
+          "legendFormat": "rpc out latency {{$group}}",
           "refId": "B"
         }
       ],
-      "title": "Rpc rt",
+      "title": "Rpc Latency",
       "type": "timeseries"
     },
     {

plugins/mysqltest/3.1.0/init.py

@@ -36,7 +36,7 @@ def parse_size(size):
         _bytes = int(size)
     else:
         units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40}
-        match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper())
+        match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper())
         _bytes = int(match.group(1)) * units[match.group(2)]
     return _bytes
 
@@ -127,6 +127,7 @@ def exec_sql(cmd):
         exec_init = 'init.sql'
         exec_mini_init = 'init_mini.sql'
         exec_init_user = 'init_user.sql|root@mysql|test'
+        exec_init_user_for_oracle = 'init_user_oracle.sql|SYS@oracle|SYS'
         client = plugin_context.clients[server]
         memory_limit = get_memory_limit(cursor, client)
         is_mini = memory_limit and parse_size(memory_limit) < (16<<30)