Add info collected from TiDB's server API (#33)

* tidb: add info collected from TiDB's server API

 * See pingcap/tidb#7082 for details of the API

* tidb: minor cleanup of pdctl code

* tidb: check wheather tidb server api is supported

* tidb: fix runtime error when collecting pdctl

Author: AstroProfundis

insight.py

@@ -29,6 +29,7 @@
 from metric.importer import prometheus as import_prom
 from runtime import perf
 from tidb import pdctl
+from tidb import tidbinfo
 from utils import fileopt
 from utils import lsof
 from utils import space
@@ -49,7 +50,7 @@ class Insight():
     insight_perf = None
     insight_logfiles = None
     insight_configfiles = None
-    insight_pdctl = None
+    insight_tidb = None
     insight_trace = None
     insight_metric = None
 
@@ -257,12 +258,16 @@ def save_configs(self, args):
             proc_cmdline = self.format_proc_info("cmd")  # cmdline of process
         self.insight_configfiles.run_collecting(proc_cmdline)
 
-    def read_pdctl(self, args):
-        if args.subcmd_tidb != "pdctl":
-            logging.debug("Ignoring collecting of PD API.")
-        self.insight_pdctl = pdctl.PDCtl(
-            args, self.full_outdir, 'pdctl', host=args.host, port=args.port)
-        self.insight_pdctl.run_collecting()
+    def read_apis(self, args):
+        if args.subcmd_tidb == "pdctl":
+            # read and save `pd-ctl` info
+            self.insight_tidb = pdctl.PDCtl(args, self.full_outdir, 'pdctl')
+            self.insight_tidb.run_collecting()
+        elif args.subcmd_tidb == 'tidbinfo':
+            # read and save TiDB's server info
+            self.insight_tidb = tidbinfo.TiDBInfo(
+                args, self.full_outdir, 'tidbinfo')
+            self.insight_tidb.run_collecting()
 
     def dump_metrics(self, args):
         if args.subcmd_metric == "prom":
@@ -334,8 +339,8 @@ def dump_metrics(self, args):
         insight.save_configs(args)
 
     if args.subcmd == "tidb":
-        # read and save `pd-ctl` info
-        insight.read_pdctl(args)
+        # read and save info from TiDB related APIs
+        insight.read_apis(args)
 
     if args.subcmd == "metric":
         insight.dump_metrics(args)

metric/prometheus.py

@@ -28,7 +28,7 @@ def __init__(self, args, basedir=None, subdir=None):
     def get_label_names(self):
         result = []
         url = '%s%s' % (self.url_base, '/label/__name__/values')
-        labels = json.loads(util.read_url(url))
+        labels = json.loads(util.read_url(url)[0])
         if labels['status'] == 'success':
             result = labels['data']
         logging.debug("Found %s available metric keys..." % len(result))
@@ -41,7 +41,7 @@ def run_collecting(self):
         for metric in self.get_label_names():
             url = '%s/query_range?query=%s&start=%s&end=%s&step=%s' % (
                 self.url_base, metric, self.start_time, self.end_time, self.resolution)
-            matrix = json.loads(util.read_url(url))
+            matrix = json.loads(util.read_url(url)[0])
             if not matrix['status'] == 'success':
                 logging.info("Error querying for key '%s'." % metric)
                 logging.debug("Output is:\n%s" % matrix)

tidb/pdctl.py

@@ -40,13 +40,13 @@ class PDCtl(MeasurementBase):
     pd_health_uri = "/health"
     pd_diagnose_uri = "/diagnose"
 
-    def __init__(self, args, basedir=None, subdir=None, host=None, port=None, api_ver=None):
+    def __init__(self, args, basedir=None, subdir=None, api_ver=None):
         # init self.options and prepare self.outdir
         super(PDCtl, self).__init__(args, basedir, subdir)
-        if host:
-            self.host = host
-        if port:
-            self.port = port
+        if args.host:
+            self.host = args.host
+        if args.port:
+            self.port = args.port
         if api_ver:
             self.api_ver = api_ver
         self.base_url = "http://%s:%s%s%s" % (
@@ -55,20 +55,20 @@ def __init__(self, args, basedir=None, subdir=None, host=None, port=None, api_ve
     def read_health(self):
         url = "http://%s:%s/pd%s" % (self.host,
                                      self.port, self.pd_health_uri)
-        return util.read_url(url)
+        return util.read_url(url)[0]
 
     def read_diagnose(self):
         url = "http://%s:%s/pd%s" % (self.host,
                                      self.port, self.pd_diagnose_uri)
-        return util.read_url(url)
+        return util.read_url(url)[0]
 
     def read_runtime_info(self):
         def build_url(uri):
             return "%s/%s" % (self.base_url, uri)
 
         runtime_info = {}
         for key, uri in self.api_map.items():
-            runtime_info[key] = util.read_url(build_url(uri))
+            runtime_info[key] = util.read_url(build_url(uri))[0]
         return runtime_info
 
     def run_collecting(self):

tidb/tidbinfo.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# Collect infomation with TiDB API
+
+import logging
+import os
+
+from utils import util
+from utils import fileopt
+from utils.measurement import MeasurementBase
+
+
+class TiDBInfo(MeasurementBase):
+    # default to localhost
+    host = "localhost"
+    port = 10080
+
+    # The API's URI
+    uri = "/info/all"
+
+    def __init__(self, args, basedir=None, subdir=None):
+        # init self.options and prepare self.outdir
+        super(TiDBInfo, self).__init__(args, basedir, subdir)
+        if args.host:
+            self.host = args.host
+        if args.port:
+            self.port = args.port
+        self.url = "http://%s:%s%s" % (
+            self.host, self.port, self.uri)
+
+    def read_api(self):
+        result, code = util.read_url(self.url)
+        if code == 404:
+            logging.info(
+                "TiDB server API is not supported by this running instance.")
+            return None
+        return result
+
+    def run_collecting(self):
+        info = self.read_api()
+        if info:
+            fileopt.write_file(os.path.join(
+                self.outdir, "%s_%s-tidb-info.json" % (self.host, self.port)), info)

utils/util.py

@@ -175,6 +175,12 @@ def parse_insight_opts():
                               help="The host of the PD server. `localhost` by default.")
     parser_pdctl.add_argument("--port", type=int, action="store", default=None,
                               help="The port of PD API service, `2379` by default.")
+    parser_tidbinfo = subparsers_tidb.add_parser(
+        "tidbinfo", help="Collect data from TiDB's server API.")
+    parser_tidbinfo.add_argument("--host", action="store", default=None,
+                                 help="The host of the TiDB server, `localhost` by default.")
+    parser_tidbinfo.add_argument("--port", type=int, action="store", default=None,
+                                 help="The port of TiDB server API port, `10080` by default.")
 ####
 
 # Sub-command: metric
@@ -228,18 +234,18 @@ def get_init_type():
 
 def read_url(url, data=None):
     if not url or url == "":
-        return None
+        return None, None
 
     try:
         logging.debug("Requesting URL: %s" % url)
         response = urlreq.urlopen(url, data)
-        return response.read()
+        return response.read(), response.getcode()
     except HTTPError as e:
         logging.debug("HTTP Error: %s" % e.read())
-        return e.read()
+        return e.read(), e.getcode()
     except URLError as e:
         logging.warning("Reading URL %s error: %s" % (url, e))
-        return None
+        return None, None
 
 
 def get_hostname():