Add a debugging flag

    If --debug is used or if DATABRICKS_DEBUGGING is set in the environment the entire contents of the http requests will be printed to stdout.

Author: areese

databricks_cli/configure/config.py

@@ -83,6 +83,10 @@ def callback(ctx, param, value): #  NOQA
 
 def _get_api_client(config, command_name=""):
     verify = config.insecure is None
+
+    if config.is_debugging and config.debugging == "1":
+        ApiClient.enable_debug_logging()
+
     if config.is_valid_with_token:
         return ApiClient(host=config.host, token=config.token, verify=verify,
                          command_name=command_name)

databricks_cli/configure/provider.py

@@ -37,6 +37,7 @@
 TOKEN = 'token'
 INSECURE = 'insecure'
 DEFAULT_SECTION = 'DEFAULT'
+DEBUG = 'debug'
 
 # User-provided override for the DatabricksConfigProvider
 _config_provider = None
@@ -238,7 +239,11 @@ def get_config(self):
         password = os.environ.get('DATABRICKS_PASSWORD')
         token = os.environ.get('DATABRICKS_TOKEN')
         insecure = os.environ.get('DATABRICKS_INSECURE')
-        config = DatabricksConfig(host, username, password, token, insecure)
+        debugging = os.environ.get('DATABRICKS_DEBUGGING')
+        if debugging is None:
+            debugging = 0
+
+        config = DatabricksConfig(host, username, password, token, insecure, debugging=debugging)
         if config.is_valid:
             return config
         return None
@@ -256,14 +261,18 @@ def get_config(self):
         password = _get_option_if_exists(raw_config, self.profile, PASSWORD)
         token = _get_option_if_exists(raw_config, self.profile, TOKEN)
         insecure = _get_option_if_exists(raw_config, self.profile, INSECURE)
-        config = DatabricksConfig(host, username, password, token, insecure)
+        debugging = _get_option_if_exists(raw_config, self.profile, DEBUG)
+        if debugging is None:
+            debugging = 0
+
+        config = DatabricksConfig(host, username, password, token, insecure, debugging=debugging)
         if config.is_valid:
             return config
         return None
 
 
 class DatabricksConfig(object):
-    def __init__(self, host, username, password, token, insecure):  # noqa
+    def __init__(self, host, username, password, token, insecure, debugging=0):  # noqa
         self.host = host
         self.username = username
 
@@ -274,6 +283,8 @@ def __init__(self, host, username, password, token, insecure):  # noqa
 
         self.token = token
         self.insecure = insecure
+        self.debugging = debugging
+
 
     @classmethod
     def from_token(cls, host, token, insecure=None):
@@ -298,3 +309,7 @@ def is_valid_with_password(self):
     @property
     def is_valid(self):
         return self.is_valid_with_token or self.is_valid_with_password
+
+    @property
+    def is_debugging(self):
+        return self.debugging

databricks_cli/sdk/api_client.py

@@ -30,6 +30,7 @@
 
 import base64
 import json
+import logging
 import warnings
 import requests
 import ssl
@@ -129,6 +130,27 @@ def perform_query(self, method, path, data = {}, headers = None):
             raise requests.exceptions.HTTPError(message, response=e.response)
         return resp.json()
 
+     @classmethod
+     def enable_debug_logging(cls):
+         # These two lines enable debugging at httplib level (requests->urllib3->http.client)
+         # You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA.
+         # The only thing missing will be the response.body which is not logged.
+         try:
+             import http.client as http_client
+         except ImportError:
+             # Python 2
+             import httplib as http_client
+
+         print ("HTTP debugging enabled")
+         http_client.HTTPConnection.debuglevel = 1
+
+         # You must initialize logging, otherwise you'll not see debug output.
+         logging.basicConfig()
+         logging.getLogger().setLevel(logging.DEBUG)
+         requests_log = logging.getLogger("requests.packages.urllib3")
+         requests_log.setLevel(logging.DEBUG)
+         requests_log.propagate = True
+
 
 def _translate_boolean_to_query_param(value):
     assert not isinstance(value, list), 'GET parameters cannot pass list of objects'