Add a debugging flag

    If --debug is used or if DATABRICKS_DEBUGGING is set in the environment the entire contents of the http requests will be printed to stdout.

Author: areese

databricks_cli/configure/cli.py

@@ -32,18 +32,27 @@
 
 PROMPT_HOST = 'Databricks Host (should begin with https://)'
 PROMPT_USERNAME = 'Username'
-PROMPT_PASSWORD = 'Password' #  NOQA
+PROMPT_PASSWORD = 'Password [enter stdin to read from stdin every time]' #  NOQA
 PROMPT_TOKEN = 'Token' #  NOQA
 
 
 def _configure_cli_token(profile, insecure):
     config = ProfileConfigProvider(profile).get_config() or DatabricksConfig.empty()
     host = click.prompt(PROMPT_HOST, default=config.host, type=_DbfsHost())
-    token = click.prompt(PROMPT_TOKEN, default=config.token)
+    token = click.prompt(PROMPT_TOKEN, default=config.token, hide_input=True)
     new_config = DatabricksConfig.from_token(host, token, insecure)
     update_and_persist_config(profile, new_config)
 
 
+def _configure_cli_stdin(profile, insecure):
+    config = ProfileConfigProvider(profile).get_config() or DatabricksConfig.empty()
+    host = click.prompt(PROMPT_HOST, default=config.host, type=_DbfsHost())
+    username = click.prompt(PROMPT_USERNAME, default=config.username)
+    password = 'stdin'
+    new_config = DatabricksConfig.from_password(host, username, password, insecure)
+    update_and_persist_config(profile, new_config)
+
+
 def _configure_cli_password(profile, insecure):
     config = ProfileConfigProvider(profile).get_config() or DatabricksConfig.empty()
     if config.password:
@@ -64,16 +73,19 @@ def _configure_cli_password(profile, insecure):
                short_help='Configures host and authentication info for the CLI.')
 @click.option('--token', show_default=True, is_flag=True, default=False)
 @click.option('--insecure', show_default=True, is_flag=True, default=None)
+@click.option('--stdin', show_default=True, is_flag=True, default=False)
 @debug_option
 @profile_option
-def configure_cli(token, insecure):
+def configure_cli(token, insecure, stdin):
     """
     Configures host and authentication info for the CLI.
     """
     profile = get_profile_from_context()
     insecure_str = str(insecure) if insecure is not None else None
     if token:
         _configure_cli_token(profile, insecure_str)
+    elif stdin:
+        _configure_cli_stdin()
     else:
         _configure_cli_password(profile, insecure_str)
 

databricks_cli/configure/config.py

@@ -83,6 +83,10 @@ def callback(ctx, param, value): #  NOQA
 
 def _get_api_client(config, command_name=""):
     verify = config.insecure is None
+
+    if config.is_debugging and config.debugging == "1":
+        ApiClient.enable_debug_logging()
+
     if config.is_valid_with_token:
         return ApiClient(host=config.host, token=config.token, verify=verify,
                          command_name=command_name)

databricks_cli/configure/provider.py

@@ -20,7 +20,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import getpass
 from abc import abstractmethod, ABCMeta
 from configparser import ConfigParser
 import os
@@ -33,10 +33,11 @@
 CONFIG_FILE_ENV_VAR = "DATABRICKS_CONFIG_FILE"
 HOST = 'host'
 USERNAME = 'username'
-PASSWORD = 'password' # NOQA
+PASSWORD = 'password'  # NOQA
 TOKEN = 'token'
 INSECURE = 'insecure'
 DEFAULT_SECTION = 'DEFAULT'
+DEBUG = 'debug'
 
 # User-provided override for the DatabricksConfigProvider
 _config_provider = None
@@ -238,7 +239,11 @@ def get_config(self):
         password = os.environ.get('DATABRICKS_PASSWORD')
         token = os.environ.get('DATABRICKS_TOKEN')
         insecure = os.environ.get('DATABRICKS_INSECURE')
-        config = DatabricksConfig(host, username, password, token, insecure)
+        debugging = os.environ.get('DATABRICKS_DEBUGGING')
+        if debugging is None:
+            debugging = 0
+
+        config = DatabricksConfig(host, username, password, token, insecure, debugging=debugging)
         if config.is_valid:
             return config
         return None
@@ -256,19 +261,30 @@ def get_config(self):
         password = _get_option_if_exists(raw_config, self.profile, PASSWORD)
         token = _get_option_if_exists(raw_config, self.profile, TOKEN)
         insecure = _get_option_if_exists(raw_config, self.profile, INSECURE)
-        config = DatabricksConfig(host, username, password, token, insecure)
+        debugging = _get_option_if_exists(raw_config, self.profile, DEBUG)
+        if debugging is None:
+            debugging = 0
+
+        config = DatabricksConfig(host, username, password, token, insecure, debugging=debugging)
         if config.is_valid:
             return config
         return None
 
 
 class DatabricksConfig(object):
-    def __init__(self, host, username, password, token, insecure): # noqa
+    def __init__(self, host, username, password, token, insecure, debugging=0):  # noqa
         self.host = host
         self.username = username
-        self.password = password
+
+        if self.password == 'stdin':
+            self.password = getpass.getpass("Password to connect to databricks at [" + self.host + "]: ")
+        else:
+            self.password = password
+
         self.token = token
         self.insecure = insecure
+        self.debugging = debugging
+
 
     @classmethod
     def from_token(cls, host, token, insecure=None):
@@ -293,3 +309,7 @@ def is_valid_with_password(self):
     @property
     def is_valid(self):
         return self.is_valid_with_token or self.is_valid_with_password
+
+    @property
+    def is_debugging(self):
+        return self.debugging

databricks_cli/jobs/api.py

@@ -31,7 +31,10 @@ def create_job(self, json, headers=None):
         return self.client.client.perform_query('POST', '/jobs/create', data=json, headers=headers)
 
     def list_jobs(self, headers=None):
-        return self.client.list_jobs(headers=headers)
+        resp = self.client.list_jobs(headers=headers)
+        if 'jobs' not in resp:
+            resp['jobs'] = []
+        return resp
 
     def delete_job(self, job_id, headers=None):
         return self.client.delete_job(job_id, headers=headers)

databricks_cli/sdk/api_client.py

@@ -30,6 +30,7 @@
 
 import base64
 import json
+import logging
 import warnings
 import requests
 import ssl
@@ -129,6 +130,27 @@ def perform_query(self, method, path, data = {}, headers = None):
             raise requests.exceptions.HTTPError(message, response=e.response)
         return resp.json()
 
+     @classmethod
+     def enable_debug_logging(cls):
+         # These two lines enable debugging at httplib level (requests->urllib3->http.client)
+         # You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA.
+         # The only thing missing will be the response.body which is not logged.
+         try:
+             import http.client as http_client
+         except ImportError:
+             # Python 2
+             import httplib as http_client
+
+         print ("HTTP debugging enabled")
+         http_client.HTTPConnection.debuglevel = 1
+
+         # You must initialize logging, otherwise you'll not see debug output.
+         logging.basicConfig()
+         logging.getLogger().setLevel(logging.DEBUG)
+         requests_log = logging.getLogger("requests.packages.urllib3")
+         requests_log.setLevel(logging.DEBUG)
+         requests_log.propagate = True
+
 
 def _translate_boolean_to_query_param(value):
     assert not isinstance(value, list), 'GET parameters cannot pass list of objects'