Added new 'AUTO' format to workspace CLI to allow importing extra files types

databricks_cli/workspace/api.py

@@ -132,7 +132,7 @@ def export_workspace(self, source_path, target_path, fmt, is_overwrite, headers=
     def delete(self, workspace_path, is_recursive, headers=None):
         self.client.delete(workspace_path, is_recursive, headers=headers)
 
-    def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files,
+    def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files, are_workspace_files,
                              headers=None):
         # pylint: disable=too-many-locals
         filenames = os.listdir(source_path)
@@ -149,16 +149,12 @@ def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidd
                                           headers=headers)
             elif os.path.isfile(cur_src):
                 ext = WorkspaceLanguage.get_extension(cur_src)
-                if ext != '':
+                if not are_workspace_files:
                     cur_dst = cur_dst[:-len(ext)]
-                    (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src)
-                    self.import_workspace(cur_src, cur_dst, language, file_format, overwrite,
-                                          headers=headers)
-                    click.echo('{} -> {}'.format(cur_src, cur_dst))
-                else:
-                    extensions = ', '.join(WorkspaceLanguage.EXTENSIONS)
-                    click.echo(('{} does not have a valid extension of {}. Skip this file and ' +
-                                'continue.').format(cur_src, extensions))
+                (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src)
+                self.import_workspace(cur_src, cur_dst, language, file_format, overwrite,
+                                      headers=headers)
+                click.echo('{} -> {}'.format(cur_src, cur_dst))
 
     def export_workspace_dir(self, source_path, target_path, overwrite, headers=None):
         if os.path.isfile(target_path):

databricks_cli/workspace/cli.py

@@ -83,19 +83,19 @@ def mkdirs_cli(api_client, workspace_path):
                short_help='Imports a file from local to the Databricks workspace.')
 @click.argument('source_path')
 @click.argument('target_path')
-@click.option('--language', '-l', required=True, type=LanguageClickType(),
+@click.option('--language', '-l', required=False, type=LanguageClickType(),
               help=', '.join(WorkspaceLanguage.ALL))
 @click.option('--format', '-f', default=WorkspaceFormat.SOURCE, type=FormatClickType())
 @click.option('--overwrite', '-o', is_flag=True, default=False)
 @debug_option
 @profile_option
 @eat_exceptions
 @provide_api_client
-def import_workspace_cli(api_client, source_path, target_path, language, format, overwrite): # NOQA
+def import_workspace_cli(api_client, source_path, target_path, format, overwrite, language=None): # NOQA
     """
     Imports a file from local to the Databricks workspace.
 
-    The format is by default SOURCE. Possible formats are SOURCE, HTML, JUPYTER, and DBC. Each
+    The format is by default SOURCE. Possible formats are SOURCE, HTML, JUPYTER, DBC, and AUTO. Each
     format is documented at
     https://docs.databricks.com/api/latest/workspace.html#notebookexportformat.
     """
@@ -176,19 +176,21 @@ def export_dir_cli(api_client, source_path, target_path, overwrite):
 @click.argument('target_path')
 @click.option('--overwrite', '-o', is_flag=True, default=False)
 @click.option('--exclude-hidden-files', '-e', is_flag=True, default=False)
+@click.option('--are_workspace_files', '-wf', is_flag=True, default=False)
 @debug_option
 @profile_option
 @eat_exceptions
 @provide_api_client
-def import_dir_cli(api_client, source_path, target_path, overwrite, exclude_hidden_files):
+def import_dir_cli(api_client, source_path, target_path, overwrite, exclude_hidden_files, are_workspace_files):
     """
     Recursively imports a directory from local to the Databricks workspace.
 
-    Only directories and files with the extensions .scala, .py, .sql, .r, .R, .ipynb are imported.
+    Only directories and files with the extensions .scala, .py, .sql, .r, .R, .ipynb are imported by default and are converted to notebooks compatible with the original extension.
     When imported, these extensions will be stripped off the name of the notebook.
+    If are_workspace_files is set to True, the files will be imported as as with their original extensions.
     """
     WorkspaceApi(api_client).import_workspace_dir(source_path, target_path, overwrite,
-                                                  exclude_hidden_files)
+                                                  exclude_hidden_files, are_workspace_files)
 
 
 @click.group(context_settings=CONTEXT_SETTINGS,

databricks_cli/workspace/types.py

@@ -50,6 +50,8 @@ def to_language_and_format(cls, path):
             language_and_format = (None, WorkspaceFormat.HTML)
         elif ext == '.dbc':
             language_and_format = (None, WorkspaceFormat.DBC)
+        else:
+            language_and_format = (None, WorkspaceFormat.AUTO)
         return language_and_format
 
     @classmethod
@@ -87,7 +89,8 @@ class WorkspaceFormat(object):
     HTML = 'HTML'
     JUPYTER = 'JUPYTER'
     DBC = 'DBC'
-    ALL = [SOURCE, HTML, JUPYTER, DBC]
+    AUTO = 'AUTO'
+    ALL = [SOURCE, HTML, JUPYTER, DBC, AUTO]
 
 
 class FormatClickType(ParamType):