diff --git a/databricks_cli/workspace/api.py b/databricks_cli/workspace/api.py index e38db7bf..28396d7b 100644 --- a/databricks_cli/workspace/api.py +++ b/databricks_cli/workspace/api.py @@ -132,7 +132,7 @@ def export_workspace(self, source_path, target_path, fmt, is_overwrite, headers= def delete(self, workspace_path, is_recursive, headers=None): self.client.delete(workspace_path, is_recursive, headers=headers) - def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files, + def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidden_files, are_workspace_files, headers=None): # pylint: disable=too-many-locals filenames = os.listdir(source_path) @@ -149,16 +149,12 @@ def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidd headers=headers) elif os.path.isfile(cur_src): ext = WorkspaceLanguage.get_extension(cur_src) - if ext != '': + if not are_workspace_files: cur_dst = cur_dst[:-len(ext)] - (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src) - self.import_workspace(cur_src, cur_dst, language, file_format, overwrite, - headers=headers) - click.echo('{} -> {}'.format(cur_src, cur_dst)) - else: - extensions = ', '.join(WorkspaceLanguage.EXTENSIONS) - click.echo(('{} does not have a valid extension of {}. Skip this file and ' + - 'continue.').format(cur_src, extensions)) + (language, file_format) = WorkspaceLanguage.to_language_and_format(cur_src) + self.import_workspace(cur_src, cur_dst, language, file_format, overwrite, + headers=headers) + click.echo('{} -> {}'.format(cur_src, cur_dst)) def export_workspace_dir(self, source_path, target_path, overwrite, headers=None): if os.path.isfile(target_path): diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index be03c0a8..b51f914a 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -83,7 +83,7 @@ def mkdirs_cli(api_client, workspace_path): short_help='Imports a file from local to the Databricks workspace.') @click.argument('source_path') @click.argument('target_path') -@click.option('--language', '-l', required=True, type=LanguageClickType(), +@click.option('--language', '-l', required=False, type=LanguageClickType(), help=', '.join(WorkspaceLanguage.ALL)) @click.option('--format', '-f', default=WorkspaceFormat.SOURCE, type=FormatClickType()) @click.option('--overwrite', '-o', is_flag=True, default=False) @@ -91,11 +91,11 @@ def mkdirs_cli(api_client, workspace_path): @profile_option @eat_exceptions @provide_api_client -def import_workspace_cli(api_client, source_path, target_path, language, format, overwrite): # NOQA +def import_workspace_cli(api_client, source_path, target_path, format, overwrite, language=None): # NOQA """ Imports a file from local to the Databricks workspace. - The format is by default SOURCE. Possible formats are SOURCE, HTML, JUPYTER, and DBC. Each + The format is by default SOURCE. Possible formats are SOURCE, HTML, JUPYTER, DBC, and AUTO. Each format is documented at https://docs.databricks.com/api/latest/workspace.html#notebookexportformat. """ @@ -176,19 +176,21 @@ def export_dir_cli(api_client, source_path, target_path, overwrite): @click.argument('target_path') @click.option('--overwrite', '-o', is_flag=True, default=False) @click.option('--exclude-hidden-files', '-e', is_flag=True, default=False) +@click.option('--are_workspace_files', '-wf', is_flag=True, default=False) @debug_option @profile_option @eat_exceptions @provide_api_client -def import_dir_cli(api_client, source_path, target_path, overwrite, exclude_hidden_files): +def import_dir_cli(api_client, source_path, target_path, overwrite, exclude_hidden_files, are_workspace_files): """ Recursively imports a directory from local to the Databricks workspace. - Only directories and files with the extensions .scala, .py, .sql, .r, .R, .ipynb are imported. + Only directories and files with the extensions .scala, .py, .sql, .r, .R, .ipynb are imported by default and are converted to notebooks compatible with the original extension. When imported, these extensions will be stripped off the name of the notebook. + If are_workspace_files is set to True, the files will be imported as as with their original extensions. """ WorkspaceApi(api_client).import_workspace_dir(source_path, target_path, overwrite, - exclude_hidden_files) + exclude_hidden_files, are_workspace_files) @click.group(context_settings=CONTEXT_SETTINGS, diff --git a/databricks_cli/workspace/types.py b/databricks_cli/workspace/types.py index 8e685eac..8670cfa8 100644 --- a/databricks_cli/workspace/types.py +++ b/databricks_cli/workspace/types.py @@ -50,6 +50,8 @@ def to_language_and_format(cls, path): language_and_format = (None, WorkspaceFormat.HTML) elif ext == '.dbc': language_and_format = (None, WorkspaceFormat.DBC) + else: + language_and_format = (None, WorkspaceFormat.AUTO) return language_and_format @classmethod @@ -87,7 +89,8 @@ class WorkspaceFormat(object): HTML = 'HTML' JUPYTER = 'JUPYTER' DBC = 'DBC' - ALL = [SOURCE, HTML, JUPYTER, DBC] + AUTO = 'AUTO' + ALL = [SOURCE, HTML, JUPYTER, DBC, AUTO] class FormatClickType(ParamType):