Add dbfs cat cli option (#226)

* Add support for dbfs to dbfs copies

This commit enables these copies by using a temp file/dir on the
local filesystem as a stepping stone.

* Add cat option to dbfs cli

This allows users to view the contents of a file in a single
command, instead of forcing them to download it first, and then use
their default shell's cat functionality

* add test and fix up

* fix lint

Author: danielvdende

databricks_cli/dbfs/api.py

@@ -24,6 +24,9 @@
 from base64 import b64encode, b64decode
 
 import os
+import shutil
+import tempfile
+
 import click
 
 from requests.exceptions import HTTPError
@@ -221,8 +224,40 @@ def cp(self, recursive, overwrite, src, dst, headers=None):
                            'To use this utility, one of the src or dst must be prefixed '
                            'with dbfs:/')
         elif DbfsPath.is_valid(src) and DbfsPath.is_valid(dst):
-            error_and_quit('Both paths provided are from the DBFS filesystem. '
-                           'To copy between the DBFS filesystem, you currently must copy the '
-                           'file from DBFS to your local filesystem and then back.')
+            with TempDir() as temp_dir:
+                # Always copy to <temp_dir>/temp since this will work no matter if it's a
+                # recursive or a non-recursive copy.
+                temp_path = temp_dir.path('temp')
+                self.cp(recursive, True, src, temp_path)
+                self.cp(recursive, overwrite, temp_path, dst)
         else:
             assert False, 'not reached'
+
+    def cat(self, src):
+        with TempDir() as temp_dir:
+            temp_path = temp_dir.path('temp')
+            self.cp(False, True, src, temp_path)
+            with open(temp_path) as f:
+                click.echo(f.read(), nl=False)
+
+
+class TempDir(object):
+    def __init__(self, remove_on_exit=True):
+        self._dir = None
+        self._path = None
+        self._remove = remove_on_exit
+
+    def __enter__(self):
+        self._path = os.path.abspath(tempfile.mkdtemp())
+        assert os.path.exists(self._path)
+        return self
+
+    def __exit__(self, tp, val, traceback):
+        if self._remove and os.path.exists(self._path):
+            shutil.rmtree(self._path)
+
+        assert not self._remove or not os.path.exists(self._path)
+        assert os.path.exists(os.getcwd())
+
+    def path(self, *path):
+        return os.path.join(self._path, *path)

databricks_cli/dbfs/cli.py

@@ -102,8 +102,7 @@ def cp_cli(api_client, recursive, overwrite, src, dst):
     """
     Copy files to and from DBFS.
 
-    Note that this function will fail if the src and dst are both on the local filesystem
-    or if they are both DBFS paths.
+    Note that this function will fail if the src and dst are both on the local filesystem.
 
     For non-recursive copies, if the dst is a directory, the file will be placed inside the
     directory. For example ``dbfs cp dbfs:/apple.txt .`` will create a file at `./apple.txt`.
@@ -146,9 +145,23 @@ def dbfs_group():
     pass
 
 
+@click.command(context_settings=CONTEXT_SETTINGS)
+@click.argument('src')
+@debug_option
+@profile_option
+@eat_exceptions
+@provide_api_client
+def cat_cli(api_client, src):
+    """
+    Show the contents of a file. Does not work for directories.
+    """
+    DbfsApi(api_client).cat(src)
+
+
 dbfs_group.add_command(configure_cli, name='configure')
 dbfs_group.add_command(ls_cli, name='ls')
 dbfs_group.add_command(mkdirs_cli, name='mkdirs')
 dbfs_group.add_command(rm_cli, name='rm')
 dbfs_group.add_command(cp_cli, name='cp')
 dbfs_group.add_command(mv_cli, name='mv')
+dbfs_group.add_command(cat_cli, name='cat')

tests/dbfs/test_api.py

@@ -151,3 +151,17 @@ def test_get_file(self, dbfs_api, tmpdir):
 
         with open(test_file_path, 'r') as f:
             assert f.read() == 'x'
+
+    def test_cat(self, dbfs_api):
+        dbfs_api.client.get_status.return_value = {
+            'path': '/test',
+            'is_dir': False,
+            'file_size': 1
+        }
+        dbfs_api.client.read.return_value = {
+            'bytes_read': 1,
+            'data': b64encode(b'a'),
+        }
+        with mock.patch('databricks_cli.dbfs.api.click') as click_mock:
+            dbfs_api.cat('dbfs:/whatever-doesnt-matter')
+            click_mock.echo.assert_called_with('a', nl=False)