fixes hdfs

Author: andrewfulton9

environment.yml

@@ -13,4 +13,9 @@ dependencies:
     - pytest
     - pylint
     - flake8
+    - pyarrow
+    - libhdfs3
+    - pip
+    - pip:
+      - hadoop-test-cluster
 

upath/core.py

@@ -207,11 +207,15 @@ def exists(self):
         """
         Whether this path exists.
         """
-        try:
-            self._accessor.stat(self)
-        except FileNotFoundError:
-            return False
-        return True
+        if not getattr(self._accessor, 'exists'):
+            try:
+                self._accessor.stat(self)
+            except (FileNotFoundError):
+                return False
+            return True
+        else:
+            return self._accessor.exists(self)
+                
 
     def is_dir(self):
         info = self._accessor.info(self)

upath/tests/conftest.py

@@ -47,3 +47,19 @@ def testingdir(clear_registry):
     yield tempdir
     shutil.rmtree(tempdir)
 
+@pytest.fixture()
+def hdfs_test_client():
+    import os
+    pyarrow = pytest.importorskip('pyarrow')
+    host = os.environ.get('ARROW_HDFS_TEST_HOST', '0.0.0.0')
+    user = os.environ.get('ARROW_HDFS_TEST_USER', 'hdfs')
+    try:
+        port = int(os.environ.get('ARROW_HDFS_TEST_PORT', 9000))
+    except ValueError:
+        raise ValueError('Env variable ARROW_HDFS_TEST_PORT was not '
+                         'an integer')
+
+    hdfs = pyarrow.hdfs.connect(host='0.0.0.0', port=9000, user=user, driver='libhdfs3')
+    hdfs.mkdir('/folder1')
+    return host, user, port
+

upath/tests/test_core.py

@@ -214,3 +214,9 @@ def test_write_text(self, mock_base, pathlib_base):
         assert pathlib_base.joinpath(fn).read_bytes() == s
 
 
+
+def test_hdfs(hdfs_test_client):
+    host, user, port = hdfs_test_client
+    path = UPath('hdfs:/', host=host, user=user, port=port, driver='libhdfs3')
+    assert path.exists()
+