Merge pull request #2 from Quansight/flit-config

add python install and github actions

Author: andrewfulton9

.github/workflows/python.yml

@@ -0,0 +1,29 @@
+name: Python package
+on: push
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip nox
+        nox -s install
+    - name: Test with pytest
+      run: |
+        nox -s smoke
+        
+    - name: Publish package
+      if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: andrewfulton9
+        password: ${{ secrets.PYPI_API_TOKEN }}

noxfile.py

@@ -0,0 +1,18 @@
+import nox
+
+
+@nox.session(python=False)
+def develop(session):
+    session.install("flit")
+    session.run(*"flit install -s".split())
+
+
+@nox.session(python=False)
+def install(session):
+    session.install(".")
+
+
+@nox.session(python=False)
+def smoke(session):
+    session.install(*"pytest".split())
+    session.run(*"pytest --skiphdfs upath".split())

pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+build-backend = "flit_core.buildapi"
+requires = ["flit_core>=2,<4"]
+
+[tool]
+[tool.flit]
+[tool.flit.metadata]
+author = "Andrew Fulton"
+author-email = "[email protected]"
+classifiers = []
+home-page = "https://github.com/Quansight/universal_pathlib"
+keywords = ""
+license = ""
+maintainer = "Andrew Fulton"
+maintainer-email = "[email protected]"
+module = "upath"
+requires = ["fsspec"]
+requires-python = ">=3.7"
+
+[tool.flit.metadata.urls]
+
+[tool.flit.metadata.requires-extra]
+dev = []
+doc = []
+test = [
+  "requests",
+  "s3fs",
+  "jupyter   ",
+  "ipython",
+  "pytest",
+  "pylint",
+  "flake8",
+  "pyarrow",
+  "moto",
+  "hadoop-test-cluster",
+]
+
+[tool.flit.scripts]
+
+[tool.flit.sdist]
+include = []
+
+[tool.flit.entrypoints]

upath/__init__.py

@@ -1 +1,4 @@
+"""pathlib with fsspec"""
+__version__ = "0.0.3"
+
 from upath.core import UPath

upath/tests/conftest.py

@@ -8,14 +8,35 @@
 
 
 import pytest
-from fsspec import filesystem
 from fsspec.implementations.local import LocalFileSystem
 from fsspec.registry import (
     register_implementation,
     _registry
 )
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--skiphdfs",
+        action="store_true",
+        default=False,
+        help="skip hdfs tests"
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "hdfs: mark test as hdfs")
+
+
+def pytest_collection_modifyitems(config, items):
+    if not config.getoption("--skiphdfs"):
+        return
+    skip_hdfs = pytest.mark.skip(reason="skipping hdfs")
+    for item in items:
+        if "hdfs" in item.keywords:
+            item.add_marker(skip_hdfs)
+
+
 class DummyTestFS(LocalFileSystem):
     protocol = "mock"
 
@@ -28,25 +49,16 @@ def clear_registry():
     finally:
         _registry.clear()
 
-        
-# folder_structure = {
-#     'folders': {'folder1': {'folders': {},
-#                             'files': {'file1.txt': 'file1.txt',
-#                                       'file2.txt': 'file2.txt'}}
-#     'files': {'file1.txt': 'hello_world',
-#               'file2.txt': 'hello_world'}
-        
-    
-# }
 
 @pytest.fixture()
 def tempdir(clear_registry):
     tempdir = tempfile.TemporaryDirectory()
     tempdir = tempdir.name
     return tempdir
 
+
 @pytest.fixture()
-def local_testdir(tempdir, clear_registry):   
+def local_testdir(tempdir, clear_registry):
     tmp = Path(tempdir)
     tmp.mkdir()
     folder1 = tmp.joinpath('folder1')
@@ -66,26 +78,28 @@ def local_testdir(tempdir, clear_registry):
     yield tempdir
     shutil.rmtree(tempdir)
 
+
 @pytest.fixture(scope='session')
 def htcluster():
     proc = subprocess.Popen(shlex.split("htcluster startup"),
-                            stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
+                            stderr=subprocess.DEVNULL,
+                            stdout=subprocess.DEVNULL)
     time.sleep(30)
     yield
     proc.terminate()
     proc.wait()
     proc1 = subprocess.Popen(shlex.split("htcluster shutdown"),
-                            stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
+                             stderr=subprocess.DEVNULL,
+                             stdout=subprocess.DEVNULL)
     proc1.terminate()
     proc1.wait()
     time.sleep(10)
-    
+
+
 @pytest.fixture()
 def hdfs(htcluster, tempdir, local_testdir):
-    
     pyarrow = pytest.importorskip('pyarrow')
     host, user, port = '0.0.0.0', 'hdfs', 9000
-    #hdfs = filesystem('hdfs', host=host, user=user, port=port, driver='libhdfs3')
     hdfs = pyarrow.hdfs.connect(host='0.0.0.0', port=9000, user=user)
     hdfs.mkdir(tempdir, create_parents=True)
     for x in Path(local_testdir).glob('**/*'):
@@ -96,10 +110,10 @@ def hdfs(htcluster, tempdir, local_testdir):
             with hdfs.open(str(x), 'wb') as f:
                 f.write(text)
         else:
-            hdfs.mkdir(str(x))    
+            hdfs.mkdir(str(x))
     hdfs.close()
     yield host, user, port
-    
+
 
 @pytest.fixture(scope='session')
 def s3_server():
@@ -131,8 +145,7 @@ def s3_server():
         time.sleep(0.1)  # pragma: no cover
     anon = False
     s3so = dict(client_kwargs={'endpoint_url': endpoint_uri},
-                use_listings_cache=False)
-    
+                use_listings_cache=False)    
     yield anon, s3so
     proc.terminate()
     proc.wait()

upath/tests/test_core.py

@@ -194,7 +194,7 @@ def test_write_text(self, pathlib_base):
         path.write_text(s)
         assert path.read_text() == s
 
-
+@pytest.mark.hdfs
 class TestUPathHDFS(TestUpath):
 
     @pytest.fixture(autouse=True)
@@ -254,6 +254,7 @@ def test_touch_unlink(self):
         # file doesn't exists, but missing_ok is True
         path.unlink(missing_ok=True)
 
+@pytest.mark.hdfs
 def test_multiple_backend_paths(local_testdir, s3, hdfs):
     anon, s3so = s3
     path = f's3:{local_testdir}'