Merge pull request #3 from martindurant/ci2

now ci runs

Author: martindurant

.github/workflows/main.yaml

@@ -28,7 +28,7 @@ jobs:
           python-version: ${{ matrix.PY }}
       - name: Install
         run: |
-          pip install ./fsspec-proxy
-          pip install ./pyscript-fsspec-client[test]
+          pip install -e ./fsspec-proxy
+          pip install -e ./pyscript-fsspec-client[test]
       - name: test
-        run: pytest -v
+        run: pytest -v -s

fsspec-proxy/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Jupyter fsspec contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

fsspec-proxy/LICENSE

@@ -1,31 +1,42 @@
 from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 import fsspec.utils
+import io
 import os
-import sys
 import yaml
 import logging
 
-logging.basicConfig(level=logging.WARNING, stream=sys.stdout)
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-fsspec.utils.setup_logging(logger_name="fsspec.memoryfs")
-this_dir = os.path.abspath(os.path.dirname(__file__))
-
+logger = logging.getLogger("fsspec_proxy")
+default_config = b"""sources:
+ - name: inmemory
+   path: memory://mytests
+ - name: local
+   path: file:///Users
+   readonly: true
+ - name: "Conda Stats"
+   path: "s3://anaconda-package-data/conda/hourly/"
+   kwargs:
+     anon: True
+ - name: "MyAnaconda"
+   path: "anaconda://my/"
+allow_reload: true
+"""
 
 class FileSystemManager:
-    def __init__(self):
+    def __init__(self, config_path=None):
         self.filesystems = {}
-        if "FSSPEC_PROXY_CONFIG" in os.environ:
-            self.config = self.load_config(os.getenv("FSSPEC_PROXY_CONFIG"))
-        else:
-            self.config = self.load_config()
+        config_path = config_path or os.getenv("FSSPEC_PROXY_CONFIG", None)
+        self.config = self.load_config(config_path)
         self.initialize_filesystems()
 
-    def load_config(self, config_path=os.path.join(this_dir, "config.yaml")):
-        if not os.path.exists(config_path):
+    def load_config(self, config_path=None):
+        if config_path is None:
+            data = default_config
+        elif not os.path.exists(config_path):
             return {}
-        with open(config_path, "r") as file:
-            config_content = yaml.safe_load(file)
+        else:
+            with open(config_path, "rb") as file:
+                data = file.read()
+        config_content = yaml.safe_load(io.BytesIO(data))
         logger.info("new config: %s", config_content)
         return config_content
 
@@ -37,7 +48,12 @@ def initialize_filesystems(self):
             fs_path = fs_config["path"]
             kwargs = fs_config.get("kwargs", {})
 
-            fs, url2 = fsspec.url_to_fs(fs_path, **kwargs)
+            try:
+                fs, url2 = fsspec.url_to_fs(fs_path, **kwargs)
+            except Exception:
+                # or we could still list show their names but not the contents
+                logger.error("Instantiating filesystem failed")
+                continue
             if not fs.async_impl:
                 fs = AsyncFileSystemWrapper(fs)
 

fsspec-proxy/fsspec_proxy/file_manager.py

@@ -8,7 +8,7 @@ authors = [
     {name = "martindurant"},
 ]
 readme = "../README.md"
-license = { file = "../LICENSE" }
+license = { file = "LICENSE" }
 requires-python = ">=3.9"
 classifiers = [
     # "Framework :: PyScript",
@@ -23,6 +23,7 @@ classifiers = [
 ]
 dependencies = [
     "fsspec>=2025.3.0",
+    "fastapi-cli",
     "fastapi"
 ]
 dynamic = ["version", "description", "urls", "keywords"]

fsspec-proxy/pyproject.toml

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Jupyter fsspec contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pyscript-fsspec-client/LICENSE

@@ -3,12 +3,12 @@ requires = ["hatchling>=1.5.0", "hatch-vcs"]
 build-backend = "hatchling.build"
 
 [project]
-name = "pyscript-client"
+name = "pyscript-fsspec-client"
 authors = [
     {name = "martindurant"},
 ]
 readme = "../README.md"
-license = { file = "../LICENSE" }
+license = { file = "LICENSE" }
 requires-python = ">=3.9"
 classifiers = [
     "License :: OSI Approved :: BSD License",

pyscript-fsspec-client/LICENSE

@@ -7,18 +7,20 @@
 from pyscript_fsspec_client import client
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def server():
     # TODO: test config in "FSSPEC_PROXY_CONFIG" location
-    P = subprocess.Popen(["fsspec-proxy", "dev"])
-    s = "http://127.0.0.1:8000"
-    count = 5
+    P = subprocess.Popen(["fsspec-proxy"])
+    s = "http://localhost:8000"
+    count = 20
     while True:
         try:
             requests.get(f"{s}/health")
             break
-        except OSError:
+        except BaseException:
             if count < 0:
+                P.terminate()
+                P.wait()
                 raise
         count -= 1
         time.sleep(0.1)
@@ -44,7 +46,7 @@ def test_file(fs):
 
 def test_config(fs):
     out = fs.ls("", detail=False)
-    assert out == ["Conda Stats", "MyAnaconda", "inmemory", "local"]
+    assert "inmemory" in out and "local" in out  # other spaces might fail
     fs.reconfigure({"sources": [{"name": "mem", "path": "memory://"}]})
     out = fs.ls("", detail=False)
     assert out == ["mem"]