Merge pull request #82 from scottyhq/http-raster

Logic for remote versus local filepaths for RasterIOSource + GitHub Actions CI

Author: martindurant

.github/workflows/main.yaml

@@ -8,12 +8,12 @@ on:
 
 jobs:
   test:
-    name: ${{ matrix.CONDA_ENV }}-build
+    name: ${{ matrix.CONDA_ENV }}-pytest
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        CONDA_ENV: [36, 36-defaults, 37-nodefaults]
+        CONDA_ENV: [36, 37, 38, 37-upstream]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -24,7 +24,7 @@ jobs:
           auto-update-conda: true
           auto-activate-base: false
           activate-environment: test_env
-          environment-file: ci/environment-${{ matrix.CONDA_ENV }}.yml
+          environment-file: ci/environment-py${{ matrix.CONDA_ENV }}.yml
 
       - name: Development Install Intake-Xarray
         shell: bash -l {0}

README.md

@@ -1,6 +1,6 @@
 # intake-xarray
 
-[![Build Status](https://travis-ci.org/intake/intake-xarray.svg?branch=master)](https://travis-ci.org/intake/intake-xarray)
+![CI](https://github.com/intake/intake-xarray/workflows/CI/badge.svg)
 
 Intake-xarray: xarray Plugin for [Intake](https://github.com/intake/intake)
 

ci/environment-py36.yml

@@ -3,12 +3,15 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - intake>=0.4.1
-  - xarray>=0.11.2
-  - zarr
-  - pytest
+  - aiohttp
+  - intake
   - netcdf4
   - pip
   - pydap
+  - pytest
   - rasterio
   - scikit-image
+  - xarray
+  - zarr
+  - pip:
+       - rangehttpserver

ci/environment-py37-upstream.yml

@@ -0,0 +1,18 @@
+name: test_env
+channels:
+  - conda-forge
+dependencies:
+  - python=3.7
+  - aiohttp
+  - netcdf4
+  - pip
+  - pydap
+  - pytest
+  - rasterio
+  - scikit-image
+  - xarray
+  - zarr
+  - pip:
+       - git+https://github.com/intake/filesystem_spec.git
+       - git+https://github.com/intake/intake.git
+       - rangehttpserver

ci/environment-py37-nodefaults.yml ci/environment-py37.ymlci/environment-py37-nodefaults.yml renamed to ci/environment-py37.yml

@@ -1,15 +1,17 @@
 name: test_env
 channels:
   - conda-forge
-  - defaults
 dependencies:
   - python=3.7
-  - intake>=0.4.1
-  - xarray
-  - zarr
-  - pytest
+  - aiohttp
+  - intake
   - netcdf4
   - pip
   - pydap
+  - pytest
   - rasterio
   - scikit-image
+  - xarray
+  - zarr
+  - pip:
+       - rangehttpserver

ci/environment-py36-defaults.yml ci/environment-py38.ymlci/environment-py36-defaults.yml renamed to ci/environment-py38.yml

@@ -1,11 +1,17 @@
 name: test_env
+channels:
+  - conda-forge
 dependencies:
-  - python=3.6
+  - python=3.8
+  - aiohttp
   - intake
-  - xarray
-  - zarr
-  - pytest
   - netcdf4
+  - pip
+  - pydap
+  - pytest
   - rasterio
   - scikit-image
-  - pip
+  - xarray
+  - zarr
+  - pip:
+       - rangehttpserver

intake_xarray/raster.py

@@ -51,6 +51,10 @@ def __init__(self, urlpath, chunks, concat_dim='concat_dim',
         self.storage_options = storage_options or {}
         self._kwargs = xarray_kwargs or {}
         self._ds = None
+        if isinstance(self.urlpath, list):
+            self._can_be_local = fsspec.utils.can_be_local(self.urlpath[0])
+        else:
+            self._can_be_local = fsspec.utils.can_be_local(self.urlpath)
         super(RasterIOSource, self).__init__(metadata=metadata)
 
     def _open_files(self, files):
@@ -74,7 +78,10 @@ def _open_files(self, files):
 
     def _open_dataset(self):
         import xarray as xr
-        files = fsspec.open_local(self.urlpath, **self.storage_options)
+        if self._can_be_local:
+            files = fsspec.open_local(self.urlpath, **self.storage_options)
+        else:
+            files = self.urlpath
         if isinstance(files, list):
             self._ds = self._open_files(files)
         else:

intake_xarray/tests/test_intake_xarray.py

@@ -16,11 +16,9 @@ def test_discover(source, netcdf_source, zarr_source, dataset):
     source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source]
     r = source.discover()
 
-    assert r['datashape'] is None
     assert r['dtype'] is None
     assert r['metadata'] is not None
 
-    assert source.datashape is None
     assert source.metadata['dims'] == dict(dataset.dims)
     assert set(source.metadata['data_vars']) == set(dataset.data_vars.keys())
     assert set(source.metadata['coords']) == set(dataset.coords.keys())

intake_xarray/tests/test_remote.py

@@ -5,13 +5,53 @@
 import subprocess
 import time
 import xarray as xr
+import fsspec
 
-
-PORT = 8425
+PORT = 8425 # for intake-server tests
 here = os.path.abspath(os.path.dirname(__file__))
 cat_file = os.path.join(here, 'data', 'catalog.yaml')
+DIRECTORY = os.path.join(here, 'data')
+
+
+@pytest.fixture(scope='module')
+def data_server():
+    ''' Serves test/data folder to http://localhost:8000 '''
+    pwd = os.getcwd()
+    os.chdir(DIRECTORY)
+    command = ['python', '-m', 'RangeHTTPServer']
+    try:
+        P = subprocess.Popen(command)
+        timeout = 10
+        while True:
+            try:
+                requests.get('http://localhost:8000')
+                break
+            except:
+                time.sleep(0.1)
+                timeout -= 0.1
+                assert timeout > 0
+        yield 'http://localhost:8000'
+    finally:
+        os.chdir(pwd)
+        P.terminate()
+        P.communicate()
+
+
+def test_list(data_server):
+    h = fsspec.filesystem("http")
+    out = h.glob(data_server + '/')
+    assert len(out) > 0
+    assert data_server+'/RGB.byte.tif' in out
+
+
+def test_open_rasterio(data_server):
+    url = f'{data_server}/RGB.byte.tif'
+    source = intake.open_rasterio(url, chunks={})
+    da = source.to_dask()
+    assert isinstance(da, xr.core.dataarray.DataArray)
 
 
+# Remote catalogs with intake-server
 @pytest.fixture(scope='module')
 def intake_server():
     command = ['intake-server', '-p', str(PORT), cat_file]

setup.py

@@ -9,7 +9,7 @@
 from setuptools import setup, find_packages
 import versioneer
 
-INSTALL_REQUIRES = ['intake >=0.5.2', 'xarray >=0.12.0', 'zarr', 'dask >=2.2', 'netcdf4']
+INSTALL_REQUIRES = ['intake >=0.5.2', 'xarray >=0.12.0', 'zarr', 'dask >=2.2', 'netcdf4', 'fsspec>0.8.3']
 
 setup(
     name='intake-xarray',