Skip to content

Commit 5486fa3

Browse files
ap--ap--
authored
Update flavours (#285)
* update requirements * fix xrootd integration * update sources
1 parent 3cc0871 commit 5486fa3

File tree

3 files changed

+30
-29
lines changed

3 files changed

+30
-29
lines changed

‎dev/fsspec_inspector/generate_flavours.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,10 +169,11 @@ def _fix_oss_file_system(x: str) -> str:
169169

170170
def _fix_xrootd_file_system(x: str) -> str:
171171
x = re.sub(
172-
r"client.URL",
173-
"urlsplit",
172+
r"return client[.]URL\(path\)[.]path_with_params",
173+
"x = urlsplit(path); return (x.path + f'?{x.query}' if x.query else '')",
174174
x,
175175
)
176+
x = re.sub(r"client[.]URL\(u\)", "urlsplit(u)", x)
176177
return re.sub(
177178
"url.hostid",
178179
"url.netloc",

‎dev/requirements.txt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
fsspec[git,hdfs,dask,http,sftp,smb]==2024.6.1
1+
fsspec[git,hdfs,dask,http,sftp,smb]==2024.9.0
22

33
# these dependencies define their own filesystems
44
adlfs==2024.7.0
55
boxfs==0.3.0
66
dropboxdrivefs==1.4.1
7-
gcsfs==2024.6.1
8-
s3fs==2024.6.1
7+
gcsfs==2024.9.0.post1
8+
s3fs==2024.9.0
99
ocifs==1.3.1
1010
webdav4[fsspec]==0.10.0
1111
# gfrivefs @ git+https://github.com/fsspec/gdrivefs@master broken ...
@@ -14,5 +14,5 @@ dvc==3.55.2
1414
huggingface_hub==0.24.6
1515
lakefs-spec==0.10.0
1616
ossfs==2023.12.0
17-
fsspec-xrootd==0.3.0
17+
fsspec-xrootd==0.4.0
1818
wandbfs==0.0.2

‎upath/_flavour_sources.py‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ def __init_subclass__(cls: Any, **kwargs):
8787

8888
class AbstractFileSystemFlavour(FileSystemFlavourBase):
8989
__orig_class__ = 'fsspec.spec.AbstractFileSystem'
90-
__orig_version__ = '2024.6.1'
90+
__orig_version__ = '2024.9.0'
9191
protocol: str | tuple[str, ...] = 'abstract'
9292
root_marker: Literal['', '/'] = ''
9393
sep: Literal['/'] = '/'
@@ -312,7 +312,7 @@ def _strip_protocol(cls, path) -> str:
312312

313313
class DaskWorkerFileSystemFlavour(AbstractFileSystemFlavour):
314314
__orig_class__ = 'fsspec.implementations.dask.DaskWorkerFileSystem'
315-
__orig_version__ = '2024.6.1'
315+
__orig_version__ = '2024.9.0'
316316
protocol = ('dask',)
317317
root_marker = ''
318318
sep = '/'
@@ -328,15 +328,15 @@ def _get_kwargs_from_urls(path):
328328

329329
class DataFileSystemFlavour(AbstractFileSystemFlavour):
330330
__orig_class__ = 'fsspec.implementations.data.DataFileSystem'
331-
__orig_version__ = '2024.6.1'
331+
__orig_version__ = '2024.9.0'
332332
protocol = ('data',)
333333
root_marker = ''
334334
sep = '/'
335335

336336

337337
class DatabricksFileSystemFlavour(AbstractFileSystemFlavour):
338338
__orig_class__ = 'fsspec.implementations.dbfs.DatabricksFileSystem'
339-
__orig_version__ = '2024.6.1'
339+
__orig_version__ = '2024.9.0'
340340
protocol = ('dbfs',)
341341
root_marker = ''
342342
sep = '/'
@@ -369,7 +369,7 @@ class DropboxDriveFileSystemFlavour(AbstractFileSystemFlavour):
369369

370370
class FTPFileSystemFlavour(AbstractFileSystemFlavour):
371371
__orig_class__ = 'fsspec.implementations.ftp.FTPFileSystem'
372-
__orig_version__ = '2024.6.1'
372+
__orig_version__ = '2024.9.0'
373373
protocol = ('ftp',)
374374
root_marker = '/'
375375
sep = '/'
@@ -388,7 +388,7 @@ def _get_kwargs_from_urls(urlpath):
388388

389389
class GCSFileSystemFlavour(AbstractFileSystemFlavour):
390390
__orig_class__ = 'gcsfs.core.GCSFileSystem'
391-
__orig_version__ = '2024.6.1'
391+
__orig_version__ = '2024.9.0post1'
392392
protocol = ('gs', 'gcs')
393393
root_marker = ''
394394
sep = '/'
@@ -465,7 +465,7 @@ def _split_path(cls, path, version_aware=False):
465465

466466
class GitFileSystemFlavour(AbstractFileSystemFlavour):
467467
__orig_class__ = 'fsspec.implementations.git.GitFileSystem'
468-
__orig_version__ = '2024.6.1'
468+
__orig_version__ = '2024.9.0'
469469
protocol = ('git',)
470470
root_marker = ''
471471
sep = '/'
@@ -493,7 +493,7 @@ def _get_kwargs_from_urls(path):
493493

494494
class GithubFileSystemFlavour(AbstractFileSystemFlavour):
495495
__orig_class__ = 'fsspec.implementations.github.GithubFileSystem'
496-
__orig_version__ = '2024.6.1'
496+
__orig_version__ = '2024.9.0'
497497
protocol = ('github',)
498498
root_marker = ''
499499
sep = '/'
@@ -518,7 +518,7 @@ def _get_kwargs_from_urls(path):
518518

519519
class HTTPFileSystemFlavour(AbstractFileSystemFlavour):
520520
__orig_class__ = 'fsspec.implementations.http.HTTPFileSystem'
521-
__orig_version__ = '2024.6.1'
521+
__orig_version__ = '2024.9.0'
522522
protocol = ('http', 'https')
523523
root_marker = ''
524524
sep = '/'
@@ -539,7 +539,7 @@ def _parent(cls, path):
539539

540540
class HadoopFileSystemFlavour(AbstractFileSystemFlavour):
541541
__orig_class__ = 'fsspec.implementations.arrow.HadoopFileSystem'
542-
__orig_version__ = '2024.6.1'
542+
__orig_version__ = '2024.9.0'
543543
protocol = ('hdfs', 'arrow_hdfs')
544544
root_marker = '/'
545545
sep = '/'
@@ -580,7 +580,7 @@ class HfFileSystemFlavour(AbstractFileSystemFlavour):
580580

581581
class JupyterFileSystemFlavour(AbstractFileSystemFlavour):
582582
__orig_class__ = 'fsspec.implementations.jupyter.JupyterFileSystem'
583-
__orig_version__ = '2024.6.1'
583+
__orig_version__ = '2024.9.0'
584584
protocol = ('jupyter', 'jlab')
585585
root_marker = ''
586586
sep = '/'
@@ -606,7 +606,7 @@ def _strip_protocol(cls, path):
606606

607607
class LibArchiveFileSystemFlavour(AbstractFileSystemFlavour):
608608
__orig_class__ = 'fsspec.implementations.libarchive.LibArchiveFileSystem'
609-
__orig_version__ = '2024.6.1'
609+
__orig_version__ = '2024.9.0'
610610
protocol = ('libarchive',)
611611
root_marker = ''
612612
sep = '/'
@@ -619,7 +619,7 @@ def _strip_protocol(cls, path):
619619

620620
class LocalFileSystemFlavour(AbstractFileSystemFlavour):
621621
__orig_class__ = 'fsspec.implementations.local.LocalFileSystem'
622-
__orig_version__ = '2024.6.1'
622+
__orig_version__ = '2024.9.0'
623623
protocol = ('file', 'local')
624624
root_marker = '/'
625625
sep = '/'
@@ -697,7 +697,7 @@ def _strip_protocol(cls, path):
697697

698698
class MemoryFileSystemFlavour(AbstractFileSystemFlavour):
699699
__orig_class__ = 'fsspec.implementations.memory.MemoryFileSystem'
700-
__orig_version__ = '2024.6.1'
700+
__orig_version__ = '2024.9.0'
701701
protocol = ('memory',)
702702
root_marker = '/'
703703
sep = '/'
@@ -796,15 +796,15 @@ class OverlayFileSystemFlavour(AbstractFileSystemFlavour):
796796

797797
class ReferenceFileSystemFlavour(AbstractFileSystemFlavour):
798798
__orig_class__ = 'fsspec.implementations.reference.ReferenceFileSystem'
799-
__orig_version__ = '2024.6.1'
799+
__orig_version__ = '2024.9.0'
800800
protocol = ('reference',)
801801
root_marker = ''
802802
sep = '/'
803803

804804

805805
class S3FileSystemFlavour(AbstractFileSystemFlavour):
806806
__orig_class__ = 's3fs.core.S3FileSystem'
807-
__orig_version__ = '2024.6.1'
807+
__orig_version__ = '2024.9.0'
808808
protocol = ('s3', 's3a')
809809
root_marker = ''
810810
sep = '/'
@@ -831,7 +831,7 @@ def _get_kwargs_from_urls(urlpath):
831831

832832
class SFTPFileSystemFlavour(AbstractFileSystemFlavour):
833833
__orig_class__ = 'fsspec.implementations.sftp.SFTPFileSystem'
834-
__orig_version__ = '2024.6.1'
834+
__orig_version__ = '2024.9.0'
835835
protocol = ('sftp', 'ssh')
836836
root_marker = ''
837837
sep = '/'
@@ -850,7 +850,7 @@ def _get_kwargs_from_urls(urlpath):
850850

851851
class SMBFileSystemFlavour(AbstractFileSystemFlavour):
852852
__orig_class__ = 'fsspec.implementations.smb.SMBFileSystem'
853-
__orig_version__ = '2024.6.1'
853+
__orig_version__ = '2024.9.0'
854854
protocol = ('smb',)
855855
root_marker = ''
856856
sep = '/'
@@ -870,7 +870,7 @@ def _get_kwargs_from_urls(path):
870870

871871
class TarFileSystemFlavour(AbstractFileSystemFlavour):
872872
__orig_class__ = 'fsspec.implementations.tar.TarFileSystem'
873-
__orig_version__ = '2024.6.1'
873+
__orig_version__ = '2024.9.0'
874874
protocol = ('tar',)
875875
root_marker = ''
876876
sep = '/'
@@ -886,7 +886,7 @@ class WandbFSFlavour(AbstractFileSystemFlavour):
886886

887887
class WebHDFSFlavour(AbstractFileSystemFlavour):
888888
__orig_class__ = 'fsspec.implementations.webhdfs.WebHDFS'
889-
__orig_version__ = '2024.6.1'
889+
__orig_version__ = '2024.9.0'
890890
protocol = ('webhdfs', 'webHDFS')
891891
root_marker = ''
892892
sep = '/'
@@ -921,7 +921,7 @@ def _strip_protocol(cls, path: str) -> str:
921921

922922
class XRootDFileSystemFlavour(AbstractFileSystemFlavour):
923923
__orig_class__ = 'fsspec_xrootd.xrootd.XRootDFileSystem'
924-
__orig_version__ = '0.3.0'
924+
__orig_version__ = '0.4.0'
925925
protocol = ('root',)
926926
root_marker = '/'
927927
sep = '/'
@@ -930,7 +930,7 @@ class XRootDFileSystemFlavour(AbstractFileSystemFlavour):
930930
def _strip_protocol(cls, path: str | list[str]) -> Any:
931931
if isinstance(path, str):
932932
if path.startswith(cls.protocol):
933-
return urlsplit(path).path.rstrip("/") or cls.root_marker
933+
x = urlsplit(path); return (x.path + f'?{x.query}' if x.query else '').rstrip("/") or cls.root_marker
934934
# assume already stripped
935935
return path.rstrip("/") or cls.root_marker
936936
elif isinstance(path, list):
@@ -947,7 +947,7 @@ def _get_kwargs_from_urls(u: str) -> dict[Any, Any]:
947947

948948
class ZipFileSystemFlavour(AbstractFileSystemFlavour):
949949
__orig_class__ = 'fsspec.implementations.zip.ZipFileSystem'
950-
__orig_version__ = '2024.6.1'
950+
__orig_version__ = '2024.9.0'
951951
protocol = ('zip',)
952952
root_marker = ''
953953
sep = '/'

0 commit comments

Comments
 (0)