File tree Expand file tree Collapse file tree 6 files changed +66
-6
lines changed
Expand file tree Collapse file tree 6 files changed +66
-6
lines changed Original file line number Diff line number Diff line change 1515import pyarrow .dataset as pds
1616import tqdm
1717from fsspec import AbstractFileSystem
18- from fsspec .core import strip_protocol
18+ from fsspec .core import split_protocol
1919
2020# Initialize logger
2121logger = logging .getLogger (__name__ )
4141from .table import PydalaTable
4242
4343
44+ def strip_protocol (path : str ) -> str :
45+ """Strips the protocol from a given path.
46+
47+ Args:
48+ path (str): The input path which may contain a protocol.
49+ Returns:
50+ str: The path without the protocol.
51+ """
52+ protocol , path = split_protocol (path )
53+ return path
54+
55+
4456class BaseDataset :
4557 """A base class for dataset operations supporting multiple file formats.
4658
Original file line number Diff line number Diff line change 55import re
66import string
77from typing import Any
8- from fsspec .core import strip_protocol
8+ from fsspec .core import split_protocol
9+
10+
11+ def strip_protocol (path : str ) -> str :
12+ """Strips the protocol from a given path.
13+
14+ Args:
15+ path (str): The input path which may contain a protocol.
16+ Returns:
17+ str: The path without the protocol.
18+ """
19+ protocol , path = split_protocol (path )
20+ return path
921
1022
1123def escape_sql_identifier (identifier : str ) -> str :
Original file line number Diff line number Diff line change 1313import pyarrow .parquet as pq
1414from fsspec import AbstractFileSystem
1515from fsspec import filesystem as fsspec_filesystem
16- from fsspec .core import strip_protocol
16+ from fsspec .core import split_protocol
1717from loguru import logger
1818
1919# Local imports
2424from .table import PydalaTable
2525
2626
27+ def strip_protocol (path : str ) -> str :
28+ """Strips the protocol from a given path.
29+
30+ Args:
31+ path (str): The input path which may contain a protocol.
32+ Returns:
33+ str: The path without the protocol.
34+ """
35+ protocol , path = split_protocol (path )
36+ return path
37+
38+
2739def write_table (
2840 table : pa .Table ,
2941 path : str ,
Original file line number Diff line number Diff line change 1414import pyarrow .fs as pfs
1515import pyarrow .parquet as pq
1616from fsspec import AbstractFileSystem
17- from fsspec .core import strip_protocol
17+ from fsspec .core import split_protocol
1818from loguru import logger
1919
2020from .filesystem import FileSystem , clear_cache
2828)
2929
3030
31+ def strip_protocol (path : str ) -> str :
32+ """Strips the protocol from a given path.
33+
34+ Args:
35+ path (str): The input path which may contain a protocol.
36+ Returns:
37+ str: The path without the protocol.
38+ """
39+ protocol , path = split_protocol (path )
40+ return path
41+
42+
3143def serialize_metadata (metadata : dict [str , pq .FileMetaData ]) -> dict [str , Any ]:
3244 """Safely serialize metadata to a JSON-compatible format.
3345
Original file line number Diff line number Diff line change 55import pyarrow .fs as pfs
66import pyarrow .parquet as pq
77from fsspec import AbstractFileSystem
8- from fsspec .core import strip_protocol
8+ from fsspec .core import split_protocol
99from .helpers .misc import read_table , run_parallel , unify_schemas_pl
1010
1111
12+ def strip_protocol (path : str ) -> str :
13+ """Strips the protocol from a given path.
14+
15+ Args:
16+ path (str): The input path which may contain a protocol.
17+ Returns:
18+ str: The path without the protocol.
19+ """
20+ protocol , path = split_protocol (path )
21+ return path
22+
23+
1224def sort_schema (schema : pa .Schema , names : list [str ] | None = None ) -> pa .Schema :
1325 """Sort fields of a pyarrow schema in alphabetical order.
1426
Original file line number Diff line number Diff line change @@ -25,7 +25,7 @@ name = "pydala2"
2525readme = " README.md"
2626# repository = "https://github.com/legout/pydala2"
2727requires-python = " >= 3.11"
28- version = " 0.22.2 "
28+ version = " 0.22.3 "
2929
3030[project .optional-dependencies ]
3131legacy = [" polars-lts-cpu>=0.20.4" ]
You can’t perform that action at this time.
0 commit comments