Skip to content

Commit 6cc1735

Browse files
committed
Add snapshot-loading-mode to RESTCatalog
This will allow to set the snapshots to be send back. In case of refs, only the snapshots referenced by branches or tags will be returned: https://github.com/apache/iceberg/blob/5d2230ead79da64a8c871a02eb1304a94aaece5c/open-api/rest-catalog-open-api.yaml#L954-L956
1 parent eb8756a commit 6cc1735

File tree

2 files changed

+54
-3
lines changed

2 files changed

+54
-3
lines changed

pyiceberg/catalog/rest/__init__.py

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ class IdentifierKind(Enum):
138138
SIGV4_REGION = "rest.signing-region"
139139
SIGV4_SERVICE = "rest.signing-name"
140140
OAUTH2_SERVER_URI = "oauth2-server-uri"
141+
SNAPSHOT_LOADING_MODE = "snapshot-loading-mode"
141142

142143
NAMESPACE_SEPARATOR = b"\x1f".decode(UTF8)
143144

@@ -678,7 +679,16 @@ def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]:
678679

679680
@retry(**_RETRY_ARGS)
680681
def load_table(self, identifier: Union[str, Identifier]) -> Table:
681-
response = self._session.get(self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier)))
682+
params = {}
683+
if mode := self.properties.get(SNAPSHOT_LOADING_MODE):
684+
if mode in {"all", "refs"}:
685+
params["snapshots"] = mode
686+
else:
687+
raise ValueError("Invalid snapshot-loading-mode: {}")
688+
689+
response = self._session.get(
690+
self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier)), params=params
691+
)
682692
try:
683693
response.raise_for_status()
684694
except HTTPError as exc:
@@ -816,7 +826,7 @@ def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identi
816826
try:
817827
response.raise_for_status()
818828
except HTTPError as exc:
819-
self._handle_non_200_response(exc, {})
829+
self._handle_non_200_response(exc, {404: NoSuchNamespaceError})
820830

821831
return ListNamespaceResponse.model_validate_json(response.text).namespaces
822832

tests/catalog/test_rest.py

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525
import pyiceberg
2626
from pyiceberg.catalog import PropertiesUpdateSummary, load_catalog
27-
from pyiceberg.catalog.rest import OAUTH2_SERVER_URI, RestCatalog
27+
from pyiceberg.catalog.rest import OAUTH2_SERVER_URI, SNAPSHOT_LOADING_MODE, RestCatalog
2828
from pyiceberg.exceptions import (
2929
AuthorizationExpiredError,
3030
NamespaceAlreadyExistsError,
@@ -555,6 +555,24 @@ def test_list_namespace_with_parent_200(rest_mock: Mocker) -> None:
555555
]
556556

557557

558+
def test_list_namespace_with_parent_404(rest_mock: Mocker) -> None:
559+
rest_mock.get(
560+
f"{TEST_URI}v1/namespaces?parent=some_namespace",
561+
json={
562+
"error": {
563+
"message": "Namespace provided in the `parent` query parameter is not found",
564+
"type": "NoSuchNamespaceException",
565+
"code": 404,
566+
}
567+
},
568+
status_code=404,
569+
request_headers=TEST_HEADERS,
570+
)
571+
572+
with pytest.raises(NoSuchNamespaceError):
573+
RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces(("some_namespace",))
574+
575+
558576
@pytest.mark.filterwarnings(
559577
"ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
560578
)
@@ -835,6 +853,29 @@ def test_load_table_200(rest_mock: Mocker, example_table_metadata_with_snapshot_
835853
assert actual == expected
836854

837855

856+
def test_load_table_200_loading_mode(
857+
rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: Dict[str, Any]
858+
) -> None:
859+
rest_mock.get(
860+
f"{TEST_URI}v1/namespaces/fokko/tables/table?snapshots=refs",
861+
json=example_table_metadata_with_snapshot_v1_rest_json,
862+
status_code=200,
863+
request_headers=TEST_HEADERS,
864+
)
865+
catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{SNAPSHOT_LOADING_MODE: "refs"})
866+
actual = catalog.load_table(("fokko", "table"))
867+
expected = Table(
868+
identifier=("fokko", "table"),
869+
metadata_location=example_table_metadata_with_snapshot_v1_rest_json["metadata-location"],
870+
metadata=TableMetadataV1(**example_table_metadata_with_snapshot_v1_rest_json["metadata"]),
871+
io=load_file_io(),
872+
catalog=catalog,
873+
)
874+
# First compare the dicts
875+
assert actual.metadata.model_dump() == expected.metadata.model_dump()
876+
assert actual == expected
877+
878+
838879
def test_load_table_honor_access_delegation(
839880
rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: Dict[str, Any]
840881
) -> None:

0 commit comments

Comments
 (0)