Skip to content

fix(tests): stabilize flaky Hub LFS integration test #7889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
The-Obstacle-Is-The-Way wants to merge 2 commits into
base: main
Choose a base branch
from
+104 −64
Open

fix(tests): stabilize flaky Hub LFS integration test #7889

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
63e9cf5
fix(tests): stabilize flaky Hub LFS integration test
The-Obstacle-Is-The-Way Nov 29, 2025
8a242d8
fix: retry LFS pointer errors in push_to_hub
The-Obstacle-Is-The-Way Dec 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 26 additions & 16 deletions src/datasets/arrow_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6005,19 +6005,25 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
create_pr=create_pr,
)
except HfHubHTTPError as err:
if (
err.__context__
and isinstance(err.__context__, HfHubHTTPError)
and err.__context__.response.status_code == 409
):
response = (
err.response if err.response is not None else getattr(err.__context__, "response", None)
)
status_code = response.status_code if response is not None else None
if status_code == 409:
# 409 is Conflict (another commit is in progress)
time.sleep(sleep_time)
logger.info(
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code})"
)
continue
else:
raise
elif status_code == 400 and "lfs pointer" in str(err).lower():
# 400 with LFS error indicates LFS objects not yet propagated
time.sleep(sleep_time)
logger.info(
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code} - LFS propagation)"
)
continue
raise
break
logger.info(
f"Commit #{i + 1} completed"
Expand Down Expand Up @@ -6055,20 +6061,24 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
parent_commit=parent_commit,
)
except HfHubHTTPError as err:
if (
err.__context__
and isinstance(err.__context__, HfHubHTTPError)
and err.__context__.response.status_code in (412, 409)
):
response = err.response if err.response is not None else getattr(err.__context__, "response", None)
status_code = response.status_code if response is not None else None
if status_code in (412, 409):
# 412 is Precondition failed (parent_commit isn't satisfied)
# 409 is Conflict (another commit is in progress)
time.sleep(sleep_time)
logger.info(
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code})"
)
continue
else:
raise
elif status_code == 400 and "lfs pointer" in str(err).lower():
# 400 with LFS error indicates LFS objects not yet propagated
time.sleep(sleep_time)
logger.info(
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code} - LFS propagation)"
)
continue
raise
break

return commit_info
Expand Down
84 changes: 52 additions & 32 deletions src/datasets/dataset_dict.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1914,19 +1914,25 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
create_pr=create_pr,
)
except HfHubHTTPError as err:
if (
err.__context__
and isinstance(err.__context__, HfHubHTTPError)
and err.__context__.response.status_code == 409
):
response = (
err.response if err.response is not None else getattr(err.__context__, "response", None)
)
status_code = response.status_code if response is not None else None
if status_code == 409:
# 409 is Conflict (another commit is in progress)
time.sleep(sleep_time)
logger.info(
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code})"
)
continue
elif status_code == 400 and "lfs pointer" in str(err).lower():
# 400 with LFS error indicates LFS objects not yet propagated
time.sleep(sleep_time)
logger.info(
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code} - LFS propagation)"
)
continue
else:
raise
raise
break
logger.info(
f"Commit #{i + 1} completed"
Expand Down Expand Up @@ -1964,20 +1970,24 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
parent_commit=parent_commit,
)
except HfHubHTTPError as err:
if (
err.__context__
and isinstance(err.__context__, HfHubHTTPError)
and err.__context__.response.status_code in (412, 409)
):
response = err.response if err.response is not None else getattr(err.__context__, "response", None)
status_code = response.status_code if response is not None else None
if status_code in (412, 409):
# 412 is Precondition failed (parent_commit isn't satisfied)
# 409 is Conflict (another commit is in progress)
time.sleep(sleep_time)
logger.info(
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code})"
)
continue
elif status_code == 400 and "lfs pointer" in str(err).lower():
# 400 with LFS error indicates LFS objects not yet propagated
time.sleep(sleep_time)
logger.info(
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code} - LFS propagation)"
)
continue
else:
raise
raise
break

return commit_info
Expand Down Expand Up @@ -2783,19 +2793,25 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
create_pr=create_pr,
)
except HfHubHTTPError as err:
if (
err.__context__
and isinstance(err.__context__, HfHubHTTPError)
and err.__context__.response.status_code == 409
):
response = (
err.response if err.response is not None else getattr(err.__context__, "response", None)
)
status_code = response.status_code if response is not None else None
if status_code == 409:
# 409 is Conflict (another commit is in progress)
time.sleep(sleep_time)
logger.info(
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code})"
)
continue
elif status_code == 400 and "lfs pointer" in str(err).lower():
# 400 with LFS error indicates LFS objects not yet propagated
time.sleep(sleep_time)
logger.info(
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code} - LFS propagation)"
)
continue
else:
raise
raise
break
logger.info(
f"Commit #{i + 1} completed"
Expand Down Expand Up @@ -2833,20 +2849,24 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
parent_commit=parent_commit,
)
except HfHubHTTPError as err:
if (
err.__context__
and isinstance(err.__context__, HfHubHTTPError)
and err.__context__.response.status_code in (412, 409)
):
response = err.response if err.response is not None else getattr(err.__context__, "response", None)
status_code = response.status_code if response is not None else None
if status_code in (412, 409):
# 412 is Precondition failed (parent_commit isn't satisfied)
# 409 is Conflict (another commit is in progress)
time.sleep(sleep_time)
logger.info(
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code})"
)
continue
elif status_code == 400 and "lfs pointer" in str(err).lower():
# 400 with LFS error indicates LFS objects not yet propagated
time.sleep(sleep_time)
logger.info(
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code} - LFS propagation)"
)
continue
else:
raise
raise
break

return commit_info
42 changes: 26 additions & 16 deletions src/datasets/iterable_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4439,19 +4439,25 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
create_pr=create_pr,
)
except HfHubHTTPError as err:
if (
err.__context__
and isinstance(err.__context__, HfHubHTTPError)
and err.__context__.response.status_code == 409
):
response = (
err.response if err.response is not None else getattr(err.__context__, "response", None)
)
status_code = response.status_code if response is not None else None
if status_code == 409:
# 409 is Conflict (another commit is in progress)
time.sleep(sleep_time)
logger.info(
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code})"
)
continue
elif status_code == 400 and "lfs pointer" in str(err).lower():
# 400 with LFS error indicates LFS objects not yet propagated
time.sleep(sleep_time)
logger.info(
f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code} - LFS propagation)"
)
continue
else:
raise
raise
break
logger.info(
f"Commit #{i + 1} completed"
Expand Down Expand Up @@ -4489,20 +4495,24 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
parent_commit=parent_commit,
)
except HfHubHTTPError as err:
if (
err.__context__
and isinstance(err.__context__, HfHubHTTPError)
and err.__context__.response.status_code in (412, 409)
):
response = err.response if err.response is not None else getattr(err.__context__, "response", None)
status_code = response.status_code if response is not None else None
if status_code in (412, 409):
# 412 is Precondition failed (parent_commit isn't satisfied)
# 409 is Conflict (another commit is in progress)
time.sleep(sleep_time)
logger.info(
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code})"
)
continue
else:
raise
elif status_code == 400 and "lfs pointer" in str(err).lower():
# 400 with LFS error indicates LFS objects not yet propagated
time.sleep(sleep_time)
logger.info(
f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {status_code} - LFS propagation)"
)
continue
raise
break

return commit_info
Expand Down
Loading