We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 9e7c1a4 commit 2caf0faCopy full SHA for 2caf0fa
modelopt/torch/utils/plugins/megatron_preprocess_data.py
@@ -239,10 +239,15 @@ def main():
239
if args.input_path is None:
240
args.input_path = []
241
242
- response = requests.get(
243
- "https://datasets-server.huggingface.co/splits?dataset={}".format(args.dataset),
244
- timeout=10,
245
- )
+ try:
+ response = requests.get(
+ f"https://datasets-server.huggingface.co/splits?dataset={args.dataset}",
+ timeout=10,
246
+ )
247
+ response.raise_for_status()
248
+ except requests.RequestException as e:
249
+ print(f"Failed to fetch dataset splits for {args.dataset}: {e}")
250
+ return
251
252
for entry in response.json()["splits"]:
253
skip_processing = False
0 commit comments