2025-06-16 09:23:54,994 DEBUG tune_controller.py:1240 -- Future TRAIN FAILED for trial lambda_ad490_00007: �[36mray::ImplicitFunc.train()�[39m (pid=2334149, ip=172.16.4.67, actor_id=42134d1d0ac57d3d88bbdb5fbc010000, repr=<lambda>)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py", line 330, in train
raise skipped from exception_cause(skipped)
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/air/_internal/util.py", line 107, in run
self._ret = self._target(*self._args, **self._kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py", line 45, in <lambda>
training_func=lambda: self._trainable_func(self.config),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py", line 261, in _trainable_func
output = fn()
^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/optimization.py", line 278, in <lambda>
lambda _: tune_objective(
^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/optimization.py", line 42, in tune_objective
obj_1, obj_2 = objective(trial, study_config, components)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/tuner/qa_tuner.py", line 359, in objective
raise ex
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/tuner/qa_tuner.py", line 349, in objective
obj1, obj2, metrics, flow_json = evaluate(params, study_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/tuner/qa_tuner.py", line 112, in evaluate
obj1, obj2, results = _evaluate(params, study_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/tuner/qa_tuner.py", line 307, in _evaluate
results: T.Dict[str, T.Any] = eval_dataset(
^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 798, in eval_dataset
results, prune_reason = async_eval_debias(
^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 657, in async_eval_debias
return _async_eval_runner(
^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 556, in _async_eval_runner
batch_result = loop.run_until_complete(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/nest_asyncio.py", line 98, in run_until_complete
return f.result()
^^^^^^^^^^
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/asyncio/futures.py", line 203, in result
raise self._exception.with_traceback(self._exception_tb)
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 314, in __step_run_and_handle_result
result = coro.send(None)
^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 515, in _aeval_all_pair_runner
r = t.result()
^^^^^^^^^^
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/asyncio/futures.py", line 203, in result
raise self._exception.with_traceback(self._exception_tb)
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 314, in __step_run_and_handle_result
result = coro.send(None)
^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 436, in _aeval_pair_debias
eval_result, evaluation_exception = await aevaluate_pair(
^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/asyncio/__init__.py", line 189, in async_wrapped
return await copy(fn, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/asyncio/__init__.py", line 111, in __call__
do = await self.iter(retry_state=retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/asyncio/__init__.py", line 153, in iter
result = await action(retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/_utils.py", line 99, in inner
return call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/__init__.py", line 418, in exc_check
raise retry_exc.reraise()
^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/__init__.py", line 185, in reraise
raise self.last_attempt.result()
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 449, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
raise self._exception
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/asyncio/__init__.py", line 114, in __call__
result = await fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 319, in aevaluate_pair
return await exception_catcher(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 235, in exception_catcher
results = await func(**kwargs)
^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/instrumentation/dispatcher.py", line 370, in async_wrapper
result = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/evaluation/correctness.py", line 137, in aevaluate
eval_response = await self._llm.apredict(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/instrumentation/dispatcher.py", line 370, in async_wrapper
result = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/llms/llm.py", line 713, in apredict
response = await self.acomplete(formatted_prompt, formatted=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/instrumentation/dispatcher.py", line 370, in async_wrapper
result = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/llms/callbacks.py", line 334, in wrapped_async_llm_predict
f_return_val = await f(_self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/llms/azure_inference/base.py", line 413, in acomplete
return await acomplete_fn(prompt, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/base/llms/generic_utils.py", line 227, in wrapper
chat_response = await func(messages, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/instrumentation/dispatcher.py", line 370, in async_wrapper
result = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/llms/callbacks.py", line 75, in wrapped_async_llm_chat
f_return_val = await f(_self, messages, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/llms/azure_inference/base.py", line 399, in achat
response = await self._async_client.complete(messages=messages, **all_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/azure/ai/inference/aio/_patch.py", line 675, in complete
map_error(status_code=response.status_code, response=response, error_map=error_map)
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/azure/core/exceptions.py", line 163, in map_error
raise error
azure.core.exceptions.ClientAuthenticationError: (None) Auth token validation failed. Please provide a valid key or a valid AAD token
Code: None
Message: Auth token validation failed. Please provide a valid key or a valid AAD token
2025-06-16 09:23:54,994 ERROR tune_controller.py:1331 -- Trial task failed for trial lambda_ad490_00007
Traceback (most recent call last):
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/air/execution/_internal/event_manager.py", line 110, in resolve_future
result = ray.get(future)
^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/_private/auto_init_hook.py", line 21, in auto_init_wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/_private/worker.py", line 2822, in get
values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/_private/worker.py", line 930, in get_objects
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(ClientAuthenticationError): �[36mray::ImplicitFunc.train()�[39m (pid=2334149, ip=172.16.4.67, actor_id=42134d1d0ac57d3d88bbdb5fbc010000, repr=<lambda>)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/tune/trainable/trainable.py", line 330, in train
raise skipped from exception_cause(skipped)
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/air/_internal/util.py", line 107, in run
self._ret = self._target(*self._args, **self._kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py", line 45, in <lambda>
training_func=lambda: self._trainable_func(self.config),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/ray/tune/trainable/function_trainable.py", line 261, in _trainable_func
output = fn()
^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/optimization.py", line 278, in <lambda>
lambda _: tune_objective(
^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/optimization.py", line 42, in tune_objective
obj_1, obj_2 = objective(trial, study_config, components)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/tuner/qa_tuner.py", line 359, in objective
raise ex
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/tuner/qa_tuner.py", line 349, in objective
obj1, obj2, metrics, flow_json = evaluate(params, study_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/tuner/qa_tuner.py", line 112, in evaluate
obj1, obj2, results = _evaluate(params, study_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/tuner/qa_tuner.py", line 307, in _evaluate
results: T.Dict[str, T.Any] = eval_dataset(
^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 798, in eval_dataset
results, prune_reason = async_eval_debias(
^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 657, in async_eval_debias
return _async_eval_runner(
^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 556, in _async_eval_runner
batch_result = loop.run_until_complete(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/nest_asyncio.py", line 98, in run_until_complete
return f.result()
^^^^^^^^^^
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/asyncio/futures.py", line 203, in result
raise self._exception.with_traceback(self._exception_tb)
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 314, in __step_run_and_handle_result
result = coro.send(None)
^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 515, in _aeval_all_pair_runner
r = t.result()
^^^^^^^^^^
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/asyncio/futures.py", line 203, in result
raise self._exception.with_traceback(self._exception_tb)
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 314, in __step_run_and_handle_result
result = coro.send(None)
^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 436, in _aeval_pair_debias
eval_result, evaluation_exception = await aevaluate_pair(
^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/asyncio/__init__.py", line 189, in async_wrapped
return await copy(fn, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/asyncio/__init__.py", line 111, in __call__
do = await self.iter(retry_state=retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/asyncio/__init__.py", line 153, in iter
result = await action(retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/_utils.py", line 99, in inner
return call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/__init__.py", line 418, in exc_check
raise retry_exc.reraise()
^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/__init__.py", line 185, in reraise
raise self.last_attempt.result()
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 449, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "/home/alex/.local/share/uv/python/cpython-3.12.7-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
raise self._exception
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/tenacity/asyncio/__init__.py", line 114, in __call__
result = await fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 319, in aevaluate_pair
return await exception_catcher(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/py_modules_files/_ray_pkg_142edfd67cb4ef74/syftr/evaluation.py", line 235, in exception_catcher
results = await func(**kwargs)
^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/instrumentation/dispatcher.py", line 370, in async_wrapper
result = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/evaluation/correctness.py", line 137, in aevaluate
eval_response = await self._llm.apredict(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/instrumentation/dispatcher.py", line 370, in async_wrapper
result = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/llms/llm.py", line 713, in apredict
response = await self.acomplete(formatted_prompt, formatted=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/instrumentation/dispatcher.py", line 370, in async_wrapper
result = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/llms/callbacks.py", line 334, in wrapped_async_llm_predict
f_return_val = await f(_self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/llms/azure_inference/base.py", line 413, in acomplete
return await acomplete_fn(prompt, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/base/llms/generic_utils.py", line 227, in wrapper
chat_response = await func(messages, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/instrumentation/dispatcher.py", line 370, in async_wrapper
result = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/core/llms/callbacks.py", line 75, in wrapped_async_llm_chat
f_return_val = await f(_self, messages, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/llama_index/llms/azure_inference/base.py", line 399, in achat
response = await self._async_client.complete(messages=messages, **all_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/azure/ai/inference/aio/_patch.py", line 675, in complete
map_error(status_code=response.status_code, response=response, error_map=error_map)
File "/tmp/ray/session_2025-06-05_07-19-04_642991_1676/runtime_resources/pip/42bb406fe48ce446a6ce2a9cea155757cb9174a4/virtualenv/lib/python3.12/site-packages/azure/core/exceptions.py", line 163, in map_error
raise error
azure.core.exceptions.ClientAuthenticationError: (None) Auth token validation failed. Please provide a valid key or a valid AAD token
Code: None
Message: Auth token validation failed. Please provide a valid key or a valid AAD token
