Skip to content

Commit 570c6da

Browse files
ziqifan617ziqifan617
committed
address jacky's comments
1 parent b1268a5 commit 570c6da

File tree

7 files changed

+59
-228
lines changed

7 files changed

+59
-228
lines changed

‎qa/L0_backend_python/parameters/response_parameters_bls_test.py‎

Lines changed: 0 additions & 171 deletions
This file was deleted.

‎qa/L0_backend_python/parameters/response_parameters_test.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
2525
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2626

27+
import os
2728
import sys
2829

2930
sys.path.append("../../common")
@@ -39,7 +40,7 @@
3940

4041
class ResponseParametersTest(unittest.TestCase):
4142
_server_address_grpc = "localhost:8001"
42-
_model_name = "response_parameters"
43+
_model_name = os.environ["MODEL_NAME"]
4344
_shape = [1, 1]
4445

4546
def setUp(self):
@@ -166,6 +167,18 @@ def test_setting_response_parameters_decoupled(self):
166167
output = str(result.as_numpy("OUTPUT")[0][0], encoding="utf-8")
167168
self.assertEqual(json.dumps(params[i]), output)
168169

170+
def test_setting_response_parameters_bls_decoupled(self):
171+
model_name = "response_parameters_bls_decoupled"
172+
params = [{"bool": False, "int": 2048}, {"str": "Hello World!"}]
173+
params_str = json.dumps(params)
174+
175+
inputs = [grpcclient.InferInput("RESPONSE_PARAMETERS", self._shape, "BYTES")]
176+
inputs[0].set_data_from_numpy(np.array([[params_str]], dtype=np.object_))
177+
178+
with self._shm_leak_detector.Probe() as shm_probe:
179+
with grpcclient.InferenceServerClient(self._server_address_grpc) as client:
180+
client.infer(model_name, inputs)
181+
169182

170183
if __name__ == "__main__":
171184
unittest.main()

‎qa/L0_backend_python/parameters/test.sh‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,18 +59,16 @@ if [ "$SERVER_PID" == "0" ]; then
5959
fi
6060

6161
set +e
62-
python3 -m pytest --junitxml=response_parameters_test.report.xml response_parameters_test.py > $TEST_LOG 2>&1
62+
MODEL_NAME=response_parameters python3 -m pytest --junitxml=response_parameters_test.report.xml response_parameters_test.py > $TEST_LOG 2>&1
6363
if [ $? -ne 0 ]; then
6464
echo -e "\n***\n*** Response parameters test FAILED\n***"
6565
cat $TEST_LOG
6666
RET=1
6767
fi
68-
set -e
6968

70-
set +e
71-
python3 -m pytest --junitxml=response_parameters_bls_test.report.xml response_parameters_bls_test.py > $TEST_BLS_LOG 2>&1
69+
MODEL_NAME=response_parameters_bls python3 -m pytest -s --junitxml=response_parameters_bls_test.report.xml response_parameters_test.py > $TEST_BLS_LOG 2>&1
7270
if [ $? -ne 0 ]; then
73-
echo -e "\n***\n*** Response parameters in BLS mode test FAILED\n***"
71+
echo -e "\n***\n*** Response parameters BLS test FAILED\n***"
7472
cat $TEST_BLS_LOG
7573
RET=1
7674
fi

‎qa/python_models/response_parameters_bls/config.pbtxt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
1+
# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
#
33
# Redistribution and use in source and binary forms, with or without
44
# modification, are permitted provided that the following conditions
@@ -49,4 +49,4 @@ instance_group [
4949
count: 1
5050
kind: KIND_CPU
5151
}
52-
]
52+
]

‎qa/python_models/response_parameters_bls/model.py‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
1+
# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
#
33
# Redistribution and use in source and binary forms, with or without
44
# modification, are permitted provided that the following conditions
@@ -45,20 +45,19 @@ def execute(self, requests):
4545
responses = []
4646

4747
for request in requests:
48+
bls_input_tensor = pb_utils.get_input_tensor_by_name(
49+
request, "RESPONSE_PARAMETERS"
50+
)
51+
bls_request = pb_utils.InferenceRequest(
52+
model_name="response_parameters",
53+
inputs=[bls_input_tensor],
54+
requested_output_names=["OUTPUT"],
55+
)
4856
try:
49-
bls_input_tensor = pb_utils.get_input_tensor_by_name(
50-
request, "RESPONSE_PARAMETERS"
51-
)
52-
bls_request = pb_utils.InferenceRequest(
53-
model_name="response_parameters",
54-
inputs=[bls_input_tensor],
55-
requested_output_names=["OUTPUT"],
56-
)
5757
bls_response = bls_request.exec()
5858
response_tensors = bls_response.output_tensors()
5959
response_parameters_str = bls_response.parameters()
6060
if bls_response.has_error():
61-
print(bls_response.error().message())
6261
raise Exception(bls_response.error().message())
6362
res_params = json.loads(response_parameters_str)
6463

‎qa/python_models/response_parameters_bls_decoupled/config.pbtxt‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,3 @@ instance_group [
5050
kind: KIND_CPU
5151
}
5252
]
53-
54-
model_transaction_policy {
55-
decoupled: True
56-
}

‎qa/python_models/response_parameters_bls_decoupled/model.py‎

Lines changed: 31 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -32,58 +32,54 @@
3232

3333
class TritonPythonModel:
3434
"""
35-
This model (A) is designed to test sending back response parameters when using BLS
35+
This model is designed to test sending back response parameters when using BLS
3636
with decoupled model transaction policy.
3737
38-
The only difference vs. response_parameters_bls model is this model turns on decoupled
39-
model transaction policy. For more details, please check response_parameters_bls.
38+
The only difference vs. response_parameters_bls model is the BLS composing model
39+
(i.e. response_parameters_decoupled) turns on decoupled model transaction policy.
40+
For more details, please check response_parameters_bls model.
4041
"""
4142

4243
def execute(self, requests):
44+
responses = []
45+
4346
for request in requests:
44-
res_params_tensor = pb_utils.get_input_tensor_by_name(
47+
bls_input_tensor = pb_utils.get_input_tensor_by_name(
4548
request, "RESPONSE_PARAMETERS"
46-
).as_numpy()
47-
res_params_str = str(res_params_tensor[0][0], encoding="utf-8")
48-
response_sender = request.get_response_sender()
49+
)
50+
bls_request = pb_utils.InferenceRequest(
51+
model_name="response_parameters_decoupled",
52+
inputs=[bls_input_tensor],
53+
requested_output_names=["OUTPUT"],
54+
)
55+
56+
res_params_numpy = bls_input_tensor.as_numpy()
57+
res_params_str = str(res_params_numpy[0][0], encoding="utf-8")
58+
res_params = json.loads(res_params_str)
4959
try:
50-
res_params = json.loads(res_params_str)
51-
for r_params in res_params:
52-
bls_input_tensor = pb_utils.Tensor(
53-
"RESPONSE_PARAMETERS",
54-
np.array([[json.dumps(r_params)]], dtype=np.object_),
55-
)
56-
bls_request = pb_utils.InferenceRequest(
57-
model_name="response_parameters",
58-
inputs=[bls_input_tensor],
59-
requested_output_names=["OUTPUT"],
60-
)
61-
bls_response = bls_request.exec()
62-
response_tensors = bls_response.output_tensors()
63-
response_parameters_str = bls_response.parameters()
60+
bls_responses = bls_request.exec(decoupled=True)
61+
62+
for bls_response, r_params in zip(bls_responses, res_params):
6463
if bls_response.has_error():
65-
print(bls_response.error().message())
6664
raise Exception(bls_response.error().message())
67-
res_params = json.loads(response_parameters_str)
68-
69-
response = pb_utils.InferenceResponse(
70-
output_tensors=response_tensors, parameters=res_params
71-
)
7265

7366
r_params_set = {}
74-
if response.parameters() != "":
75-
r_params_set = json.loads(response.parameters())
76-
if r_params_set != r_params:
77-
raise Exception("Response parameters set differ from provided")
67+
if bls_response.parameters() != "":
68+
r_params_set = json.loads(bls_response.parameters())
69+
if r_params_set != r_params:
70+
raise Exception(
71+
"Response parameters set differ from provided"
72+
)
7873

79-
response_sender.send(response)
74+
# no need to send back anything in the response since we already do the
75+
# parameters matching checking above.
76+
response = pb_utils.InferenceResponse()
8077
except Exception as e:
8178
error = pb_utils.TritonError(
8279
message=str(e), code=pb_utils.TritonError.INVALID_ARG
8380
)
8481
response = pb_utils.InferenceResponse(error=error)
85-
response_sender.send(response)
8682

87-
response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL)
83+
responses.append(response)
8884

89-
return None
85+
return responses

0 commit comments

Comments
 (0)