Skip to content

Commit fd921c5

Browse files
ziqifan617ziqifan617
authored
test: Add tests for response parameters support for BLS in Python backend (#7987)
1 parent 18a9c51 commit fd921c5

File tree

4 files changed

+191
-0
lines changed

4 files changed

+191
-0
lines changed

‎qa/L0_backend_python/parameters/response_parameters_test.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,31 @@ def test_setting_response_parameters_decoupled(self):
166166
output = str(result.as_numpy("OUTPUT")[0][0], encoding="utf-8")
167167
self.assertEqual(json.dumps(params[i]), output)
168168

169+
def test_setting_response_parameters_bls(self):
170+
model_name = "response_parameters_bls"
171+
params = {"bool": False, "int": 2048, "str": "Hello World!"}
172+
params_decoupled = [{}, {"bool": True, "int": 10000}, {"str": "?"}]
173+
params_str = json.dumps(params)
174+
params_decoupled_str = json.dumps(params_decoupled)
175+
176+
inputs = [
177+
grpcclient.InferInput("RESPONSE_PARAMETERS", self._shape, "BYTES"),
178+
grpcclient.InferInput(
179+
"RESPONSE_PARAMETERS_DECOUPLED", self._shape, "BYTES"
180+
),
181+
]
182+
inputs[0].set_data_from_numpy(np.array([[params_str]], dtype=np.object_))
183+
inputs[1].set_data_from_numpy(
184+
np.array([[params_decoupled_str]], dtype=np.object_)
185+
)
186+
187+
with self._shm_leak_detector.Probe() as shm_probe:
188+
with grpcclient.InferenceServerClient(self._server_address_grpc) as client:
189+
result = client.infer(model_name, inputs)
190+
191+
output = str(result.as_numpy("OUTPUT")[0][0], encoding="utf-8")
192+
self.assertEqual(output, "True")
193+
169194

170195
if __name__ == "__main__":
171196
unittest.main()

‎qa/L0_backend_python/parameters/test.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ mkdir -p models/response_parameters/1 && \
3939
mkdir -p models/response_parameters_decoupled/1 && \
4040
cp ../../python_models/response_parameters_decoupled/model.py models/response_parameters_decoupled/1 && \
4141
cp ../../python_models/response_parameters_decoupled/config.pbtxt models/response_parameters_decoupled
42+
mkdir -p models/response_parameters_bls/1 && \
43+
cp ../../python_models/response_parameters_bls/model.py models/response_parameters_bls/1 && \
44+
cp ../../python_models/response_parameters_bls/config.pbtxt models/response_parameters_bls
4245

4346
TEST_LOG="response_parameters_test.log"
4447
SERVER_LOG="response_parameters_test.server.log"

‎qa/python_models/response_parameters_bls/config.pbtxt

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
name: "response_parameters_bls"
28+
backend: "python"
29+
max_batch_size: 8
30+
31+
input [
32+
{
33+
name: "RESPONSE_PARAMETERS"
34+
data_type: TYPE_STRING
35+
dims: [ 1 ]
36+
},
37+
{
38+
name: "RESPONSE_PARAMETERS_DECOUPLED"
39+
data_type: TYPE_STRING
40+
dims: [ 1 ]
41+
}
42+
]
43+
44+
output [
45+
{
46+
name: "OUTPUT"
47+
data_type: TYPE_STRING
48+
dims: [ 1 ]
49+
}
50+
]
51+
52+
instance_group [
53+
{
54+
count: 1
55+
kind: KIND_CPU
56+
}
57+
]

‎qa/python_models/response_parameters_bls/model.py

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
import json
28+
29+
import numpy as np
30+
import triton_python_backend_utils as pb_utils
31+
32+
33+
class TritonPythonModel:
34+
"""
35+
This model (A) is designed to test sending back response parameters when using BLS.
36+
It takes one input tensor, which is the RESPONSE_PARAMETERS and uses BLS to
37+
call response_parameters model (B). Model B would set RESPONSE_PARAMETERS (with a bit
38+
of data massage) as its response parameters. In the end, model A would also set its
39+
response parameters from model B's response parameters.
40+
41+
With above model set up, we can easily test whether the real response parameters are
42+
the same as the input response parameters.
43+
"""
44+
45+
def execute(self, requests):
46+
responses = []
47+
48+
for request in requests:
49+
passed = True
50+
51+
# test bls response parameters from a regular model
52+
res_params_tensor = pb_utils.get_input_tensor_by_name(
53+
request, "RESPONSE_PARAMETERS"
54+
).as_numpy()
55+
res_params_str = str(res_params_tensor[0][0], encoding="utf-8")
56+
res_params = json.loads(res_params_str)
57+
bls_input_tensor = pb_utils.Tensor("RESPONSE_PARAMETERS", res_params_tensor)
58+
bls_req = pb_utils.InferenceRequest(
59+
model_name="response_parameters",
60+
inputs=[bls_input_tensor],
61+
requested_output_names=["OUTPUT"],
62+
)
63+
bls_res = bls_req.exec() # decoupled=False
64+
bls_res_params_str = bls_res.parameters()
65+
bls_res_params = (
66+
json.loads(bls_res_params_str) if bls_res_params_str != "" else {}
67+
)
68+
passed = passed and bls_res_params == res_params
69+
70+
# test bls response parameters from a decoupled model
71+
res_params_decoupled_tensor = pb_utils.get_input_tensor_by_name(
72+
request, "RESPONSE_PARAMETERS_DECOUPLED"
73+
).as_numpy()
74+
res_params_decoupled_str = str(
75+
res_params_decoupled_tensor[0][0], encoding="utf-8"
76+
)
77+
res_params_decoupled = json.loads(res_params_decoupled_str)
78+
bls_decoupled_input_tensor = pb_utils.Tensor(
79+
"RESPONSE_PARAMETERS", res_params_decoupled_tensor
80+
) # response_parameters_decoupled model input name is RESPONSE_PARAMETERS
81+
bls_decoupled_req = pb_utils.InferenceRequest(
82+
model_name="response_parameters_decoupled",
83+
inputs=[bls_decoupled_input_tensor],
84+
requested_output_names=["OUTPUT"],
85+
)
86+
bls_decoupled_res = bls_decoupled_req.exec(decoupled=True)
87+
for bls_decoupled_r in bls_decoupled_res:
88+
if len(bls_decoupled_r.output_tensors()) == 0:
89+
break # meaning reached final response
90+
bls_decoupled_r_params_str = bls_decoupled_r.parameters()
91+
bls_decoupled_r_params = (
92+
json.loads(bls_decoupled_r_params_str)
93+
if bls_decoupled_r_params_str != ""
94+
else {}
95+
)
96+
passed = passed and bls_decoupled_r_params in res_params_decoupled
97+
res_params_decoupled.remove(bls_decoupled_r_params)
98+
passed = passed and len(res_params_decoupled) == 0
99+
100+
output_tensor = pb_utils.Tensor(
101+
"OUTPUT", np.array([[str(passed)]], dtype=np.object_)
102+
)
103+
response = pb_utils.InferenceResponse(output_tensors=[output_tensor])
104+
responses.append(response)
105+
106+
return responses

0 commit comments

Comments
 (0)