Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 2361815

Browse files
sangjanaisangjanai
committed
fix: comments
1 parent ff17419 commit 2361815

File tree

2 files changed

+47
-67
lines changed

2 files changed

+47
-67
lines changed

‎engine/controllers/server.cc‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,15 @@ void server::ChatCompletion(
2828
LOG_DEBUG << "Start chat completion";
2929
auto json_body = req->getJsonObject();
3030
bool is_stream = (*json_body).get("stream", false).asBool();
31-
std::string model_id = (*json_body).get("model", "invalid_model").asString();
32-
std::string engine_type;
33-
if (!inference_svc_->HasFieldInReq(json_body, "engine")) {
34-
engine_type = kLlamaRepo;
35-
} else {
36-
engine_type = (*(json_body)).get("engine", kLlamaRepo).asString();
37-
}
31+
auto model_id = (*json_body).get("model", "invalid_model").asString();
32+
auto engine_type = [this, &json_body]() -> std::string {
33+
if (!inference_svc_->HasFieldInReq(json_body, "engine")) {
34+
return kLlamaRepo;
35+
} else {
36+
return (*(json_body)).get("engine", kLlamaRepo).asString();
37+
}
38+
}();
39+
3840
LOG_DEBUG << "request body: " << json_body->toStyledString();
3941
auto q = std::make_shared<services::SyncQueue>();
4042
auto ir = inference_svc_->HandleChatCompletion(q, json_body);

‎engine/services/inference_service.cc‎

Lines changed: 38 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -24,24 +24,18 @@ cpp::result<void, InferResult> InferenceService::HandleChatCompletion(
2424
return cpp::fail(std::make_pair(stt, res));
2525
}
2626

27+
auto cb = [q, tool_choice](Json::Value status, Json::Value res) {
28+
if (!tool_choice.isNull()) {
29+
res["tool_choice"] = tool_choice;
30+
}
31+
q->push(std::make_pair(status, res));
32+
};
2733
if (std::holds_alternative<EngineI*>(engine_result.value())) {
2834
std::get<EngineI*>(engine_result.value())
29-
->HandleChatCompletion(
30-
json_body, [q, tool_choice](Json::Value status, Json::Value res) {
31-
if (!tool_choice.isNull()) {
32-
res["tool_choice"] = tool_choice;
33-
}
34-
q->push(std::make_pair(status, res));
35-
});
35+
->HandleChatCompletion(json_body, std::move(cb));
3636
} else {
3737
std::get<RemoteEngineI*>(engine_result.value())
38-
->HandleChatCompletion(
39-
json_body, [q, tool_choice](Json::Value status, Json::Value res) {
40-
if (!tool_choice.isNull()) {
41-
res["tool_choice"] = tool_choice;
42-
}
43-
q->push(std::make_pair(status, res));
44-
});
38+
->HandleChatCompletion(json_body, std::move(cb));
4539
}
4640

4741
return {};
@@ -66,16 +60,15 @@ cpp::result<void, InferResult> InferenceService::HandleEmbedding(
6660
return cpp::fail(std::make_pair(stt, res));
6761
}
6862

63+
auto cb = [q](Json::Value status, Json::Value res) {
64+
q->push(std::make_pair(status, res));
65+
};
6966
if (std::holds_alternative<EngineI*>(engine_result.value())) {
7067
std::get<EngineI*>(engine_result.value())
71-
->HandleEmbedding(json_body, [q](Json::Value status, Json::Value res) {
72-
q->push(std::make_pair(status, res));
73-
});
68+
->HandleEmbedding(json_body, std::move(cb));
7469
} else {
7570
std::get<RemoteEngineI*>(engine_result.value())
76-
->HandleEmbedding(json_body, [q](Json::Value status, Json::Value res) {
77-
q->push(std::make_pair(status, res));
78-
});
71+
->HandleEmbedding(json_body, std::move(cb));
7972
}
8073
return {};
8174
}
@@ -104,18 +97,16 @@ InferResult InferenceService::LoadModel(
10497
// might need mutex here
10598
auto engine_result = engine_service_->GetLoadedEngine(engine_type);
10699

100+
auto cb = [&stt, &r](Json::Value status, Json::Value res) {
101+
stt = status;
102+
r = res;
103+
};
107104
if (std::holds_alternative<EngineI*>(engine_result.value())) {
108105
std::get<EngineI*>(engine_result.value())
109-
->LoadModel(json_body, [&stt, &r](Json::Value status, Json::Value res) {
110-
stt = status;
111-
r = res;
112-
});
106+
->LoadModel(json_body, std::move(cb));
113107
} else {
114108
std::get<RemoteEngineI*>(engine_result.value())
115-
->LoadModel(json_body, [&stt, &r](Json::Value status, Json::Value res) {
116-
stt = status;
117-
r = res;
118-
});
109+
->LoadModel(json_body, std::move(cb));
119110
}
120111
return std::make_pair(stt, r);
121112
}
@@ -139,20 +130,16 @@ InferResult InferenceService::UnloadModel(const std::string& engine_name,
139130
json_body["model"] = model_id;
140131

141132
LOG_TRACE << "Start unload model";
133+
auto cb = [&r, &stt](Json::Value status, Json::Value res) {
134+
stt = status;
135+
r = res;
136+
};
142137
if (std::holds_alternative<EngineI*>(engine_result.value())) {
143138
std::get<EngineI*>(engine_result.value())
144-
->UnloadModel(std::make_shared<Json::Value>(json_body),
145-
[&r, &stt](Json::Value status, Json::Value res) {
146-
stt = status;
147-
r = res;
148-
});
139+
->UnloadModel(std::make_shared<Json::Value>(json_body), std::move(cb));
149140
} else {
150141
std::get<RemoteEngineI*>(engine_result.value())
151-
->UnloadModel(std::make_shared<Json::Value>(json_body),
152-
[&r, &stt](Json::Value status, Json::Value res) {
153-
stt = status;
154-
r = res;
155-
});
142+
->UnloadModel(std::make_shared<Json::Value>(json_body), std::move(cb));
156143
}
157144

158145
return std::make_pair(stt, r);
@@ -181,20 +168,16 @@ InferResult InferenceService::GetModelStatus(
181168

182169
LOG_TRACE << "Start to get model status";
183170

171+
auto cb = [&stt, &r](Json::Value status, Json::Value res) {
172+
stt = status;
173+
r = res;
174+
};
184175
if (std::holds_alternative<EngineI*>(engine_result.value())) {
185176
std::get<EngineI*>(engine_result.value())
186-
->GetModelStatus(json_body,
187-
[&stt, &r](Json::Value status, Json::Value res) {
188-
stt = status;
189-
r = res;
190-
});
177+
->GetModelStatus(json_body, std::move(cb));
191178
} else {
192179
std::get<RemoteEngineI*>(engine_result.value())
193-
->GetModelStatus(json_body,
194-
[&stt, &r](Json::Value status, Json::Value res) {
195-
stt = status;
196-
r = res;
197-
});
180+
->GetModelStatus(json_body, std::move(cb));
198181
}
199182

200183
return std::make_pair(stt, r);
@@ -214,25 +197,20 @@ InferResult InferenceService::GetModels(
214197

215198
LOG_TRACE << "Start to get models";
216199
Json::Value resp_data(Json::arrayValue);
200+
auto cb = [&resp_data](Json::Value status, Json::Value res) {
201+
for (auto r : res["data"]) {
202+
resp_data.append(r);
203+
}
204+
};
217205
for (const auto& loaded_engine : loaded_engines) {
218206
if (std::holds_alternative<EngineI*>(loaded_engine)) {
219207
auto e = std::get<EngineI*>(loaded_engine);
220208
if (e->IsSupported("GetModels")) {
221-
e->GetModels(json_body,
222-
[&resp_data](Json::Value status, Json::Value res) {
223-
for (auto r : res["data"]) {
224-
resp_data.append(r);
225-
}
226-
});
209+
e->GetModels(json_body, std::move(cb));
227210
}
228211
} else {
229212
std::get<RemoteEngineI*>(loaded_engine)
230-
->GetModels(json_body,
231-
[&resp_data](Json::Value status, Json::Value res) {
232-
for (auto r : res["data"]) {
233-
resp_data.append(r);
234-
}
235-
});
213+
->GetModels(json_body, std::move(cb));
236214
}
237215
}
238216

0 commit comments

Comments
 (0)