Skip to content

Commit 5f8e9a5

Browse files
tybalextybalex
committed
support streaming
1 parent 24a85a3 commit 5f8e9a5

File tree

3 files changed

+76
-18
lines changed

3 files changed

+76
-18
lines changed

‎examples/server/function-call-parser.hpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ json clean_json_strings(const std::string& input_str) {
9696

9797
std::vector<json> rubra_fc_json_tool_extractor(const std::string& output_str) {
9898
std::vector<json> result;
99-
std::cout << "Output to Parse : " << output_str.c_str() << std::endl;
10099
if (output_str.find("endtoolcall") == std::string::npos) {
101100
return result;
102101
}

‎examples/server/server.cpp‎

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1585,6 +1585,66 @@ struct server_context {
15851585
}
15861586
}
15871587

1588+
void receive_cmpl_results_stream_handlefc(
1589+
const std::unordered_set<int> & id_tasks, const
1590+
json& data, const
1591+
std::function<bool(server_task_result&)> & result_handler, const
1592+
std::function<void(json)> & error_handler) {
1593+
size_t n_finished = 0;
1594+
std::string last_str = "";
1595+
bool is_function_call = false;
1596+
bool checked_function_call = false;
1597+
json last_result_data;
1598+
while (true) {
1599+
server_task_result result = queue_results.recv(id_tasks);
1600+
if (data.contains("tool_field")) {
1601+
result.data["tool_field"] = data["tool_field"];
1602+
}
1603+
std::string content = result.data.value("content", "");
1604+
if (!is_function_call) {
1605+
std::string str_to_check = last_str + content;
1606+
is_function_call = (str_to_check.find("starttool") != std::string::npos);
1607+
}
1608+
if (!is_function_call && !last_str.empty()) {
1609+
std::string temp_str = content;
1610+
result.data["content"] = last_str;
1611+
last_str = temp_str;
1612+
if (!result_handler(result)) {
1613+
cancel_tasks(id_tasks);
1614+
break;
1615+
}
1616+
} else {
1617+
last_str += content;
1618+
}
1619+
1620+
if (result.error) {
1621+
error_handler(result.data);
1622+
cancel_tasks(id_tasks);
1623+
break;
1624+
}
1625+
1626+
if (result.stop) {
1627+
if (!last_str.empty()) {
1628+
last_result_data["content"] = last_str;
1629+
server_task_result temp_result = result;
1630+
temp_result.data = last_result_data;
1631+
if (!result_handler(temp_result)) {
1632+
cancel_tasks(id_tasks);
1633+
break;
1634+
}
1635+
}
1636+
if (!result_handler(result)) {
1637+
cancel_tasks(id_tasks);
1638+
break;
1639+
}
1640+
if (++n_finished == id_tasks.size()) {
1641+
break;
1642+
}
1643+
}
1644+
last_result_data = result.data;
1645+
}
1646+
}
1647+
15881648
//
15891649
// Functions to process the task
15901650
//
@@ -3019,8 +3079,8 @@ int main(int argc, char ** argv) {
30193079

30203080
ctx_server.queue_results.remove_waiting_task_ids(task_ids);
30213081
} else {
3022-
const auto chunked_content_provider = [task_ids, &ctx_server, completion_id](size_t, httplib::DataSink & sink) {
3023-
ctx_server.receive_cmpl_results_stream(task_ids, [&](const server_task_result & result) -> bool {
3082+
const auto chunked_content_provider = [task_ids, &ctx_server, completion_id, data](size_t, httplib::DataSink & sink) {
3083+
ctx_server.receive_cmpl_results_stream_handlefc(task_ids, data, [&](const server_task_result & result) -> bool {
30243084
std::vector<json> result_array = format_partial_response_oaicompat(result.data, completion_id);
30253085
for (auto & event_data : result_array) {
30263086
if (event_data.empty()) {

‎test_llamacpp_streaming.ipynb‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
},
1010
{
1111
"cell_type": "code",
12-
"execution_count": 1,
12+
"execution_count": 37,
1313
"metadata": {},
1414
"outputs": [],
1515
"source": [
@@ -424,7 +424,7 @@
424424
},
425425
{
426426
"cell_type": "code",
427-
"execution_count": 2,
427+
"execution_count": 38,
428428
"metadata": {},
429429
"outputs": [],
430430
"source": [
@@ -444,7 +444,7 @@
444444
},
445445
{
446446
"cell_type": "code",
447-
"execution_count": 3,
447+
"execution_count": 39,
448448
"metadata": {},
449449
"outputs": [
450450
{
@@ -471,7 +471,7 @@
471471
},
472472
{
473473
"cell_type": "code",
474-
"execution_count": 4,
474+
"execution_count": 40,
475475
"metadata": {},
476476
"outputs": [
477477
{
@@ -515,7 +515,7 @@
515515
},
516516
{
517517
"cell_type": "code",
518-
"execution_count": 5,
518+
"execution_count": 41,
519519
"metadata": {},
520520
"outputs": [
521521
{
@@ -559,7 +559,7 @@
559559
},
560560
{
561561
"cell_type": "code",
562-
"execution_count": 6,
562+
"execution_count": 42,
563563
"metadata": {},
564564
"outputs": [
565565
{
@@ -576,11 +576,11 @@
576576
"[AI response]:\n",
577577
"[AI calling functions]:\n",
578578
"Tool Call: ChoiceDeltaToolCallFunction(arguments='{\"length\":8}', name='generate_password')\n",
579-
"Observation: Password generated: 0cc31a6f\n",
579+
"Observation: Password generated: be676e08\n",
580580
"\n",
581581
"<Turn 2>:\n",
582582
"[AI response]:\n",
583-
"Your order for 3 umbrellas has been placed, and the price is $10 per umbrella. A random password of length 8 has also been generated: 0cc31a6f."
583+
"Your order for 3 umbrellas has been placed, and the price is $10 each. Additionally, a password 'be676e08' of length 8 has been generated for you."
584584
]
585585
}
586586
],
@@ -598,7 +598,7 @@
598598
},
599599
{
600600
"cell_type": "code",
601-
"execution_count": 7,
601+
"execution_count": 43,
602602
"metadata": {},
603603
"outputs": [
604604
{
@@ -651,7 +651,7 @@
651651
},
652652
{
653653
"cell_type": "code",
654-
"execution_count": 8,
654+
"execution_count": 44,
655655
"metadata": {},
656656
"outputs": [
657657
{
@@ -684,7 +684,7 @@
684684
},
685685
{
686686
"cell_type": "code",
687-
"execution_count": 9,
687+
"execution_count": 45,
688688
"metadata": {},
689689
"outputs": [
690690
{
@@ -718,7 +718,7 @@
718718
},
719719
{
720720
"cell_type": "code",
721-
"execution_count": 10,
721+
"execution_count": 46,
722722
"metadata": {},
723723
"outputs": [
724724
{
@@ -757,8 +757,7 @@
757757
"The sizes of the files in the 'documents' directory are as follows:\n",
758758
"- 'report.docx' is 100 bytes.\n",
759759
"- 'task.txt' is 100 bytes.\n",
760-
"- 'notes.txt' is 100 bytes.\n",
761-
"Is there anything else you would like to know or another task you need assistance with?"
760+
"- 'notes.txt' is 100 bytes."
762761
]
763762
}
764763
],
@@ -769,7 +768,7 @@
769768
},
770769
{
771770
"cell_type": "code",
772-
"execution_count": 13,
771+
"execution_count": 47,
773772
"metadata": {},
774773
"outputs": [
775774
{

0 commit comments

Comments
 (0)