Load test

apps/api/src/api/fake_request_router.py

@@ -0,0 +1,17 @@
+import asyncio
+import json
+from collections.abc import AsyncGenerator
+
+from fastapi import APIRouter, Query
+from fastapi.responses import StreamingResponse
+
+fake_request_router = APIRouter(prefix="/fake-request")
+
+async def generate_chunks(num: int, delay: float) -> AsyncGenerator[str, None]:
+    for i in range(num):
+        await asyncio.sleep(delay / 1000)
+        yield json.dumps({"chunk": i}) + "\n"
+
+@fake_request_router.get("")
+async def fake_response(num: int = Query(default=10), delay: float = Query(default=100.0)) -> StreamingResponse:
+    return StreamingResponse(generate_chunks(num=num, delay=delay), media_type="application/jsonl")

apps/api/src/api/thread/chat/pydantic_inference/backends/pydantic_ai_test.py

@@ -1,6 +1,17 @@
-from pydantic_ai.models import Model
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from pydantic_ai.models import Model, StreamedResponse
 from pydantic_ai.models.test import TestModel
 
 
+class SleepTestModel(TestModel):
+    @asynccontextmanager
+    async def request_stream(self, *args: object, **kwargs: object) -> AsyncIterator[StreamedResponse]:
+        await asyncio.sleep(1)
+        async with super().request_stream(*args, **kwargs) as stream:  # type: ignore[arg-type]
+            yield stream
+
 def get_test_model() -> Model:
-    return TestModel()
+    return SleepTestModel()

apps/api/src/api/v5.py

@@ -2,6 +2,7 @@
 
 from api.attribution.attribution_router import attribution_router
 from api.event import event_router
+from api.fake_request_router import fake_request_router
 from api.message.message_router import message_router
 from api.model.model_router import model_router
 from api.model_config.admin.model_config_admin_router import model_config_admin_router
@@ -12,6 +13,8 @@
 
 v5_router = APIRouter(prefix="/v5")
 
+# testing
+v5_router.include_router(fake_request_router)
 
 # public routes
 v5_router.include_router(event_router)

apps/flask-api/dev.sh

@@ -2,7 +2,7 @@
 pwd
 exec \
     gunicorn \
-    --workers 1 \
+    --workers 9 \
     --timeout 0 \
     --bind 0.0.0.0:8000 \
     --enable-stdio-inheritance \

apps/flask-api/pyproject.toml

@@ -49,7 +49,7 @@ dependencies = [
 
 [dependency-groups]
 dev = [
-    "pytest==8.3.2",
+    "pytest>=8.3.3",
     "pytest-mock==3.15.1",
     "pytest-postgresql==7.0.2",
     "time-machine==2.16.0",

apps/flask-api/src/fake_request/fake_request_blueprint.py

@@ -0,0 +1,25 @@
+import json
+import time
+from collections.abc import Generator
+
+from flask import Blueprint, Response, request, stream_with_context
+from src.flask_pydantic_api.api_wrapper import pydantic_api
+
+
+def generate_chunks(num: int, delay: float) -> Generator[str, None, None]:
+    for i in range(num):
+        time.sleep(delay / 1000)
+        yield json.dumps({"chunk": i}) + "\n"
+
+def create_fake_request_blueprint() -> Blueprint:
+    fake_request_blueprint = Blueprint("fake-request", __name__)
+
+    @fake_request_blueprint.get("")
+    @pydantic_api(name="fake-request")
+    def fake_response() -> Response:
+        num = int(request.args.get("num", 10))
+        delay = float(request.args.get("delay", 100.0))
+
+        return Response(stream_with_context(generate_chunks(num, delay)), mimetype="application/jsonl")
+
+    return fake_request_blueprint

apps/flask-api/src/message/create_message_service/endpoint.py

@@ -178,15 +178,15 @@ def stream_message_from_model(
 
     new_message_id = create_message_id()
 
-    validate_message_security_and_safety(
-        request=mapped_request,
-        client_auth=client_auth,
-        checker_type=checker_type,
-        user_ip_address=user_ip_address,
-        user_agent=user_agent,
-        storage_client=storage_client,
-        message_id=new_message_id,
-    )
+    # validate_message_security_and_safety(
+    #     request=mapped_request,
+    #     client_auth=client_auth,
+    #     checker_type=checker_type,
+    #     user_ip_address=user_ip_address,
+    #     user_agent=user_agent,
+    #     storage_client=storage_client,
+    #     message_id=new_message_id,
+    # )
 
     return create_new_message(
         mapped_request,

apps/flask-api/src/pydantic_inference/backends/pydantic_ai_test.py

@@ -1,6 +1,9 @@
+import time
+
 from pydantic_ai.models import Model
 from pydantic_ai.models.test import TestModel
 
 
 def get_test_model() -> Model:
+    time.sleep(1)
     return TestModel()

apps/flask-api/src/v4.py

@@ -4,6 +4,7 @@
 from src import db
 from src.admin.admin_blueprint import create_admin_blueprint
 from src.agent.agent_blueprint import create_agents_blueprint
+from src.fake_request.fake_request_blueprint import create_fake_request_blueprint
 from src.message.GoogleCloudStorage import GoogleCloudStorage
 from src.model_config.model_config_blueprint import create_model_config_blueprint
 from src.prompt_template.prompt_template_blueprint import create_prompt_template_blueprint
@@ -14,6 +15,9 @@
 def create_v4_blueprint(dbc: db.Client, storage_client: GoogleCloudStorage, session_maker: sessionmaker[Session]):
     v4_blueprint = Blueprint(name="v4", import_name=__name__)
 
+    # testing
+    v4_blueprint.register_blueprint(create_fake_request_blueprint(), url_prefix="/fake-request", name="fake-request")
+
     v4_blueprint.register_blueprint(create_model_config_blueprint(session_maker), url_prefix="/models", name="models")
     v4_blueprint.register_blueprint(
         create_admin_blueprint(session_maker),

apps/load_test/.python-version

@@ -0,0 +1 @@
+3.14

apps/load_test/cli_csv/fastapi_1s_delay_u200_r10_t60_exceptions.csv

@@ -0,0 +1 @@
+Count,Message,Traceback,Nodes

apps/load_test/cli_csv/fastapi_1s_delay_u200_r10_t60_failures.csv

@@ -0,0 +1 @@
+Method,Name,Error,Occurrences

apps/load_test/cli_csv/fastapi_1s_delay_u200_r10_t60_stats.csv

@@ -0,0 +1,4 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%
+POST,/v5/threads/chat,8424,0,100.0,136.4929881366607,4.094459000043571,696.1732920026407,0.0,140.612691173591,0.0,100,150,190,220,310,380,430,520,680,700,700
+POST,/v5/threads/chat (TTFT),8424,0,1000.0972920097411,1044.79000043121,1000.0972920097411,1490.5228749848902,0.0,140.612691173591,0.0,1000,1000,1100,1100,1100,1100,1200,1200,1400,1500,1500
+,Aggregated,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0,281.225382347182,0.0,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500

apps/load_test/cli_csv/fastapi_1s_delay_u200_r10_t60_stats_history.csv

@@ -0,0 +1,72 @@
+Timestamp,User Count,Type,Name,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%,Total Request Count,Total Failure Count,Total Median Response Time,Total Average Response Time,Total Min Response Time,Total Max Response Time,Total Average Content Size
+1772213437,0,,Aggregated,0.000000,0.000000,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,0,0,0,0.0,0,0,0
+1772213438,10,,Aggregated,0.000000,0.000000,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,0,0,0,0.0,0,0,0
+1772213439,20,,Aggregated,0.000000,0.000000,1000,1000,1000,1000,1200,1200,1200,1200,1200,1200,1200,20,0,340.0,670.7036396503099,122.50399999902584,1214.639542013174,0.0
+1772213440,30,,Aggregated,0.000000,0.000000,1000,1000,1000,1000,1000,1200,1200,1200,1200,1200,1200,60,0,340.0,606.5508313006527,33.469459012849256,1214.639542013174,0.0
+1772213441,40,,Aggregated,0.000000,0.000000,1000,1000,1000,1000,1000,1000,1200,1200,1200,1200,1200,120,0,340.0,567.1795468665854,13.889624999137595,1214.639542013174,0.0
+1772213442,50,,Aggregated,10.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,1200,200,0,340.0,550.1767908445618,5.058124981587753,1214.639542013174,0.0
+1772213443,60,,Aggregated,20.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,1200,300,0,340.0,539.5896466926206,4.8100840067490935,1214.639542013174,0.0
+1772213444,70,,Aggregated,30.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,420,0,340.0,536.0681714424919,4.613458993844688,1214.639542013174,0.0
+1772213445,80,,Aggregated,40.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,560,0,340.0,532.7699008404514,4.613458993844688,1214.639542013174,0.0
+1772213446,90,,Aggregated,50.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,720,0,340.0,531.5520794047188,4.592334007611498,1214.639542013174,0.0
+1772213447,100,,Aggregated,70.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,900,0,340.0,530.7004623669652,4.592334007611498,1214.639542013174,0.0
+1772213448,110,,Aggregated,80.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,1100,0,340.0,530.2511975431645,4.592334007611498,1214.639542013174,0.0
+1772213449,120,,Aggregated,88.400000,0.000000,1000,1000,1000,1000,1000,1000,1000,1100,1200,1200,1200,1320,0,340.0,532.8993547042377,4.592334007611498,1214.639542013174,0.0
+1772213450,130,,Aggregated,108.200000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,1560,0,340.0,533.8408555828769,4.321541986428201,1214.639542013174,0.0
+1772213451,140,,Aggregated,128.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,1820,0,340.0,534.0450789613917,4.321541986428201,1214.639542013174,0.0
+1772213452,150,,Aggregated,144.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,2100,0,340.0,533.944835596,4.321541986428201,1214.639542013174,0.0
+1772213453,160,,Aggregated,163.600000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,2400,0,340.0,533.9468668374078,4.321541986428201,1214.639542013174,0.0
+1772213454,170,,Aggregated,182.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,2692,0,340.0,532.8417837099199,4.321541986428201,1214.639542013174,0.0
+1772213455,180,,Aggregated,198.600000,0.000000,1000,1000,1000,1000,1000,1000,1000,1000,1200,1200,1200,3030,0,340.0,532.2991196518548,4.321541986428201,1214.639542013174,0.0
+1772213456,190,,Aggregated,217.600000,0.000000,1000,1000,1000,1000,1000,1000,1000,1100,1200,1200,1200,3366,0,340.0,532.8757048903608,4.321541986428201,1214.639542013174,0.0
+1772213457,200,,Aggregated,236.000000,0.000000,1000,1000,1000,1000,1000,1000,1000,1100,1200,1200,1200,3702,0,340.0,535.5602198013239,4.321541986428201,1219.598333002068,0.0
+1772213458,200,,Aggregated,249.400000,0.000000,1000,1000,1000,1000,1000,1000,1100,1100,1200,1200,1200,4058,0,340.0,538.8068460641464,4.321541986428201,1219.598333002068,0.0
+1772213459,200,,Aggregated,268.200000,0.000000,1000,1000,1000,1000,1000,1000,1100,1100,1200,1200,1200,4428,0,340.0,543.2210793094139,4.321541986428201,1219.598333002068,0.0
+1772213460,200,,Aggregated,288.400000,0.000000,1000,1000,1000,1000,1000,1000,1100,1100,1200,1200,1200,4682,0,340.0,543.4043658030201,4.321541986428201,1219.598333002068,0.0
+1772213461,200,,Aggregated,297.000000,0.000000,1000,1000,1000,1000,1000,1100,1100,1100,1200,1200,1200,5026,0,430.0,554.9568519979681,4.321541986428201,1219.598333002068,0.0
+1772213462,200,,Aggregated,314.000000,0.000000,1000,1000,1000,1000,1000,1100,1100,1100,1200,1200,1200,5330,0,430.0,557.5502113849327,4.321541986428201,1219.598333002068,0.0
+1772213463,200,,Aggregated,312.600000,0.000000,1000,1000,1000,1000,1000,1100,1100,1100,1200,1200,1200,5668,0,430.0,562.9823354579968,4.321541986428201,1219.598333002068,0.0
+1772213464,200,,Aggregated,318.600000,0.000000,1000,1000,1000,1000,1000,1100,1100,1100,1200,1200,1200,5964,0,430.0,561.7082809701825,4.094459000043571,1247.7302499755751,0.0
+1772213465,200,,Aggregated,325.400000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1200,1300,1300,6298,0,430.0,563.03687426547,4.094459000043571,1255.8666669938248,0.0
+1772213466,200,,Aggregated,333.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,6694,0,430.0,567.7144835964954,4.094459000043571,1266.8643330107443,0.0
+1772213467,200,,Aggregated,332.200000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,7026,0,430.0,567.7671973598043,4.094459000043571,1266.8643330107443,0.0
+1772213468,200,,Aggregated,333.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,7398,0,430.0,566.5098064600495,4.094459000043571,1266.8643330107443,0.0
+1772213469,200,,Aggregated,332.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,7768,0,430.0,567.053696824991,4.094459000043571,1266.8643330107443,0.0
+1772213470,200,,Aggregated,330.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,8094,0,430.0,566.473939078047,4.094459000043571,1266.8643330107443,0.0
+1772213471,200,,Aggregated,335.400000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,8434,0,430.0,566.4161835892648,4.094459000043571,1266.8643330107443,0.0
+1772213472,200,,Aggregated,333.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,8752,0,430.0,568.2891524678904,4.094459000043571,1266.8643330107443,0.0
+1772213473,200,,Aggregated,347.400000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,9112,0,430.0,568.8298865264817,4.094459000043571,1342.800000013085,0.0
+1772213474,200,,Aggregated,343.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1300,1300,1300,9396,0,480.0,573.2416838269463,4.094459000043571,1342.800000013085,0.0
+1772213475,200,,Aggregated,347.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,9724,0,480.0,573.7312979134206,4.094459000043571,1342.800000013085,0.0
+1772213476,200,,Aggregated,330.000000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,10114,0,480.0,574.067401689511,4.094459000043571,1342.800000013085,0.0
+1772213477,200,,Aggregated,336.400000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,10448,0,480.0,573.9638736298626,4.094459000043571,1342.800000013085,0.0
+1772213478,200,,Aggregated,342.000000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,10780,0,480.0,573.5771411198767,4.094459000043571,1342.800000013085,0.0
+1772213479,200,,Aggregated,340.200000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,11148,0,480.0,573.9996147309377,4.094459000043571,1342.800000013085,0.0
+1772213480,200,,Aggregated,339.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,11520,0,480.0,572.5287967378147,4.094459000043571,1342.800000013085,0.0
+1772213481,200,,Aggregated,338.400000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1300,1300,1300,11880,0,480.0,572.4371208202587,4.094459000043571,1342.800000013085,0.0
+1772213482,200,,Aggregated,341.200000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1400,1400,12184,0,480.0,572.6954921185968,4.094459000043571,1379.0982500067912,0.0
+1772213483,200,,Aggregated,335.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,12404,0,700.0,576.3628520780164,4.094459000043571,1490.5228749848902,0.0
+1772213484,200,,Aggregated,346.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,12712,0,700.0,579.2223717094564,4.094459000043571,1490.5228749848902,0.0
+1772213485,200,,Aggregated,322.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,13096,0,700.0,581.0579117761533,4.094459000043571,1490.5228749848902,0.0
+1772213486,200,,Aggregated,340.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,13376,0,700.0,580.6003047835076,4.094459000043571,1490.5228749848902,0.0
+1772213487,200,,Aggregated,331.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,13710,0,700.0,582.6057089405689,4.094459000043571,1490.5228749848902,0.0
+1772213488,200,,Aggregated,331.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,14046,0,700.0,582.8416168777694,4.094459000043571,1490.5228749848902,0.0
+1772213489,200,,Aggregated,328.200000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,14350,0,700.0,583.7080640917831,4.094459000043571,1490.5228749848902,0.0
+1772213490,200,,Aggregated,325.400000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,14646,0,700.0,584.6529592446273,4.094459000043571,1490.5228749848902,0.0
+1772213491,200,,Aggregated,322.600000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,14944,0,700.0,586.6863726033082,4.094459000043571,1490.5228749848902,0.0
+1772213492,200,,Aggregated,317.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1400,1500,1500,15344,0,700.0,586.8834169226869,4.094459000043571,1490.5228749848902,0.0
+1772213493,200,,Aggregated,310.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1100,1400,1500,1500,15616,0,700.0,586.7467804989578,4.094459000043571,1490.5228749848902,0.0
+1772213494,200,,Aggregated,310.200000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,15960,0,700.0,589.4979090657564,4.094459000043571,1490.5228749848902,0.0
+1772213495,200,,Aggregated,328.400000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16214,0,700.0,589.4992713876018,4.094459000043571,1490.5228749848902,0.0
+1772213496,200,,Aggregated,315.000000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16586,0,700.0,590.5738292984171,4.094459000043571,1490.5228749848902,0.0
+1772213497,200,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213498,200,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213499,200,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213500,200,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213501,200,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213502,200,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213503,200,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213504,196,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213505,191,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213506,156,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0
+1772213507,79,,Aggregated,316.800000,0.000000,1000,1000,1000,1000,1100,1100,1100,1200,1400,1500,1500,16848,0,700.0,590.6414942839378,4.094459000043571,1490.5228749848902,0.0

apps/load_test/cli_csv/fastapi_1s_delay_u50_r10_t10_exceptions.csv

@@ -0,0 +1 @@
+Count,Message,Traceback,Nodes

apps/load_test/cli_csv/fastapi_1s_delay_u50_r10_t10_failures.csv

@@ -0,0 +1 @@
+Method,Name,Error,Occurrences

apps/load_test/cli_csv/fastapi_1s_delay_u50_r10_t10_stats.csv

@@ -0,0 +1,4 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%
+POST,/v5/threads/chat,350,0,59,59.65363394319346,5.000375007512048,186.04520900407806,0.0,36.17967111097156,0.0,59,71,77,81,97,110,170,180,190,190,190
+POST,/v5/threads/chat (TTFT),350,0,1001.62429100601,1020.2213440299965,1001.62429100601,1133.2984170003328,0.0,36.17967111097156,0.0,1000,1000,1000,1000,1000,1000,1100,1100,1100,1100,1100
+,Aggregated,700,0,190.0,539.937488986595,5.000375007512048,1133.2984170003328,0.0,72.35934222194312,0.0,1000,1000,1000,1000,1000,1000,1000,1100,1100,1100,1100