Merge pull request #23 from FelixNgFender/main

chore: temporal evals & perf benchmarks

Author: YiqinZhao

.vscode/settings.json

@@ -62,6 +62,8 @@
     "cli",
     "client",
     "examples",
+    "evaluations",
+    "benchmarks",
     "others"
   ]
 }

python/.gitignore

@@ -185,3 +185,7 @@ pyrightconfig.json
 # benchmark artifacts
 benchmarks/payload
 benchmarks/results
+benchmarks/scenarios/**/payload
+
+# evaluation artifacts
+!evaluations/**/*.rrd

python/Dockerfile

@@ -1,35 +1,15 @@
-# TODO: remove 
-# FROm python:3.12-slim-bookworm
-#
-# WORKDIR /app
-# COPY python_grpc_bench /app
-# COPY proto /app/proto
-#
-# RUN python -m pip install grpcio grpcio-tools
-# RUN python -m grpc_tools.protoc -I/app/proto/helloworld --python_out=. --grpc_python_out=. helloworld.proto
-#
-# ENTRYPOINT [ "python", "/app/server.py" ]
-
-# FROM python:3.12-alpine AS base
 FROM python:3.12-slim-bookworm AS base
 
 ENV VIRTUAL_ENV=/app/.venv \
   PATH="/app/.venv/bin:$PATH"
 
-# RUN apk update && \
-#   apk add libpq
-
-
 FROM base AS builder
 
 ENV POETRY_NO_INTERACTION=1 \
   POETRY_VIRTUALENVS_IN_PROJECT=1 \
   POETRY_VIRTUALENVS_CREATE=1 \
   POETRY_CACHE_DIR=/tmp/poetry_cache
 
-# RUN apk update && \
-#   apk add musl-dev build-base gcc gfortran openblas-dev
-
 WORKDIR /app
 
 RUN pip install --no-cache-dir poetry==1.8.4
@@ -52,5 +32,4 @@ EXPOSE 8500
 
 ENV PORT=8500
 
-# TODO: Vary entrypoint to include save mode
 ENTRYPOINT ["python", "-m", "arflow._cli", "view"]

python/benchmarks/BENCHMARKING.md

@@ -1,6 +1,12 @@
+# Benchmarking ARFlow
+
 Taken from <https://github.com/LesnyRumcajs/grpc_bench>. Instructions to run are
 in their README.
 
+Simple start:
+
 ```shell
-poetry run ./generate_payload.py
+./build.sh
+
+./bench.sh
 ```

python/benchmarks/analyze.sh

@@ -19,8 +19,9 @@ $(git log -1 --pretty="%h %cD %cn %s")
 - GRPC_SERVER_RAM=${GRPC_SERVER_RAM}
 - GRPC_CLIENT_CONNECTIONS=${GRPC_CLIENT_CONNECTIONS}
 - GRPC_CLIENT_CONCURRENCY=${GRPC_CLIENT_CONCURRENCY}
-- GRPC_CLIENT_QPS=${GRPC_CLIENT_QPS}
+- GRPC_CLIENT_RPS=${GRPC_CLIENT_RPS}
 - GRPC_CLIENT_CPUS=${GRPC_CLIENT_CPUS}
+- GRPC_CLIENT_FRAMES_PER_REQUEST=${GRPC_CLIENT_FRAMES_PER_REQUEST}
 - GRPC_REQUEST_SCENARIO=${GRPC_REQUEST_SCENARIO}
 - GRPC_GHZ_TAG=${GRPC_GHZ_TAG}
 EOF

python/benchmarks/archived_results/batching/heavy/250222T205146/arflow_server_bench.report

@@ -0,0 +1,35 @@
+
+Summary:
+  Count:	1199
+  Total:	60.01 s
+  Slowest:	51.18 ms
+  Fastest:	21.35 ms
+  Average:	31.08 ms
+  Requests/sec:	19.98
+
+Response time histogram:
+  21.352 [1]   |
+  24.336 [54]  |∎∎∎∎∎
+  27.319 [144] |∎∎∎∎∎∎∎∎∎∎∎∎∎
+  30.302 [280] |∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
+  33.285 [427] |∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
+  36.268 [196] |∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
+  39.251 [63]  |∎∎∎∎∎∎
+  42.234 [20]  |∎∎
+  45.217 [12]  |∎
+  48.200 [0]   |
+  51.183 [2]   |
+
+Latency distribution:
+  10 % in 25.61 ms 
+  25 % in 28.83 ms 
+  50 % in 31.02 ms 
+  75 % in 33.23 ms 
+  90 % in 35.79 ms 
+  95 % in 37.62 ms 
+  99 % in 42.88 ms 
+
+Status code distribution:
+  [OK]   1199 responses   
+
+

python/benchmarks/archived_results/batching/heavy/250222T205146/arflow_server_bench.stats

@@ -0,0 +1,9 @@
+93.57%    333.5MiB / 4GiB
+82.20%    328.7MiB / 4GiB
+82.58%    302MiB / 4GiB
+79.53%    327MiB / 4GiB
+79.22%    332.3MiB / 4GiB
+78.96%    329.4MiB / 4GiB
+82.16%    309.2MiB / 4GiB
+79.62%    329.1MiB / 4GiB
+77.09%    333.5MiB / 4GiB

python/benchmarks/archived_results/batching/heavy/250222T205146/bench.params

@@ -0,0 +1,13 @@
+Benchmark Execution Parameters:
+a896bbc Sat, 22 Feb 2025 19:16:24 -0500 felixngfender test(server): add color modality to saving AR frames test
+- GRPC_BENCHMARK_DURATION=60s
+- GRPC_BENCHMARK_WARMUP=10s
+- GRPC_SERVER_CPUS=1
+- GRPC_SERVER_RAM=4096m
+- GRPC_CLIENT_CONNECTIONS=5
+- GRPC_CLIENT_CONCURRENCY=5
+- GRPC_CLIENT_RPS=20
+- GRPC_CLIENT_CPUS=5
+- GRPC_CLIENT_FRAMES_PER_REQUEST=15
+- GRPC_REQUEST_SCENARIO=heavy
+- GRPC_GHZ_TAG=0.114.0

python/benchmarks/archived_results/batching/heavy/250222T205311/arflow_server_bench.report

@@ -0,0 +1,42 @@
+
+Summary:
+  Count:	10602
+  Total:	60.00 s
+  Slowest:	77.30 ms
+  Fastest:	4.40 ms
+  Average:	28.08 ms
+  Requests/sec:	176.70
+
+Response time histogram:
+  4.396  [1]    |
+  11.687 [1581] |∎∎∎∎∎∎∎∎∎∎∎
+  18.977 [5870] |∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
+  26.267 [149]  |∎
+  33.557 [1]    |
+  40.847 [0]    |
+  48.138 [0]    |
+  55.428 [2]    |
+  62.718 [737]  |∎∎∎∎∎
+  70.008 [2135] |∎∎∎∎∎∎∎∎∎∎∎∎∎∎∎
+  77.298 [121]  |∎
+
+Latency distribution:
+  10 % in 11.12 ms 
+  25 % in 12.60 ms 
+  50 % in 14.71 ms 
+  75 % in 61.37 ms 
+  90 % in 65.74 ms 
+  95 % in 67.43 ms 
+  99 % in 70.23 ms 
+
+Status code distribution:
+  [Unavailable]   5 responses       
+  [OK]            10597 responses   
+
+Error distribution:
+  [1]   rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:58822->127.0.0.1:50051: use of closed network connection   
+  [1]   rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:58838->127.0.0.1:50051: use of closed network connection   
+  [1]   rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:58830->127.0.0.1:50051: use of closed network connection   
+  [1]   rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:58808->127.0.0.1:50051: use of closed network connection   
+  [1]   rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:58814->127.0.0.1:50051: use of closed network connection   
+

python/benchmarks/archived_results/batching/heavy/250222T205311/arflow_server_bench.stats

@@ -0,0 +1,9 @@
+104.69%   104.4MiB / 4GiB
+103.08%   105.9MiB / 4GiB
+102.06%   103.5MiB / 4GiB
+102.64%   103.7MiB / 4GiB
+102.48%   100.8MiB / 4GiB
+101.71%   107MiB / 4GiB
+101.63%   102.9MiB / 4GiB
+104.59%   101.4MiB / 4GiB
+103.40%   105.4MiB / 4GiB