feat(perf): add base for scenarios (#2690)

* feat(perf): add base for scenarios

* Update benchmarks/README.rst

Co-authored-by: Brett Langdon <[email protected]>

Co-authored-by: Brett Langdon <[email protected]>

Author: majorgreys

benchmarks/README.rst

@@ -4,12 +4,13 @@ Performance Testing
 New scenario
 ------------
 
-Copy ``sample`` directory and edit the ``meta.yaml`` file to specify the executable. Add any additional requirements to the ``requirements.txt``.
+Create a new directory for a scenario which includes a ``scenario.py`` script to be run. Add any additional dependencies in a ``requirements_scenario.txt``.
 
-Build image::
+First you need to build the scenario image::
 
-  docker build -t benchmark-<scenario> --build-arg BENCHMARK=<scenario> -f base.Dockerfile .
+  docker build -t perf-<scenario> --build-arg SCENARIO=<scenario> -f base/Dockerfile .
 
-Run scenario::
+You can now run the scenario image with two versions of the library. The environment variables ``DDTRACE_INSTALL_{V1,V2}`` can be set to a PEP 508 specification or a git url:
 
-  docker run -it --rm benchmark-<scenario>
+  docker run -it --rm -e DDTRACE_INSTALL_V1="ddtrace" -e DDTRACE_INSTALL_V2="ddtrace==0.50.0" perf-<scenario>
+  docker run -it --rm -e DDTRACE_INSTALL_V1=git+https://github.com/Datadog/dd-trace-py@master -e DDTRACE_INSTALL_V2=git+https://github.com/Datadog/[email protected] perf-<scenario>

benchmarks/base.Dockerfile renamed to benchmarks/base/Dockerfile

@@ -2,8 +2,7 @@ ARG PYTHON_VERSION=3.9-slim-buster
 
 FROM python:${PYTHON_VERSION} as python
 
-ARG BENCHMARK=sample
-ARG ARTIFACTS=/artifacts
+ARG SCENARIO=base
 
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
@@ -22,27 +21,22 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
-# sirun
-RUN curl -sL https://github.com/DataDog/sirun/releases/download/v0.1.8/sirun-v0.1.8-x86_64-unknown-linux-gnu.tar.gz | tar zxf - -C /tmp && mv /tmp/sirun /usr/bin
+# Add base common files used by all scenarios
+COPY ./base/ /app/
 
-COPY ./common/entrypoint /app/
-COPY ./common/benchmark /app/
+# Add scenario code, overriding anything from base
+COPY ./${SCENARIO}/ /app/
 
-# Add benchmark scenario code, overriding anything from common
-COPY ./${BENCHMARK}/ /app/
-
-# Create venv
+# Create venv for scenario requirements (other than ddtrace)
 ENV VIRTUAL_ENV=/app/.venv
 RUN python3 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
 RUN pip install -r requirements.txt
 
-# For performance testing
-ENV DDTRACE_GIT_COMMIT_ID ""
-ENV RUN_ID ""
-ENV PIP_INSTALL_WHEELS ""
-ENV SIRUN_NO_STDIO 0
+# Use separate venvs for the two versions of the library being compared
+ENV VENV_DDTRACE_V1=/app/.venv_ddtrace_v1/
+ENV VENV_DDTRACE_V2=/app/.venv_ddtrace_v2/
+ENV SCENARIO=${SCENARIO}
 
 ENTRYPOINT ["/app/entrypoint"]
 CMD ["/app/benchmark"]

benchmarks/base/benchmark

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -ex
+
+if [[ -z "${RUN_ID}" ]]; then
+    export RUN_ID=$(uuidgen)
+fi
+
+ARTIFACTS=/artifacts/${RUN_ID}/${SCENARIO}
+mkdir -p ${ARTIFACTS}
+RESULTS_V1=${ARTIFACTS}/${DDTRACE_V1}.json
+RESULTS_V2=${ARTIFACTS}/${DDTRACE_V2}.json
+
+# append venvs with ddtrace to sys.path
+
+PYTHONPATH=${VENV_DDTRACE_V1}/lib/python3.9/site-packages \
+    python scenario.py \
+    --copy-env \
+    --fast \
+    -o ${RESULTS_V1}
+
+PYTHONPATH=${VENV_DDTRACE_V2}/lib/python3.9/site-packages \
+    python scenario.py \
+    --copy-env \
+    --fast \
+    -o ${RESULTS_V2}
+
+pyperf compare_to --table ${RESULTS_V2} ${RESULTS_V1}

benchmarks/base/entrypoint

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -ex
+
+python3 -m venv ${VENV_DDTRACE_V1}
+source ${VENV_DDTRACE_V1}/bin/activate
+pip install ${DDTRACE_INSTALL_V1}
+export DDTRACE_V1=$(python -c "import ddtrace; print(ddtrace.__version__)")
+deactivate
+
+python3 -m venv ${VENV_DDTRACE_V2}
+source ${VENV_DDTRACE_V2}/bin/activate
+pip install ${DDTRACE_INSTALL_V2}
+export DDTRACE_V2=$(python -c "import ddtrace; print(ddtrace.__version__)")
+deactivate
+
+exec "$@"

benchmarks/base/requirements.txt

@@ -0,0 +1,3 @@
+-r requirements_scenario.txt
+
+pyperf