fix(lib-injection): support non-root users [backport 1.17] (#6434)

Backport b048030 from #6047 to 1.17.

The current mechanism attempts a `pip install` using the current user.
Many users run their containers with a non-root user (`USER 1000` or
similar in their Dockerfiles) for security purposes. The injection
mechanism doesn't work with this as it requires modifying the
site-packages in the application container to install the ddtrace
package which most often requires super user permissions.

A solution is to go back to providing an installation of the package
directly via the volume which does not require and runtime writing to
system directories. This solution was attempted before but was flawed in
that it tried to provide all the binaries in a single directory. This
was broken because `ddtrace` has varying dependency versions across
Python versions which resulted in conflicts. With this approach the
installation directories are split by Python version and the C-library
runtime. The `sitecustomize.py` file is updated to select the
appropriate installation directory at runtime.

Several new test cases are added to cover this regression.

A reproduction of the issue is provided with
[49203e6](49203e6).
The simple django test image is modified to reproduce the issue and
assert that the problem is fixed.


## Risks

Clashing dependencies. There is no dependency clash checking
implemented. This means that if the user has a version of a dependency
that ddtrace does, then the ddtrace version could take precedence
(depending on the PYTHONPATH). This can be mitigated in the future by
doing a run-time check on the installed versions.



## Checklist

- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/contributing.html#Release-Note-Guidelines)
are followed.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).

## Reviewer Checklist

- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.

Co-authored-by: Kyle Verhoog <[email protected]>

Author: github-actions[bot]

.github/workflows/lib-injection.yml

@@ -9,7 +9,7 @@ jobs:
     secrets:
       token: ${{ secrets.GITHUB_TOKEN }}
     with:
-      ddtrace-version: v1.10
+      ddtrace-version: v1.16.1
       image-tag: ${{ github.sha }}
 
   test:
@@ -51,56 +51,32 @@ jobs:
         variant: [
           'dd-lib-python-init-test-django',
           'dd-lib-python-init-test-django-gunicorn',
+          'dd-lib-python-init-test-django-gunicorn-alpine',
           'dd-lib-python-init-test-django-uvicorn',
+          'dd-lib-python-init-test-django-no-perms',
+          'dd-lib-python-init-test-django-pre-installed',
         ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
-      - name: Create a docker network for the app and test agent
-        run: |
-          docker network create test-inject
-      - name: Run the test agent
-        run: |
-          docker run \
-            -d \
-            --name=testagent \
-            --network=test-inject \
-            -p 8126:8126 \
-            ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.7.2
-      - name: Prepare the volume
+      - name: Build and run the app
         run: |
+          SRC="$(pwd)"
           cd lib-injection
-          mkdir -p lib-injection/ddtrace_pkgs
-          cp sitecustomize.py lib-injection/
-          ./dl_wheels.py \
-            --ddtrace-version=v1.10 \
-            --python-version=3.11 \
-            --python-version=3.10 \
-            --python-version=3.9 \
-            --python-version=3.8 \
-            --python-version=3.7 \
-            --arch x86_64 \
-            --platform manylinux2014 \
-            --output-dir ddtrace_pkgs \
-            --verbose
-      - name: Build test app
-        run: |
-          docker build \
-            -t ${{ matrix.variant }} \
-            tests/lib-injection/${{ matrix.variant }}/
-      - name: Run the app
-        run: |
-          docker run -d \
-            --name ${{ matrix.variant }} \
-            --network test-inject \
-            -p 18080:18080 \
-            -e PYTHONPATH=/lib-injection \
-            -e DD_TRACE_AGENT_URL=http://testagent:8126 \
-            -v $PWD/lib-injection:/lib-injection \
-            ${{matrix.variant}}
+          export DDTRACE_PYTHON_VERSION="v1.16.1"
+          export APP_CONTEXT="${SRC}/tests/lib-injection/${{matrix.variant}}"
+          export TEMP_DIR="${SRC}/tmp/ddtrace"
+          mkdir -p "${TEMP_DIR}"
+          # Give the temp dir permissions, by default the docker user doesn't have permissions
+          # to write to the filesystem.
+          chmod 777 $TEMP_DIR
+          # Start the lib_inject to get the files copied. This avoids a race condition with the startup of the
+          # application.
+          docker-compose up --build lib_inject
+          docker-compose up --build -d
           # Wait for the app to start
-          sleep 20
-          docker logs ${{matrix.variant}}
+          sleep 60
+          docker-compose logs
       - name: Test the app
         run: |
           curl http://localhost:18080
@@ -111,3 +87,7 @@ jobs:
         run: |
           N=$(curl http://localhost:8126/test/traces | jq -r -e 'length')
           [[ $N == "1" ]]
+      - name: Output app logs (LOOK HERE IF THE JOB FAILS)
+        if: success() || failure()
+        run: |
+          docker-compose logs

.gitlab/build-deb-rpm.sh

@@ -8,7 +8,6 @@ echo -n $PYTHON_PACKAGE_VERSION > auto_inject-python.version
 
 source common_build_functions.sh
 
-# Download the megawheel
 mkdir -p dd-trace.dir/lib
 
 # Install known compatible pip as default version shipped in Ubuntu (20.0.2)
@@ -29,7 +28,7 @@ echo `pwd`
     --arch aarch64 \
     --platform musllinux_1_1 \
     --platform manylinux2014 \
-    --output-dir dd-trace.dir/lib \
+    --output-dir dd-trace.dir/lib/ddtrace_pkgs \
     --verbose
 
 cp ../lib-injection/sitecustomize.py dd-trace.dir/lib/

lib-injection/README.md

@@ -14,7 +14,9 @@ the necessary files to a given directory.
 
 The `dl_wheels.py` script provides a portable set of `ddtrace` wheels. It is
 responsible for downloading the published wheels of `ddtrace` and its
-dependencies.
+dependencies. It installs the wheels to a set of per Python, per platform site-packages
+directories. These directories can be directly added to the `PYTHONPATH` to use the
+`ddtrace` package.
 
 The Datadog Admission Controller injects the InitContainer with a new volume
 mount to the application deployment. The script to copy files out of the
@@ -24,12 +26,26 @@ volume mount.
 
 The files copied to the volume are:
 
-- `sitecustomize.py`: Python module that gets run automatically on interpreter startup when it is detected in the `PYTHONPATH`. When executed, it updates the Python path further to include the `ddtrace_pkgs/` directory and then calls `import ddtrace.bootstrap.sitecustomize` which performs automatic instrumentation.
-- `ddtrace_pkgs/`: Directory containing the `ddtrace` package and its dependencies for each Python version, platform and architecture.
+- `sitecustomize.py`: Python module that gets run automatically on interpreter startup when it is detected in the `PYTHONPATH`. When executed, it updates the Python path further to include the compatible `site-packages` directory and then calls `import ddtrace.bootstrap.sitecustomize` which performs automatic instrumentation.
+- `ddtrace_pkgs/`: Directory containing the per-Python, per-runtime `site-packages` directories which contain `ddtrace` package and its dependencies.
 
 
 The `PYTHONPATH` environment variable is set to the shared volume directory
 which contains `sitecustomize.py` and `ddtrace_pkgs`. The environment variable
-is injected into the the application container. This enables the
+is injected into the application container. This enables the
 `sitecustomize.py` file to execute on any Python interpreter startup which
-results in the automatic instrument being applied to the application.
+results in the automatic instrumentation being applied to the application.
+
+
+## Testing
+
+To test this feature locally use the provided `docker-compose.yml`.
+
+```bash
+export DDTRACE_PYTHON_VERSION=v1.16.1
+export APP_CONTEXT=$REPO_ROOT/tests/lib-injection/dd-lib-python-init-test-django
+export TEMP_DIR="/tmp/ddtrace"
+rm -rf $TEMP_DIR && docker-compose up --build lib_inject && docker-compose up --build
+```
+
+Note that the `lib_inject` step is separate to ensure the files are copied to the volume before the app starts up.

lib-injection/dl_wheels.py

@@ -2,25 +2,23 @@
 """
 Script to download all required wheels (including dependencies) of the ddtrace
 Python package for relevant Python versions (+ abis), C library platforms and
-architectures and merge them into a "megawheel" directory.
+architectures and unpack them into Python-specific site-packages directories.
 
-This directory provides a portable installation of ddtrace which can be used
-on multiple platforms and architectures.
+These site-package directories provide a portable installation of ddtrace which can be
+used on multiple platforms and architectures.
 
-Currently the only OS supported is Linux.
+Currently, the only OS supported is Linux.
 
-This script has been tested with 21.0.0 and is confirmed to not work with
+This script has been tested with pip 21.0.0 and is confirmed to not work with
 20.0.2.
 
 Usage:
         ./dl_wheels.py --help
 
-
-The downloaded wheels can then be installed locally using:
-        pip install --no-index --find-links <dir_of_downloaded_wheels> ddtrace
 """
 import argparse
 import itertools
+import os
 import subprocess
 import sys
 
@@ -73,34 +71,54 @@
 dl_dir = args.output_dir
 print("saving wheels to %s" % dl_dir)
 
-for python_version, arch, platform in itertools.product(args.python_version, args.arch, args.platform):
-    print("Downloading %s %s %s wheel" % (python_version, arch, platform))
-    abi = "cp%s" % python_version.replace(".", "")
-    # Have to special-case these versions of Python for some reason.
-    if python_version in ["2.7", "3.5", "3.6", "3.7"]:
-        abi += "m"
-
-    # See the docs for an explanation of all the options used:
-    # https://pip.pypa.io/en/stable/cli/pip_download/
-    #   only-binary=:all: is specified to ensure we get all the dependencies of ddtrace as well.
-    cmd = [
-        sys.executable,
-        "-m",
-        "pip",
-        "download",
-        "ddtrace==%s" % args.ddtrace_version,
-        "--platform",
-        "%s_%s" % (platform, arch),
-        "--python-version",
-        python_version,
-        "--abi",
-        abi,
-        "--only-binary=:all:",
-        "--dest",
-        dl_dir,
-    ]
-    if args.verbose:
-        print(" ".join(cmd))
-
-    if not args.dry_run:
-        subprocess.run(cmd, capture_output=not args.verbose, check=True)
+
+for python_version, platform in itertools.product(args.python_version, args.platform):
+    for arch in args.arch:
+        print("Downloading %s %s %s wheel" % (python_version, arch, platform))
+        abi = "cp%s" % python_version.replace(".", "")
+        # Have to special-case these versions of Python for some reason.
+        if python_version in ["2.7", "3.5", "3.6", "3.7"]:
+            abi += "m"
+
+        # See the docs for an explanation of all the options used:
+        # https://pip.pypa.io/en/stable/cli/pip_download/
+        #   only-binary=:all: is specified to ensure we get all the dependencies of ddtrace as well.
+        cmd = [
+            sys.executable,
+            "-m",
+            "pip",
+            "download",
+            "ddtrace==%s" % args.ddtrace_version,
+            "--platform",
+            "%s_%s" % (platform, arch),
+            "--python-version",
+            python_version,
+            "--abi",
+            abi,
+            "--only-binary=:all:",
+            "--dest",
+            dl_dir,
+        ]
+        if args.verbose:
+            print(" ".join(cmd))
+
+        if not args.dry_run:
+            subprocess.run(cmd, capture_output=not args.verbose, check=True)
+
+    wheel_files = [f for f in os.listdir(dl_dir) if f.endswith(".whl")]
+    for whl in wheel_files:
+        wheel_file = os.path.join(dl_dir, whl)
+        print("Unpacking %s" % wheel_file)
+        # -q for quieter output, else we get all the files being unzipped.
+        subprocess.run(
+            [
+                "unzip",
+                "-q",
+                "-o",
+                wheel_file,
+                "-d",
+                os.path.join(dl_dir, "site-packages-ddtrace-py%s-%s" % (python_version, platform)),
+            ]
+        )
+        # Remove the wheel as it has been unpacked
+        os.remove(wheel_file)

lib-injection/docker-compose.yml

@@ -0,0 +1,50 @@
+# Emulate what the Cluster Agent does as closely as possible.
+# The Cluster Agent creates an InitContainer with the init image and runs the copy-lib.sh script.
+# It then patches the pods to include the PYTHONPATH environment variable and the volume mount.
+version: "3"
+
+services:
+  lib_inject:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - DDTRACE_PYTHON_VERSION=${DDTRACE_PYTHON_VERSION}
+    command: sh copy-lib.sh /datadog-lib
+    volumes:
+      # A host mount is used rather than named volumes as they run into permission issues when copying files.
+      # The injection image is run with a non-root user which does not have permission to write to the named volume.
+      - ${TEMP_DIR:-/tmp/ddtrace_test}:/datadog-lib
+
+  # testagent is used to collect data from the library to validate.
+  testagent:
+    image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
+    ports:
+      - "8126:8126"
+
+  # app is parametrized to generically run images with the library injected and submit data to the test agent.
+  app:
+    depends_on:
+      - lib_inject
+    image: ${APP_IMAGE:-python:3.10}
+    environment:
+      - PYTHONPATH=/datadog-lib
+      - DD_TRACE_AGENT_URL=http://testagent:8126
+    volumes:
+      - ${TEMP_DIR:-/tmp/ddtrace_test}:/datadog-lib
+
+  # same as app but a local docker file can be used.
+  app_local:
+    depends_on:
+      - lib_inject
+    build:
+      context: ${APP_CONTEXT}
+    ports:
+      - "0.0.0.0:18080:18080"
+    environment:
+      - PYTHONPATH=/datadog-lib
+      - DD_TRACE_AGENT_URL=http://testagent:8126
+      - DD_TRACE_DEBUG=1
+      - DD_CALL_BASIC_CONFIG=1
+    volumes:
+      - ${TEMP_DIR:-/tmp/ddtrace_test}:/datadog-lib