Merge remote-tracking branch 'origin/master' into glopes/sidecar-ffi-tel-metrics

Author: cataphract

.github/workflows/prof_asan.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         php-version: [8.3, 8.4]
     container:
-      image: datadog/dd-trace-ci:php-${{matrix.php-version}}_bookworm-5
+      image: datadog/dd-trace-ci:php-${{matrix.php-version}}_bookworm-6
       # https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
       options: --user root --privileged
 

.github/workflows/prof_correctness.yml

@@ -102,6 +102,10 @@ jobs:
           export DD_PROFILING_OUTPUT_PPROF=$PWD/profiling/tests/correctness/allocations_1byte/test.pprof
           export DD_PROFILING_ALLOCATION_SAMPLING_DISTANCE=1
           php -d extension=$PWD/target/profiler-release/libdatadog_php_profiling.so profiling/tests/correctness/allocations.php
+          mkdir -p profiling/tests/correctness/allocations_1byte_no_zend_alloc/
+          export DD_PROFILING_OUTPUT_PPROF=$PWD/profiling/tests/correctness/allocations_1byte_no_zend_alloc/test.pprof
+          export DD_PROFILING_ALLOCATION_SAMPLING_DISTANCE=1
+          USE_ZEND_ALLOC=0 php -d extension=$PWD/target/profiler-release/libdatadog_php_profiling.so profiling/tests/correctness/allocations.php
           unset DD_PROFILING_ALLOCATION_SAMPLING_DISTANCE
 
       - name: Run ZTS tests
@@ -131,6 +135,12 @@ jobs:
           expected_json: profiling/tests/correctness/allocations.json
           pprof_path: profiling/tests/correctness/allocations_1byte/
 
+      - name: Check profiler correctness for allocations with 1 byte sampling distance and `USE_ZEND_ALLOC=0`
+        uses: Datadog/prof-correctness/analyze@main
+        with:
+          expected_json: profiling/tests/correctness/allocations.json
+          pprof_path: profiling/tests/correctness/allocations_1byte_no_zend_alloc/
+
       - name: Check profiler correctness for time
         uses: Datadog/prof-correctness/analyze@main
         with:

.github/workflows/prune-stale.yaml

@@ -0,0 +1,24 @@
+name: Prune
+on:
+  schedule:
+    - cron: '0 3 1 * *'
+  workflow_dispatch:
+
+jobs:
+  prune:
+    name: Prune
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+    - name: Prune
+      uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0
+      with:
+        days-before-stale: 180
+        stale-pr-message: 'This pull request has been marked as stale because it has not had activity in the past half year. It will be closed in 7 days if no further activity occurs. Feel free to reopen it if you are still working on it.'
+        close-pr-message: 'This pull request has been closed because it has not had activity over the past half year. Feel free to reopen it if you are still working on it.'
+        stale-issue-message: 'This issue has been marked as stale because it has not had activity in the past half year. It will be closed in 7 days if no further activity occurs. Feel free to reopen it if you are still working on it.'
+        close-issue-message: 'This issue has been closed because it has not had activity over the past half year. Feel free to reopen it if you are still working on it.'
+        stale-pr-label: 'tag: stale'
+        operations-per-run: 50

.gitlab/collect_artifacts.sh

@@ -6,3 +6,5 @@ mkdir -p "${CI_PROJECT_DIR}/artifacts/core_dumps"
 find . -type f -name "core*" -exec head -c 4 "{}" \; -exec echo " {}" \;  | grep -a ^.ELF | cut -d' ' -f2 | xargs -I % -n 1 cp % "${CI_PROJECT_DIR}/artifacts/core_dumps" || true
 mkdir -p "${CI_PROJECT_DIR}/artifacts/diffs"
 find . -type f -name '*.diff' -not -path "*/vendor/*" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/diffs" \; || true
+mkdir -p "${CI_PROJECT_DIR}/artifacts/tests"
+find . -type f -name '*.xml' -path "*/artifacts/tests/*" -exec cp '{}' "${CI_PROJECT_DIR}/artifacts/tests/" \; || true

.gitlab/dockerhub-login.sh

@@ -0,0 +1,87 @@
+#!/bin/sh
+
+set -e
+
+export VAULT_VERSION="1.20.0"
+
+echo "=== Setting up Docker Hub authentication ==="
+
+# Determine architecture for binary downloads
+arch="$(uname -m)"
+case "${arch}" in
+  x86_64)
+    vault_arch="amd64"
+    ;;
+  aarch64|arm64)
+    vault_arch="arm64"
+    ;;
+  *)
+    echo "Warning: Unsupported architecture: ${arch}. Skipping Docker Hub authentication." >&2
+    exit 0
+    ;;
+esac
+
+# Install jq if not already available
+if ! command -v jq > /dev/null 2>&1; then
+  echo "Installing jq..."
+
+  jq_path="/tmp/jq"
+
+  if ! curl -L --fail "https://github.com/jqlang/jq/releases/latest/download/jq-linux-${vault_arch}" \
+      --output "${jq_path}"; then
+    echo "Warning: Failed to download jq. Skipping Docker Hub authentication." >&2
+    exit 0
+  fi
+
+  chmod +x "${jq_path}"
+  export PATH="/tmp:${PATH}"
+fi
+
+# Install Vault if not already available
+vault_cmd="vault"
+if ! command -v vault > /dev/null 2>&1; then
+  echo "Installing Vault CLI..."
+
+  vault_path="/tmp/vault"
+  vault_zip="${vault_path}.zip"
+
+  if ! curl -L --fail "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_${vault_arch}.zip" \
+      --output "${vault_zip}"; then
+    echo "Warning: Failed to download Vault. Skipping Docker Hub authentication." >&2
+    exit 0
+  fi
+
+  if ! unzip -q "${vault_zip}" -d /tmp; then
+    echo "Warning: Failed to extract Vault. Skipping Docker Hub authentication." >&2
+    exit 0
+  fi
+
+  chmod +x "${vault_path}"
+  rm -f "${vault_zip}"
+
+  vault_cmd="${vault_path}"
+fi
+
+# Fetch Docker Hub credentials from Vault
+echo "Fetching Docker Hub credentials from Vault..."
+vaultoutput="$("${vault_cmd}" kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/dockerhub)" || {
+  echo "Warning: Failed to fetch Docker Hub credentials from Vault. Skipping Docker Hub authentication." >&2
+  exit 0
+}
+
+user="$(echo "$vaultoutput" | jq -r '.data.data.user')"
+token="$(echo "$vaultoutput" | jq -r '.data.data.token')"
+
+if [ -z "${user}" ] || [ -z "${token}" ] || [ "${user}" = "null" ] || [ "${token}" = "null" ]; then
+  echo "Warning: Docker Hub credentials are empty or invalid. Skipping Docker Hub authentication." >&2
+  exit 0
+fi
+
+echo "Docker Hub user: ${user}"
+echo "Logging in to Docker Hub..."
+if ! echo "${token}" | docker login -u "${user}" --password-stdin docker.io; then
+  echo "Warning: Failed to login to Docker Hub. Continuing without authentication." >&2
+  exit 0
+fi
+
+echo "=== Docker Hub authentication successful ==="

.gitlab/generate-appsec.php

@@ -62,26 +62,13 @@
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
   before_script:
 <?php echo $ecrLoginSnippet, "\n"; ?>
-    - |
-      echo "Logging in to Docker Hub"
-      if [ "$CI_REGISTRY_USER" = "" ]; then
-        echo "Fetching Docker Hub credentials from vault"
-        vaultoutput=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/dockerhub)
-        user=$(echo "$vaultoutput" | jq -r .data.data.user)
-        token=$(echo "$vaultoutput" | jq -r .data.data.token)
-      else
-        user="$CI_REGISTRY_USER"
-        token="$CI_REGISTRY_TOKEN"
-      fi
-
-      echo "Docker Hub user: $user"
-      docker login -u "$user" -p "$token" docker.io
-    - apt update && apt install -y default-jre
+<?php dockerhub_login() ?>
+    - apt update && apt install -y openjdk-17-jre
 
 "test appsec extension":
   stage: test
   extends: .appsec_test
-  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-${PHP_MAJOR_MINOR}_bookworm-5
+  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-${PHP_MAJOR_MINOR}_bookworm-6
   variables:
     KUBERNETES_CPU_REQUEST: 3
     KUBERNETES_MEMORY_REQUEST: 3Gi
@@ -141,8 +128,9 @@
           - test8.5-release-zts
   before_script:
 <?php echo $ecrLoginSnippet, "\n"; ?>
+<?php dockerhub_login() ?>
   script:
-    - apt update && apt install -y default-jre
+    - apt update && apt install -y openjdk-17-jre
     - find "$CI_PROJECT_DIR"/appsec/tests/integration/build || true
     - |
       cd appsec/tests/integration
@@ -154,6 +142,16 @@
 
       TERM=dumb ./gradlew $targets --info -Pbuildscan --scan
       TERM=dumb ./gradlew saveCaches --info
+  after_script:
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts"
+    - find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \;
+    - .gitlab/upload-junit-to-datadog.sh "test.source.file:appsec"
+  artifacts:
+    reports:
+      junit: "artifacts/**/test-results/**/TEST-*.xml"
+    paths:
+      - "artifacts/"
+    when: "always"
   cache:
     - key: "appsec int test cache"
       paths:
@@ -162,7 +160,7 @@
 "appsec code coverage":
   stage: test
   extends: .appsec_test
-  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-8.3_bookworm-5
+  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-8.3_bookworm-6
   variables:
     KUBERNETES_CPU_REQUEST: 3
     KUBERNETES_MEMORY_REQUEST: 3Gi
@@ -265,7 +263,7 @@
 "appsec lint":
   stage: test
   extends: .appsec_test
-  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-8.3_bookworm-5
+  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-8.3_bookworm-6
   variables:
     KUBERNETES_CPU_REQUEST: 3
     KUBERNETES_MEMORY_REQUEST: 9Gi
@@ -287,7 +285,7 @@
 "test appsec helper asan":
   stage: test
   extends: .appsec_test
-  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-5
+  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-6
   variables:
     KUBERNETES_CPU_REQUEST: 3
     KUBERNETES_MEMORY_REQUEST: 3Gi
@@ -313,7 +311,7 @@
 #"fuzz appsec helper":
 #  stage: test
 #  extends: .appsec_test
-#  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-5
+#  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-6
 #  variables:
 #    KUBERNETES_CPU_REQUEST: 3
 #    KUBERNETES_MEMORY_REQUEST: 5Gi
@@ -358,3 +356,52 @@
 #  artifacts:
 #    paths:
 #     - appsec/fuzzer-coverage.html
+
+"check libxml2 version":
+  stage: test
+  image: registry.ddbuild.io/images/mirror/python:3.12-slim
+  tags: [ "arch:amd64" ]
+  needs: []
+  allow_failure: true
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  script:
+    - |
+      python3 - <<'EOF'
+      import urllib.request
+      import json
+      import re
+      import sys
+
+      # Read local version
+      with open("appsec/third_party/libxml2/VERSION") as f:
+          local_version = f.read().strip()
+      print(f"Local libxml2 version: {local_version}")
+
+      # Fetch latest version from GNOME GitLab
+      url = "https://gitlab.gnome.org/api/v4/projects/GNOME%2Flibxml2/repository/tags?per_page=100&order_by=updated&sort=desc"
+      with urllib.request.urlopen(url) as response:
+          tags = json.load(response)
+
+      # Extract version numbers and find the latest
+      versions = []
+      for tag in tags:
+          match = re.match(r"v(\d+\.\d+\.\d+)$", tag["name"])
+          if match:
+              versions.append(match.group(1))
+
+      # Sort by version number
+      versions.sort(key=lambda v: tuple(map(int, v.split("."))))
+      latest_version = versions[-1] if versions else None
+
+      print(f"Latest libxml2 version: {latest_version}")
+
+      if local_version != latest_version:
+          print("ERROR: libxml2 version mismatch!")
+          print(f"Local version:  {local_version}")
+          print(f"Latest version: {latest_version}")
+          print("Please update appsec/third_party/libxml2 to the latest version.")
+          sys.exit(1)
+
+      print("libxml2 version is up to date.")
+      EOF

.gitlab/generate-common.php

@@ -31,7 +31,6 @@
 
 function unset_dd_runner_env_vars() {
 ?>
-
     # DD env vars auto-added to GitLab runners for infra purposes
     - unset DD_SERVICE
     - unset DD_ENV
@@ -40,6 +39,12 @@ function unset_dd_runner_env_vars() {
 <?php
 }
 
+function dockerhub_login() {
+?>
+    - if command -v docker > /dev/null 2>&1; then .gitlab/dockerhub-login.sh; fi
+<?php
+}
+
 ?>
 default:
   retry:
@@ -50,7 +55,6 @@ function unset_dd_runner_env_vars() {
       - runner_system_failure
       - scheduler_failure
       - api_failure
-      - script_failure
       - stuck_or_timeout_failure
       - job_execution_timeout
 
@@ -139,10 +143,27 @@ function unset_dd_runner_env_vars() {
       ZOOKEEPER_TICK_TIME: 2000
       ALLOW_ANONYMOUS_LOGIN: "yes"
       ZOOKEEPER_ADMIN_ENABLE_SERVER: "false"
+      KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=srvr,ruok"
 
   kafka:
     name: registry.ddbuild.io/images/mirror/confluentinc/cp-kafka:7.8.0
     alias: kafka-integration
+    entrypoint: ["/bin/bash"]
+    command:
+      - -c
+      - |
+        # Wait for Zookeeper to be ready before starting Kafka
+        echo "Waiting for Zookeeper to be ready..."
+        for i in $(seq 1 30); do
+          if echo "ruok" | nc zookeeper 2181 2>/dev/null | grep -q "imok"; then
+            echo "Zookeeper is ready, starting Kafka..."
+            break
+          fi
+          echo "Waiting for Zookeeper... attempt $i/30"
+          sleep 2
+        done
+        # Start Kafka with original entrypoint
+        exec /etc/confluent/docker/run
     variables:
       KAFKA_BROKER_ID: 111
       KAFKA_CREATE_TOPICS: test-lowlevel:1:1,test-highlevel:1:1