ci: add telemetry source tracking for build environment identification (#18653)

Add DATABEND_TELEMETRY_SOURCE environment variable to distinguish
different build sources in telemetry data:

- pr-test: PR and development testing
- production: Production binary releases
- docker-release: Docker image builds
- ci-main: Main branch CI builds

This enables better telemetry data analysis by identifying the
build context without affecting core functionality.

- Add DATABEND_TELEMETRY_SOURCE support in build system
- Update CI workflows to set appropriate source values
- Include source in telemetry observability data
- Upgrade telemetry schema version to 1.1

Author: BohuTANG

.github/workflows/cloud.yml

@@ -70,6 +70,7 @@ jobs:
           DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
           DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}}
           DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}}
+          DATABEND_TELEMETRY_SOURCE: "pr-test"
         with:
           sha: ${{ needs.info.outputs.sha }}
           target: ${{ matrix.arch }}-unknown-linux-gnu

.github/workflows/main.yml

@@ -47,6 +47,7 @@ jobs:
       runner_provider: aws
       runner_arch: ARM64
       license_type: enterprise
+      telemetry_source: "ci-main"
 
   # macos:
   #   needs: linux

.github/workflows/merge_group.yml

@@ -57,6 +57,7 @@ jobs:
       build_profile: ci
       runner_provider: aws
       license_type: enterprise
+      telemetry_source: "pr-test"
 
   ready:
     if: always()

.github/workflows/release.yml

@@ -142,6 +142,7 @@ jobs:
           DATABEND_ENTERPRISE_LICENSE_EMBEDDED: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_RELEASE }}
           DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}}
           DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}}
+          DATABEND_TELEMETRY_SOURCE: "production"
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}
@@ -183,6 +184,7 @@ jobs:
           DATABEND_ENTERPRISE_LICENSE_EMBEDDED: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_RELEASE }}
           DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}}
           DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}}
+          DATABEND_TELEMETRY_SOURCE: "docker-release"
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}

.github/workflows/reuse.linux.yml

@@ -23,6 +23,11 @@ on:
         type: string
         required: true
         default: "trial"
+      telemetry_source:
+        description: "Telemetry source identifier"
+        type: string
+        required: false
+        default: "pr-test"
 
 env:
   BUILD_PROFILE: ${{ inputs.build_profile }}
@@ -83,6 +88,7 @@ jobs:
           DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
           DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}}
           DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}}
+          DATABEND_TELEMETRY_SOURCE: ${{ inputs.telemetry_source }}
         with:
           sha: ${{ github.sha }}
           target: ${{ needs.info.outputs.target }}

src/common/building/src/lib.rs

@@ -124,6 +124,10 @@ pub fn add_env_telemetry() {
     println!("cargo:rerun-if-env-changed=DATABEND_TELEMETRY_API_KEY");
     let api_key = env::var("DATABEND_TELEMETRY_API_KEY").unwrap_or_default();
     println!("cargo:rustc-env=DATABEND_TELEMETRY_API_KEY={api_key}");
+
+    println!("cargo:rerun-if-env-changed=DATABEND_TELEMETRY_SOURCE");
+    let source = env::var("DATABEND_TELEMETRY_SOURCE").unwrap_or_default();
+    println!("cargo:rustc-env=DATABEND_TELEMETRY_SOURCE={source}");
 }
 
 pub fn add_env_commit_authors(repo: &Repository) {

src/common/version/src/lib.rs

@@ -49,6 +49,8 @@ pub const DATABEND_TELEMETRY_ENDPOINT: &str = env!("DATABEND_TELEMETRY_ENDPOINT"
 
 pub const DATABEND_TELEMETRY_API_KEY: &str = env!("DATABEND_TELEMETRY_API_KEY");
 
+pub const DATABEND_TELEMETRY_SOURCE: &str = env!("DATABEND_TELEMETRY_SOURCE");
+
 pub static DATABEND_SEMVER: LazyLock<Version> = LazyLock::new(|| {
     let semver = DATABEND_GIT_SEMVER
         .strip_prefix('v')

src/query/service/src/clusters/cluster.rs

@@ -58,6 +58,7 @@ use databend_common_settings::Settings;
 use databend_common_telemetry::report_node_telemetry;
 use databend_common_version::DATABEND_TELEMETRY_API_KEY;
 use databend_common_version::DATABEND_TELEMETRY_ENDPOINT;
+use databend_common_version::DATABEND_TELEMETRY_SOURCE;
 use databend_enterprise_resources_management::ResourcesManagement;
 use futures::future::select;
 use futures::future::Either;
@@ -832,7 +833,8 @@ impl ClusterDiscovery {
                     .duration_since(std::time::UNIX_EPOCH)
                     .map(|d| d.as_secs())
                     .unwrap_or(0),
-                "schema_version": "1.0"
+                "source": DATABEND_TELEMETRY_SOURCE,
+                "schema_version": "1.1"
             }
         });
 

tests/nox/java_client/perpare.py

@@ -15,7 +15,6 @@ def main():
 
 
 def download_jdbc(version):
-
     files = [f"databend-jdbc-{version}.jar", f"databend-jdbc-{version}-tests.jar"]
 
     for filename in files:

tests/nox/noxfile.py

@@ -7,7 +7,6 @@
 @nox.session
 @nox.parametrize("driver_version", PYTHON_DRIVER)
 def python_client(session, driver_version):
-
     session.install("pytest")
     session.install(f"databend-driver=={driver_version}")
     session.run("pytest", "python_client")
@@ -19,7 +18,6 @@ def python_client(session, driver_version):
 @nox.session
 @nox.parametrize("driver_version", JDBC_DRIVER)
 def java_client(session, driver_version):
-
     session.install("requests")
     session.run("python", "java_client/perpare.py", driver_version)
     run_jdbc_test(session, driver_version)