chore: try telemetry (#18526)

* init

* ci: add env

* telemetry more

* fix endpoint

* Update Cargo.toml

Author: BohuTANG

.github/actions/build_linux/action.yml

@@ -30,7 +30,7 @@ runs:
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}
-        bypass_env_vars: RUSTFLAGS,RUST_LOG,DATABEND_RELEASE_VERSION,DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY,DATABEND_ENTERPRISE_LICENSE_EMBEDDED
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,DATABEND_RELEASE_VERSION,DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY,DATABEND_ENTERPRISE_LICENSE_EMBEDDED,DATABEND_TELEMETRY_ENDPOINT,DATABEND_TELEMETRY_API_KEY
 
     - name: Cross setup
       if: startsWith(inputs.target, 'aarch64-')

.github/workflows/cloud.yml

@@ -67,6 +67,8 @@ jobs:
         timeout-minutes: 60
         env:
           DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
+          DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}}
+          DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}}
         with:
           sha: ${{ needs.info.outputs.sha }}
           target: ${{ matrix.arch }}-unknown-linux-gnu
@@ -75,7 +77,7 @@ jobs:
           features: python-udf
 
   docker:
-    needs: [info, build]
+    needs: [ info, build ]
     timeout-minutes: 10
     runs-on:
       - self-hosted
@@ -161,7 +163,7 @@ jobs:
 
   benchmark:
     if: needs.info.outputs.target
-    needs: [info, build, docker]
+    needs: [ info, build, docker ]
     uses: ./.github/workflows/reuse.benchmark.yml
     secrets: inherit
     with:

.github/workflows/release.yml

@@ -140,6 +140,8 @@ jobs:
           DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }}
           DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
           DATABEND_ENTERPRISE_LICENSE_EMBEDDED: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_RELEASE }}
+          DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}}
+          DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}}
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}
@@ -179,6 +181,8 @@ jobs:
           DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }}
           DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
           DATABEND_ENTERPRISE_LICENSE_EMBEDDED: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_RELEASE }}
+          DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}}
+          DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}}
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}
@@ -193,7 +197,7 @@ jobs:
       - Linux
       - 2c8g
       - aws
-    needs: [create_release, build_default]
+    needs: [ create_release, build_default ]
     strategy:
       fail-fast: false
       matrix:
@@ -231,7 +235,7 @@ jobs:
       - Linux
       - 2c8g
       - aws
-    needs: [create_release, build_default, build_docker]
+    needs: [ create_release, build_default, build_docker ]
     strategy:
       fail-fast: false
       matrix:
@@ -274,7 +278,7 @@ jobs:
       - Linux
       - 2c8g
       - aws
-    needs: [create_release, build_default]
+    needs: [ create_release, build_default ]
     strategy:
       fail-fast: false
       matrix:
@@ -323,7 +327,7 @@ jobs:
       - Linux
       - 2c8g
       - aws
-    needs: [create_release, build_default]
+    needs: [ create_release, build_default ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -402,7 +406,7 @@ jobs:
       - Linux
       - 2c8g
       - aws
-    needs: [create_release, build_docker]
+    needs: [ create_release, build_docker ]
     strategy:
       fail-fast: false
       matrix:
@@ -477,7 +481,7 @@ jobs:
       - Linux
       - 2c8g
       - aws
-    needs: [create_release, build_default]
+    needs: [ create_release, build_default ]
     strategy:
       matrix:
         arch:
@@ -534,7 +538,7 @@ jobs:
           gh release edit ${{ needs.create_release.outputs.version }} --draft=false
 
   sha256sums:
-    needs: [create_release, publish, distribution]
+    needs: [ create_release, publish, distribution ]
     runs-on: ubuntu-latest
     steps:
       - name: checkout
@@ -567,7 +571,7 @@ jobs:
       - Linux
       - 2c8g
       - aws
-    needs: [create_release, notify]
+    needs: [ create_release, notify ]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -597,7 +601,7 @@ jobs:
       - Linux
       - 4c16g
       - aws
-    needs: [create_release, notify]
+    needs: [ create_release, notify ]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -623,7 +627,7 @@ jobs:
             await script({context, core})
 
   benchmark:
-    needs: [create_release, docker_service, notify]
+    needs: [ create_release, docker_service, notify ]
     uses: ./.github/workflows/reuse.benchmark.yml
     secrets: inherit
     with:
@@ -635,6 +639,6 @@ jobs:
       target: all
 
   deb:
-    needs: [create_release, distribution, notify]
+    needs: [ create_release, distribution, notify ]
     uses: ./.github/workflows/deb.yml
     secrets: inherit

.github/workflows/reuse.linux.yml

@@ -63,6 +63,8 @@ jobs:
         timeout-minutes: 60
         env:
           DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
+          DATABEND_TELEMETRY_ENDPOINT: ${{ secrets.DATABEND_TELEMETRY_ENDPOINT}}
+          DATABEND_TELEMETRY_API_KEY: ${{ secrets.DATABEND_TELEMETRY_API_KEY}}
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.arch }}-unknown-linux-gnu
@@ -115,7 +117,7 @@ jobs:
         timeout-minutes: 60
 
   test_metactl:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -128,7 +130,7 @@ jobs:
         timeout-minutes: 10
 
   test_compat_meta_query:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -141,7 +143,7 @@ jobs:
         timeout-minutes: 10
 
   test_compat_fuse:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -154,7 +156,7 @@ jobs:
         timeout-minutes: 20
 
   test_compat_meta_meta:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -167,7 +169,7 @@ jobs:
         timeout-minutes: 20
 
   test_logs:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -201,7 +203,7 @@ jobs:
         timeout-minutes: 20
 
   test_meta_cluster:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -214,7 +216,7 @@ jobs:
         timeout-minutes: 10
 
   test_stateless_standalone:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -227,7 +229,7 @@ jobs:
         timeout-minutes: 18
 
   test_stateless_cluster:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -244,7 +246,7 @@ jobs:
         timeout-minutes: 18
 
   test_stateful_standalone:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -262,7 +264,7 @@ jobs:
           name: test-stateful-standalone-linux
 
   test_stateful_cluster:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -285,7 +287,7 @@ jobs:
 
   test_stateful_large_data:
     if: contains(github.event.pull_request.labels.*.name, 'ci-largedata')
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -298,7 +300,7 @@ jobs:
         timeout-minutes: 60
 
   test_stateful_iceberg_catalogs:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -316,7 +318,7 @@ jobs:
           name: test-stateful-iceberg-catalogs-standalone
 
   test_stateful_hive_standalone:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -348,7 +350,7 @@ jobs:
   #       continue-on-error: true
 
   test_ee_standalone:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -397,7 +399,7 @@ jobs:
   #          name: test-stateful-standalone-fake-time-linux
 
   test_ee_management_mode:
-    needs: [build, check]
+    needs: [ build, check ]
     runs-on:
       - self-hosted
       - X64
@@ -420,7 +422,7 @@ jobs:
           name: test-ee-management-mode-linux
 
   sqllogic:
-    needs: [build, check]
+    needs: [ build, check ]
     uses: ./.github/workflows/reuse.sqllogic.yml
     secrets: inherit
     with:

Cargo.lock

@@ -113,6 +113,7 @@ members = [
     "src/bendsave",
     "src/bendpy",
     "src/meta/app-storage",
+    "src/common/telemetry",
 ]
 
 # Workspace dependencies
@@ -186,6 +187,7 @@ databend-common-storages-stage = { path = "src/query/storages/stage" }
 databend-common-storages-stream = { path = "src/query/storages/stream" }
 databend-common-storages-system = { path = "src/query/storages/system" }
 databend-common-storages-view = { path = "src/query/storages/view" }
+databend-common-telemetry = { path = "src/common/telemetry" }
 databend-common-tracing = { path = "src/common/tracing" }
 databend-common-users = { path = "src/query/users" }
 databend-common-vector = { path = "src/common/vector" }

Cargo.toml

@@ -59,6 +59,7 @@ pub fn add_building_env_vars() {
     add_env_version();
     add_env_license();
     add_license_public_key();
+    add_env_telemetry();
 }
 
 pub fn set_env_config() {
@@ -103,6 +104,19 @@ pub fn add_license_public_key() {
     println!("cargo:rustc-env=DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY={v}");
 }
 
+pub fn add_env_telemetry() {
+    println!("cargo:rerun-if-env-changed=DATABEND_TELEMETRY_ENDPOINT");
+    let endpoint = env::var("DATABEND_TELEMETRY_ENDPOINT")
+        .ok()
+        .filter(|s| !s.is_empty())
+        .unwrap_or_else(|| "https://telemetry.databend.cloud/v1/report".to_string());
+    println!("cargo:rustc-env=DATABEND_TELEMETRY_ENDPOINT={endpoint}");
+
+    println!("cargo:rerun-if-env-changed=DATABEND_TELEMETRY_API_KEY");
+    let api_key = env::var("DATABEND_TELEMETRY_API_KEY").unwrap_or_default();
+    println!("cargo:rustc-env=DATABEND_TELEMETRY_API_KEY={api_key}");
+}
+
 pub fn add_env_commit_authors(repo: &Repository) {
     match git::get_commit_authors(repo) {
         Ok(authors) => println!("cargo:rustc-env=DATABEND_COMMIT_AUTHORS={}", authors),

src/common/building/src/lib.rs

@@ -0,0 +1,12 @@
+[package]
+name = "databend-common-telemetry"
+version = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+publish = { workspace = true }
+edition = { workspace = true }
+
+[dependencies]
+reqwest = { workspace = true, features = ["json", "rustls-tls"] }
+serde_json = { workspace = true }
+tokio = { workspace = true }