Skip to content

chore(query): Tuning sort spill memory usage #18520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
+734 −214
Open

chore(query): Tuning sort spill memory usage #18520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
94d057a
refine
forsaken628 Aug 6, 2025
10dbb13
update
forsaken628 Aug 7, 2025
c6efb16
update
forsaken628 Aug 8, 2025
41a85ad
update
forsaken628 Aug 9, 2025
4bf7bfe
update
forsaken628 Aug 11, 2025
94c203a
dump fastrace
forsaken628 Aug 12, 2025
79d7cb4
update
forsaken628 Aug 12, 2025
f9173e3
update
forsaken628 Aug 12, 2025
208c308
fix
forsaken628 Aug 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 122 additions & 29 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 7 additions & 6 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ byte-unit = "5.1.6"
bytemuck = { version = "1", features = ["derive"] }
byteorder = "1.4.3"
bytes = "1.5.0"
bytesize = "1.1.0"
bytesize = "2"
cbordata = { version = "0.6.0" }
cfg-if = "1.0.0"
chrono = { version = "0.4.40", features = ["serde"] }
Expand Down Expand Up @@ -570,19 +570,20 @@ vergen = { version = "8.3.1", default-features = false, features = ["build", "ca

# Observability
env_logger = "0.11"
fastrace = { version = "0.7.4", features = ["enable"] }
fastrace-opentelemetry = "0.7.4"
fastrace = { version = "0.7.14", features = ["enable"] }
fastrace-opentelemetry = "0.10"
log = { version = "0.4.27", features = ["serde", "kv_unstable_std"] }
logcall = "0.1.9"
opentelemetry = { version = "0.26.0", features = ["trace", "logs"] }
opentelemetry-otlp = { version = "0.26.0", features = [
opentelemetry = { version = "0.29", features = ["trace", "logs"] }
opentelemetry-otlp = { version = "0.29", features = [
"trace",
"logs",
"grpc-tonic",
"http-proto",
"reqwest-client",
"gzip-tonic",
] }
opentelemetry_sdk = { version = "0.26.0", features = ["trace", "logs", "rt-tokio"] }
opentelemetry_sdk = { version = "0.29", features = ["trace", "logs", "rt-tokio"] }
prometheus-client = "0.22"
prometheus-parse = "0.2.3"
tracing = "0.1.40"
Expand Down
7 changes: 4 additions & 3 deletions scripts/ci/deploy/databend-query-cluster-tracing.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,21 +29,22 @@ python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191

export RUST_BACKTRACE=1
export LOG_TRACING_ON=true
export LOG_TRACING_CAPTURE_LOG_LEVEL="Warn,databend_=Debug"

echo 'Start databend-query node-1'
nohup target/${BUILD_PROFILE}/databend-query -c scripts/ci/deploy/config/databend-query-node-1.toml --internal-enable-sandbox-tenant --log-tracing-level Warn,databend_=Debug,openraft=Info >./.databend/query-1.out 2>&1 &
nohup target/${BUILD_PROFILE}/databend-query -c scripts/ci/deploy/config/databend-query-node-1.toml --internal-enable-sandbox-tenant >./.databend/query-1.out 2>&1 &

echo "Waiting on node-1..."
python3 scripts/ci/wait_tcp.py --timeout 30 --port 9091

echo 'Start databend-query node-2'
nohup target/${BUILD_PROFILE}/databend-query -c scripts/ci/deploy/config/databend-query-node-2.toml --internal-enable-sandbox-tenant --log-tracing-level Warn,databend_=Debug,openraft=Info >./.databend/query-2.out 2>&1 &
nohup target/${BUILD_PROFILE}/databend-query -c scripts/ci/deploy/config/databend-query-node-2.toml --internal-enable-sandbox-tenant >./.databend/query-2.out 2>&1 &

echo "Waiting on node-2..."
python3 scripts/ci/wait_tcp.py --timeout 30 --port 9092

echo 'Start databend-query node-3'
nohup target/${BUILD_PROFILE}/databend-query -c scripts/ci/deploy/config/databend-query-node-3.toml --internal-enable-sandbox-tenant --log-tracing-level Warn,databend_=Debug,openraft=Info >./.databend/query-3.out 2>&1 &
nohup target/${BUILD_PROFILE}/databend-query -c scripts/ci/deploy/config/databend-query-node-3.toml --internal-enable-sandbox-tenant >./.databend/query-3.out 2>&1 &

echo "Waiting on node-3..."
python3 scripts/ci/wait_tcp.py --timeout 30 --port 9093
Expand Down
2 changes: 1 addition & 1 deletion src/common/tracing/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ log = { workspace = true }
logforth = { workspace = true }
opendal = { workspace = true }
opentelemetry = { workspace = true }
opentelemetry-otlp = { workspace = true, features = ["reqwest-client"] }
opentelemetry-otlp = { workspace = true }
opentelemetry_sdk = { workspace = true }
parquet = { workspace = true }
serde = { workspace = true }
Expand Down
Loading
Loading