From d58a1562a136e81ae1f8d1822ca1dec4b5dceba0 Mon Sep 17 00:00:00 2001 From: Tommy Reilly Date: Fri, 16 Jan 2026 00:26:47 -0600 Subject: [PATCH] Increase trace_event buffer size if GPU profiling is enabled --- go.mod | 2 +- go.sum | 4 ++-- main.go | 34 ++++++++++++++++++++-------------- reporter/parca_reporter.go | 2 ++ 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 680b82423d..13fe15176a 100644 --- a/go.mod +++ b/go.mod @@ -177,4 +177,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace go.opentelemetry.io/ebpf-profiler => github.com/parca-dev/opentelemetry-ebpf-profiler v0.0.0-20260115180451-d79ff27bb66a +replace go.opentelemetry.io/ebpf-profiler => github.com/parca-dev/opentelemetry-ebpf-profiler v0.0.0-20260120173907-dbdd569b8667 diff --git a/go.sum b/go.sum index d155ea7057..898a4b596b 100644 --- a/go.sum +++ b/go.sum @@ -299,8 +299,8 @@ github.com/opencontainers/selinux v1.13.0/go.mod h1:XxWTed+A/s5NNq4GmYScVy+9jzXh github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/parca-dev/oomprof v0.1.6 h1:potfd09aphNKqsIF54ZsiddTvksVMjQiaKnczFOsVGM= github.com/parca-dev/oomprof v0.1.6/go.mod h1:iqI6XrmiNWOa8m2vEIKo+GtQrqbWCMLFpBWuk8RuAPs= -github.com/parca-dev/opentelemetry-ebpf-profiler v0.0.0-20260115180451-d79ff27bb66a h1:sHdTnHKnIgO/BHpzP61h78oAPZV0/uwZTloM0pZi2yU= -github.com/parca-dev/opentelemetry-ebpf-profiler v0.0.0-20260115180451-d79ff27bb66a/go.mod h1:yVpeERcH/++ahci/FAPA8l4TL+y0JY3T6z4xu/r3+HM= +github.com/parca-dev/opentelemetry-ebpf-profiler v0.0.0-20260120173907-dbdd569b8667 h1:x7EC4K8Zl4WF1f71shBoGPFut1iTLj526I+yfeOWDlo= +github.com/parca-dev/opentelemetry-ebpf-profiler v0.0.0-20260120173907-dbdd569b8667/go.mod h1:yVpeERcH/++ahci/FAPA8l4TL+y0JY3T6z4xu/r3+HM= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= diff --git a/main.go b/main.go index d674f769b9..91093f0e3f 100644 --- a/main.go +++ b/main.go @@ -429,21 +429,27 @@ func mainWithExitCode() flags.ExitCode { } // Load the eBPF code and map definitions + traceBufferMultiplier := 1 + if f.InstrumentCudaLaunch { + // GPU profiling generates high trace volume, increase buffer + traceBufferMultiplier = 50 + } trc, err := tracer.NewTracer(mainCtx, &tracer.Config{ - VerboseMode: f.BPF.VerboseLogging, - Reporter: rep, - Intervals: intervals, - IncludeTracers: includeTracers, - SamplesPerSecond: f.Profiling.CPUSamplingFrequency, - MapScaleFactor: f.BPF.MapScaleFactor, - FilterErrorFrames: !f.Profiling.EnableErrorFrames, - KernelVersionCheck: !f.Hidden.IgnoreUnsafeKernelVersion, - BPFVerifierLogLevel: f.BPF.VerifierLogLevel, - ProbabilisticInterval: f.Profiling.ProbabilisticInterval, - ProbabilisticThreshold: f.Profiling.ProbabilisticThreshold, - OffCPUThreshold: uint32(f.OffCPUThreshold * math.MaxUint32), - IncludeEnvVars: includeEnvVars, - InstrumentCudaLaunch: f.InstrumentCudaLaunch, + VerboseMode: f.BPF.VerboseLogging, + Reporter: rep, + Intervals: intervals, + IncludeTracers: includeTracers, + SamplesPerSecond: f.Profiling.CPUSamplingFrequency, + MapScaleFactor: f.BPF.MapScaleFactor, + FilterErrorFrames: !f.Profiling.EnableErrorFrames, + KernelVersionCheck: !f.Hidden.IgnoreUnsafeKernelVersion, + BPFVerifierLogLevel: f.BPF.VerifierLogLevel, + ProbabilisticInterval: f.Profiling.ProbabilisticInterval, + ProbabilisticThreshold: f.Profiling.ProbabilisticThreshold, + OffCPUThreshold: uint32(f.OffCPUThreshold * math.MaxUint32), + IncludeEnvVars: includeEnvVars, + InstrumentCudaLaunch: f.InstrumentCudaLaunch, + TraceBufferSizeMultiplier: traceBufferMultiplier, }) metrics.SetReporter(parcaReporter) if err != nil { diff --git a/reporter/parca_reporter.go b/reporter/parca_reporter.go index 99f0ef3a5b..67bce3d0b8 100644 --- a/reporter/parca_reporter.go +++ b/reporter/parca_reporter.go @@ -1067,6 +1067,8 @@ func (r *ParcaReporter) reportDataToBackend(ctx context.Context, buf *bytes.Buff return err } + log.Infof("[reporter] sending %d samples (%d bytes) to server", record.NumRows(), buf.Len()) + client, err := r.client.Write(ctx) if err != nil { return err