diff --git a/Makefile b/Makefile index 8e6722150e..ec032423fa 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ lint: go/lint helm/lint buf/lint goreleaser/lint ## Lint Go, Helm and protobuf test: go/test ## Run unit tests .PHONY: generate -generate: $(BIN)/buf $(BIN)/protoc-gen-go $(BIN)/protoc-gen-go-vtproto $(BIN)/protoc-gen-openapiv2 $(BIN)/protoc-gen-grpc-gateway $(BIN)/protoc-gen-connect-go $(BIN)/protoc-gen-connect-go-mux $(BIN)/gomodifytags ## Regenerate protobuf +generate: $(BIN)/buf $(BIN)/protoc-gen-go $(BIN)/protoc-gen-go-vtproto $(BIN)/protoc-gen-openapiv2 $(BIN)/protoc-gen-grpc-gateway $(BIN)/protoc-gen-connect-go $(BIN)/protoc-gen-connect-go-mux $(BIN)/gomodifytags examples/generate ## Regenerate protobuf rm -Rf api/openapiv2/gen/ api/gen find pkg/ \( -name \*.pb.go -o -name \*.connect\*.go \) -delete cd api/ && PATH=$(BIN) $(BIN)/buf generate @@ -94,6 +94,11 @@ examples/test: RUN := .* examples/test: $(BIN)/gotestsum $(BIN)/gotestsum --format testname --rerun-fails=2 --packages ./examples -- --count 1 --parallel 2 --timeout 1h --tags examples -run "$(RUN)" +# Run template generation on examples +.PHONY: examples/generate +examples/generate: + go run ./tools/generate-examples + .PHONY: build build: frontend/build go/bin ## Do a production build (requiring the frontend build to be present) diff --git a/examples/_templates/README.md b/examples/_templates/README.md new file mode 100644 index 0000000000..41ac20dd12 --- /dev/null +++ b/examples/_templates/README.md @@ -0,0 +1,6 @@ +# Examples templates + +In order to make the examples more understandable for our users, we have copied very similar code paths around. This folder tries to make sure the different configs do not diverge too much. + + + diff --git a/examples/_templates/templates.go b/examples/_templates/templates.go new file mode 100644 index 0000000000..0050f432ea --- /dev/null +++ b/examples/_templates/templates.go @@ -0,0 +1,26 @@ +package templates + +import "embed" + +var templates []*Template + +func Add(t *Template) { + templates = append(templates, t) +} + +func Range(f func(t *Template) error) error { + for _, t := range templates { + if err := f(t); err != nil { + return err + } + } + return nil +} + +type Template struct { + Name string + Description string + CleanUpPaths []string // Paths that should be cleared before we copy + Destinations []string // Destination paths + Assets embed.FS +} diff --git a/examples/_templates/tempo/assets/tempo/tempo.yml b/examples/_templates/tempo/assets/tempo/tempo.yml new file mode 100644 index 0000000000..6db4f354ad --- /dev/null +++ b/examples/_templates/tempo/assets/tempo/tempo.yml @@ -0,0 +1,53 @@ +server: + http_listen_port: 3200 + +query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + trace_by_id: + duration_slo: 5s + +distributor: + receivers: # this configuration will listen on all ports and protocols that tempo is capable of. + jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can + protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver + thrift_http: # + grpc: # for a production deployment you should only enable the receivers you need! + thrift_binary: + thrift_compact: + zipkin: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + opencensus: + +overrides: + metrics_generator_processors: + - span-metrics + - service-graphs + - local-blocks + +ingester: + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally + +compactor: + compaction: + block_retention: 1h # overall Tempo trace retention. set for demo purposes + +storage: + trace: + backend: local # backend configuration to use + wal: + path: /tmp/tempo/wal # where to store the wal locally + local: + path: /tmp/tempo/blocks + +metrics_generator: + storage: + path: /var/tempo/storage + traces_storage: + path: /var/tempo/traces_storage diff --git a/examples/_templates/tempo/tempo.go b/examples/_templates/tempo/tempo.go new file mode 100644 index 0000000000..420d062ca8 --- /dev/null +++ b/examples/_templates/tempo/tempo.go @@ -0,0 +1,27 @@ +package tempo + +import ( + "embed" + + templates "github.com/grafana/pyroscope/examples/_templates" +) + +//go:embed assets/* +var f embed.FS + +func init() { + templates.Add(&templates.Template{ + Name: "tempo", + Description: "Tempo is a distributed tracing system.", + Assets: f, + CleanUpPaths: []string{"tempo"}, + Destinations: []string{ + "examples/tracing/dotnet", + "examples/tracing/golang", + "examples/tracing/java", + "examples/tracing/python", + "examples/tracing/ruby", + "examples/tracing/tempo", + }, + }) +} diff --git a/examples/tracing/dotnet/tempo/tempo.yml b/examples/tracing/dotnet/tempo/tempo.yml index 6c536317dd..6db4f354ad 100644 --- a/examples/tracing/dotnet/tempo/tempo.yml +++ b/examples/tracing/dotnet/tempo/tempo.yml @@ -25,6 +25,12 @@ distributor: endpoint: 0.0.0.0:4317 opencensus: +overrides: + metrics_generator_processors: + - span-metrics + - service-graphs + - local-blocks + ingester: max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally @@ -39,3 +45,9 @@ storage: path: /tmp/tempo/wal # where to store the wal locally local: path: /tmp/tempo/blocks + +metrics_generator: + storage: + path: /var/tempo/storage + traces_storage: + path: /var/tempo/traces_storage diff --git a/examples/tracing/java/tempo/tempo.yml b/examples/tracing/java/tempo/tempo.yml index 6c536317dd..6db4f354ad 100644 --- a/examples/tracing/java/tempo/tempo.yml +++ b/examples/tracing/java/tempo/tempo.yml @@ -25,6 +25,12 @@ distributor: endpoint: 0.0.0.0:4317 opencensus: +overrides: + metrics_generator_processors: + - span-metrics + - service-graphs + - local-blocks + ingester: max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally @@ -39,3 +45,9 @@ storage: path: /tmp/tempo/wal # where to store the wal locally local: path: /tmp/tempo/blocks + +metrics_generator: + storage: + path: /var/tempo/storage + traces_storage: + path: /var/tempo/traces_storage diff --git a/examples/tracing/python/tempo/tempo.yml b/examples/tracing/python/tempo/tempo.yml index 6c536317dd..6db4f354ad 100644 --- a/examples/tracing/python/tempo/tempo.yml +++ b/examples/tracing/python/tempo/tempo.yml @@ -25,6 +25,12 @@ distributor: endpoint: 0.0.0.0:4317 opencensus: +overrides: + metrics_generator_processors: + - span-metrics + - service-graphs + - local-blocks + ingester: max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally @@ -39,3 +45,9 @@ storage: path: /tmp/tempo/wal # where to store the wal locally local: path: /tmp/tempo/blocks + +metrics_generator: + storage: + path: /var/tempo/storage + traces_storage: + path: /var/tempo/traces_storage diff --git a/examples/tracing/ruby/tempo/tempo.yml b/examples/tracing/ruby/tempo/tempo.yml index 6c536317dd..6db4f354ad 100644 --- a/examples/tracing/ruby/tempo/tempo.yml +++ b/examples/tracing/ruby/tempo/tempo.yml @@ -25,6 +25,12 @@ distributor: endpoint: 0.0.0.0:4317 opencensus: +overrides: + metrics_generator_processors: + - span-metrics + - service-graphs + - local-blocks + ingester: max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally @@ -39,3 +45,9 @@ storage: path: /tmp/tempo/wal # where to store the wal locally local: path: /tmp/tempo/blocks + +metrics_generator: + storage: + path: /var/tempo/storage + traces_storage: + path: /var/tempo/traces_storage diff --git a/examples/tracing/tempo/tempo/tempo.yml b/examples/tracing/tempo/tempo/tempo.yml index 6c536317dd..6db4f354ad 100644 --- a/examples/tracing/tempo/tempo/tempo.yml +++ b/examples/tracing/tempo/tempo/tempo.yml @@ -25,6 +25,12 @@ distributor: endpoint: 0.0.0.0:4317 opencensus: +overrides: + metrics_generator_processors: + - span-metrics + - service-graphs + - local-blocks + ingester: max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally @@ -39,3 +45,9 @@ storage: path: /tmp/tempo/wal # where to store the wal locally local: path: /tmp/tempo/blocks + +metrics_generator: + storage: + path: /var/tempo/storage + traces_storage: + path: /var/tempo/traces_storage diff --git a/tools/generate-examples/main.go b/tools/generate-examples/main.go new file mode 100644 index 0000000000..5484ee8188 --- /dev/null +++ b/tools/generate-examples/main.go @@ -0,0 +1,98 @@ +package main + +import ( + "io" + "io/fs" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + templates "github.com/grafana/pyroscope/examples/_templates" + _ "github.com/grafana/pyroscope/examples/_templates/tempo" +) + +func rootPath() (string, error) { + cmd := exec.Command( + "git", + "rev-parse", + "--show-toplevel", + ) + output, err := cmd.Output() + if err != nil { + return "", err + } + + return strings.TrimSpace(string(output)), nil +} + +func run() error { + rootPath, err := rootPath() + if err != nil { + return err + } + + cleanUpPaths := make(map[string]struct{}) + + if err := templates.Range(func(t *templates.Template) error { + for _, path := range t.Destinations { + for _, cleanUpPath := range t.CleanUpPaths { + cleanUpPaths[filepath.Join(rootPath, path, cleanUpPath)] = struct{}{} + } + } + return nil + }); err != nil { + return err + } + + // remove all the paths in cleanUpPaths + for path := range cleanUpPaths { + if err := os.RemoveAll(path); err != nil { + return err + } + } + + if err := templates.Range(func(t *templates.Template) error { + for _, path := range t.Destinations { + fullPath := filepath.Join(rootPath, path) + log.Println("Copying", t.Name, "to", fullPath) + prefixDir := "assets" + err := fs.WalkDir(t.Assets, prefixDir, func(path string, d fs.DirEntry, err error) error { + fullPath := filepath.Join(fullPath, strings.TrimPrefix(path, prefixDir+"/")) + if d.IsDir() { + return os.MkdirAll(fullPath, 0755) + } + + src, err := t.Assets.Open(path) + if err != nil { + return err + } + defer src.Close() + dst, err := os.Create(fullPath) + if err != nil { + return err + } + defer dst.Close() + _, err = io.Copy(dst, src) + if err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + } + return nil + }); err != nil { + return err + } + return nil +} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +}