chore: Add tool for examples templating

Makefile

@@ -63,7 +63,7 @@ lint: go/lint helm/lint buf/lint goreleaser/lint ## Lint Go, Helm and protobuf
 test: go/test ## Run unit tests
 
 .PHONY: generate
-generate: $(BIN)/buf $(BIN)/protoc-gen-go $(BIN)/protoc-gen-go-vtproto $(BIN)/protoc-gen-openapiv2 $(BIN)/protoc-gen-grpc-gateway $(BIN)/protoc-gen-connect-go $(BIN)/protoc-gen-connect-go-mux $(BIN)/gomodifytags ## Regenerate protobuf
+generate: $(BIN)/buf $(BIN)/protoc-gen-go $(BIN)/protoc-gen-go-vtproto $(BIN)/protoc-gen-openapiv2 $(BIN)/protoc-gen-grpc-gateway $(BIN)/protoc-gen-connect-go $(BIN)/protoc-gen-connect-go-mux $(BIN)/gomodifytags examples/generate ## Regenerate protobuf
 	rm -Rf api/openapiv2/gen/ api/gen
 	find pkg/ \( -name \*.pb.go -o -name \*.connect\*.go \) -delete
 	cd api/ && PATH=$(BIN) $(BIN)/buf generate
@@ -94,6 +94,11 @@ examples/test: RUN := .*
 examples/test: $(BIN)/gotestsum
 	$(BIN)/gotestsum --format testname --rerun-fails=2 --packages ./examples -- --count 1 --parallel 2 --timeout 1h --tags examples -run "$(RUN)"
 
+# Run template generation on examples
+.PHONY: examples/generate
+examples/generate:
+	go run ./tools/generate-examples
+
 .PHONY: build
 build: frontend/build go/bin ## Do a production build (requiring the frontend build to be present)
 

examples/_templates/README.md

@@ -0,0 +1,6 @@
+# Examples templates
+
+In order to make the examples more understandable for our users, we have copied very similar code paths around. This folder tries to make sure the different configs do not diverge too much.
+
+
+

examples/_templates/templates.go

@@ -0,0 +1,26 @@
+package templates
+
+import "embed"
+
+var templates []*Template
+
+func Add(t *Template) {
+	templates = append(templates, t)
+}
+
+func Range(f func(t *Template) error) error {
+	for _, t := range templates {
+		if err := f(t); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type Template struct {
+	Name         string
+	Description  string
+	CleanUpPaths []string // Paths that should be cleared before we copy
+	Destinations []string // Destination paths
+	Assets       embed.FS
+}

examples/_templates/tempo/assets/tempo/tempo.yml

@@ -0,0 +1,53 @@
+server:
+  http_listen_port: 3200
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+          endpoint: 0.0.0.0:4318
+        grpc:
+          endpoint: 0.0.0.0:4317
+    opencensus:
+
+overrides:
+  metrics_generator_processors:
+    - span-metrics
+    - service-graphs
+    - local-blocks
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the wal locally
+    local:
+      path: /tmp/tempo/blocks
+
+metrics_generator:
+  storage:
+    path: /var/tempo/storage
+  traces_storage:
+    path: /var/tempo/traces_storage

examples/_templates/tempo/tempo.go

@@ -0,0 +1,27 @@
+package tempo
+
+import (
+	"embed"
+
+	templates "github.com/grafana/pyroscope/examples/_templates"
+)
+
+//go:embed assets/*
+var f embed.FS
+
+func init() {
+	templates.Add(&templates.Template{
+		Name:         "tempo",
+		Description:  "Tempo is a distributed tracing system.",
+		Assets:       f,
+		CleanUpPaths: []string{"tempo"},
+		Destinations: []string{
+			"examples/tracing/dotnet",
+			"examples/tracing/golang",
+			"examples/tracing/java",
+			"examples/tracing/python",
+			"examples/tracing/ruby",
+			"examples/tracing/tempo",
+		},
+	})
+}

examples/tracing/dotnet/tempo/tempo.yml

@@ -25,6 +25,12 @@ distributor:
           endpoint: 0.0.0.0:4317
     opencensus:
 
+overrides:
+  metrics_generator_processors:
+    - span-metrics
+    - service-graphs
+    - local-blocks
+
 ingester:
   max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
 
@@ -39,3 +45,9 @@ storage:
       path: /tmp/tempo/wal             # where to store the wal locally
     local:
       path: /tmp/tempo/blocks
+
+metrics_generator:
+  storage:
+    path: /var/tempo/storage
+  traces_storage:
+    path: /var/tempo/traces_storage

examples/tracing/java/tempo/tempo.yml

@@ -25,6 +25,12 @@ distributor:
           endpoint: 0.0.0.0:4317
     opencensus:
 
+overrides:
+  metrics_generator_processors:
+    - span-metrics
+    - service-graphs
+    - local-blocks
+
 ingester:
   max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
 
@@ -39,3 +45,9 @@ storage:
       path: /tmp/tempo/wal             # where to store the wal locally
     local:
       path: /tmp/tempo/blocks
+
+metrics_generator:
+  storage:
+    path: /var/tempo/storage
+  traces_storage:
+    path: /var/tempo/traces_storage

examples/tracing/python/tempo/tempo.yml

@@ -25,6 +25,12 @@ distributor:
           endpoint: 0.0.0.0:4317
     opencensus:
 
+overrides:
+  metrics_generator_processors:
+    - span-metrics
+    - service-graphs
+    - local-blocks
+
 ingester:
   max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
 
@@ -39,3 +45,9 @@ storage:
       path: /tmp/tempo/wal             # where to store the wal locally
     local:
       path: /tmp/tempo/blocks
+
+metrics_generator:
+  storage:
+    path: /var/tempo/storage
+  traces_storage:
+    path: /var/tempo/traces_storage

examples/tracing/ruby/tempo/tempo.yml

@@ -25,6 +25,12 @@ distributor:
           endpoint: 0.0.0.0:4317
     opencensus:
 
+overrides:
+  metrics_generator_processors:
+    - span-metrics
+    - service-graphs
+    - local-blocks
+
 ingester:
   max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
 
@@ -39,3 +45,9 @@ storage:
       path: /tmp/tempo/wal             # where to store the wal locally
     local:
       path: /tmp/tempo/blocks
+
+metrics_generator:
+  storage:
+    path: /var/tempo/storage
+  traces_storage:
+    path: /var/tempo/traces_storage

examples/tracing/tempo/tempo/tempo.yml

@@ -25,6 +25,12 @@ distributor:
           endpoint: 0.0.0.0:4317
     opencensus:
 
+overrides:
+  metrics_generator_processors:
+    - span-metrics
+    - service-graphs
+    - local-blocks
+
 ingester:
   max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
 
@@ -39,3 +45,9 @@ storage:
       path: /tmp/tempo/wal             # where to store the wal locally
     local:
       path: /tmp/tempo/blocks
+
+metrics_generator:
+  storage:
+    path: /var/tempo/storage
+  traces_storage:
+    path: /var/tempo/traces_storage

tools/generate-examples/main.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	templates "github.com/grafana/pyroscope/examples/_templates"
+	_ "github.com/grafana/pyroscope/examples/_templates/tempo"
+)
+
+func rootPath() (string, error) {
+	cmd := exec.Command(
+		"git",
+		"rev-parse",
+		"--show-toplevel",
+	)
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(string(output)), nil
+}
+
+func run() error {
+	rootPath, err := rootPath()
+	if err != nil {
+		return err
+	}
+
+	cleanUpPaths := make(map[string]struct{})
+
+	if err := templates.Range(func(t *templates.Template) error {
+		for _, path := range t.Destinations {
+			for _, cleanUpPath := range t.CleanUpPaths {
+				cleanUpPaths[filepath.Join(rootPath, path, cleanUpPath)] = struct{}{}
+			}
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// remove all the paths in cleanUpPaths
+	for path := range cleanUpPaths {
+		if err := os.RemoveAll(path); err != nil {
+			return err
+		}
+	}
+
+	if err := templates.Range(func(t *templates.Template) error {
+		for _, path := range t.Destinations {
+			fullPath := filepath.Join(rootPath, path)
+			log.Println("Copying", t.Name, "to", fullPath)
+			prefixDir := "assets"
+			err := fs.WalkDir(t.Assets, prefixDir, func(path string, d fs.DirEntry, err error) error {
+				fullPath := filepath.Join(fullPath, strings.TrimPrefix(path, prefixDir+"/"))
+				if d.IsDir() {
+					return os.MkdirAll(fullPath, 0755)
+				}
+
+				src, err := t.Assets.Open(path)
+				if err != nil {
+					return err
+				}
+				defer src.Close()
+				dst, err := os.Create(fullPath)
+				if err != nil {
+					return err
+				}
+				defer dst.Close()
+				_, err = io.Copy(dst, src)
+				if err != nil {
+					return err
+				}
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}