wip: otlp-bench

Author: felixge

otlp-bench/.gitignore

@@ -0,0 +1,2 @@
+/versions/.opentelemetry-proto-go
+/testdata/versions/

otlp-bench/README.md

@@ -0,0 +1,32 @@
+# otlp-bench
+
+`otlp-bench` is a tool for comparing different variants for encoding profiling data as OTLP.
+
+A variant is a combination of a specific version (commit) of the opentelemetry-proto repository as well as code for encoding profiling data as OTLP.
+
+## Managing Versions
+
+### Example Usage
+
+Adding new versions of the OTLP profiling protocol is done via the `version add` command.
+
+```bash
+# Add a specific version from upstream
+./otlp-bench version add <name> <git-ref>
+
+# Add a version from a fork
+./otlp-bench version add <name> <git-ref> <git-remote-url>
+```
+
+### Flags
+
+- `--verbose` prints every shell command that the tool launches.
+
+Generated assets are stored in `./versions` using the system `git`, `make`, and `go` binaries, and cloning from the upstream `opentelemetry-proto-go` repository. Each invocation writes metadata to `./versions/versions.json` and initialises/updates `./versions/go.work` so that the generated modules can be consumed immediately.
+
+### How It Works
+
+1. Clones the latest `opentelemetry-proto-go` (build system)
+2. Points its submodule at the specified commit/fork
+3. Generates Go code with custom module paths
+4. Makes versions available via Go workspace

otlp-bench/bench.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/open-telemetry/sig-profiling/otlp-bench/versions/1.9/slim/otlp/profiles/v1development"
+	"github.com/urfave/cli/v3"
+	"google.golang.org/protobuf/proto"
+)
+
+// Flag and argument identifiers
+const (
+	benchOutFlag      = "out"
+	benchTruncateFlag = "truncate"
+	benchFilesArg     = "files"
+)
+
+// newBenchCommand creates the bench command for benchmarking OTLP profile files
+func (a *App) newBenchCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "bench",
+		Usage: "Benchmark OTLP encoded profile files",
+		Description: `Analyzes one or more OTLP encoded profile files and outputs
+benchmark metrics to a CSV file.
+
+The command takes one or more OTLP profile files as input and generates
+benchmark results in CSV format. Use the --out flag to specify the output
+file path (defaults to otlp-bench.csv) and --truncate to overwrite an
+existing output file.`,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    benchOutFlag,
+				Aliases: []string{"o"},
+				Usage:   "Output CSV file path",
+				Value:   "otlp-bench.csv",
+			},
+			&cli.BoolFlag{
+				Name:    benchTruncateFlag,
+				Aliases: []string{"t"},
+				Usage:   "Truncate output file if it exists. By default existing records are updated as needed.",
+			},
+		},
+		Arguments: []cli.Argument{
+			&cli.StringArgs{
+				Name:      benchFilesArg,
+				UsageText: "<file1> [file2] ...",
+				Min:       1,
+				Max:       -1, // Unlimited
+			},
+		},
+		Action: a.benchAction,
+	}
+}
+
+// benchAction is the main action handler for the bench command
+func (a *App) benchAction(ctx context.Context, cmd *cli.Command) error {
+	args, err := a.parseBenchArgs(cmd)
+	if err != nil {
+		return err
+	}
+
+	a.Log.Info("bench command invoked",
+		"files", args.Files,
+		"out", args.Out,
+		"truncate", args.Truncate,
+	)
+
+	for _, file := range args.Files {
+		if err := a.benchFile(file); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (a *App) benchFile(file string) error {
+	data, err := os.ReadFile(file)
+	if err != nil {
+		return fmt.Errorf("read file: %w", err)
+	}
+
+	profilesData := &v1development.ProfilesData{}
+	if err := proto.Unmarshal(data, profilesData); err != nil {
+		return fmt.Errorf("unmarshal profiles data: %w", err)
+	}
+
+	fmt.Printf("len(profilesData.Dictionary.StringTable): %v\n", len(profilesData.Dictionary.StringTable))
+
+	return nil
+}
+
+// benchArgs holds the parsed command arguments and flags
+type benchArgs struct {
+	Files    []string
+	Out      string
+	Truncate bool
+}
+
+// parseBenchArgs parses and validates the bench command arguments
+func (a *App) parseBenchArgs(cmd *cli.Command) (benchArgs, error) {
+	args := benchArgs{
+		Files:    cmd.StringArgs(benchFilesArg),
+		Out:      cmd.String(benchOutFlag),
+		Truncate: cmd.Bool(benchTruncateFlag),
+	}
+
+	if len(args.Files) == 0 {
+		return benchArgs{}, cli.Exit("at least one input file required", 1)
+	}
+
+	return args, nil
+}

otlp-bench/go.mod

@@ -0,0 +1,9 @@
+module github.com/open-telemetry/sig-profiling/otlp-bench
+
+go 1.25.3
+
+require (
+	github.com/lmittmann/tint v1.1.2 // indirect
+	github.com/urfave/cli/v3 v3.5.0 // indirect
+	google.golang.org/protobuf v1.36.10 // indirect
+)

otlp-bench/go.sum

@@ -0,0 +1,6 @@
+github.com/lmittmann/tint v1.1.2 h1:2CQzrL6rslrsyjqLDwD11bZ5OpLBPU+g3G/r5LSfS8w=
+github.com/lmittmann/tint v1.1.2/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
+github.com/urfave/cli/v3 v3.5.0 h1:qCuFMmdayTF3zmjG8TSsoBzrDqszNrklYg2x3g4MSgw=
+github.com/urfave/cli/v3 v3.5.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
+google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=

otlp-bench/main.go

@@ -0,0 +1,127 @@
+package main
+
+import (
+	"cmp"
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/lmittmann/tint"
+	"github.com/urfave/cli/v3"
+)
+
+func main() {
+	var app App
+	app.Run(context.Background(), os.Args)
+}
+
+type App struct {
+	Writer         io.Writer
+	ErrWriter      io.Writer
+	Log            *slog.Logger
+	VersionDir     string
+	ExitErrHandler func(ctx context.Context, c *cli.Command, err error)
+}
+
+func (b *App) Run(ctx context.Context, args []string) error {
+	cmd := b.newRootCommand()
+	return cmd.Run(ctx, args)
+}
+
+func (a *App) newRootCommand() *cli.Command {
+	// Start app initialization
+	a.Writer = cmp.Or(a.Writer, io.Writer(os.Stdout))
+	a.ErrWriter = cmp.Or(a.ErrWriter, io.Writer(os.Stderr))
+	if a.ExitErrHandler == nil {
+		a.ExitErrHandler = func(ctx context.Context, c *cli.Command, err error) {
+			a.Log.Error("command failed", "error", err)
+			os.Exit(1)
+		}
+	}
+
+	root := &cli.Command{
+		Name:  "otlp-bench",
+		Usage: "Manage OTLP profiling benchmarks",
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:  "verbose",
+				Usage: "Enable verbose command output",
+			},
+		},
+		Before:         a.before,
+		Writer:         a.Writer,
+		ErrWriter:      a.ErrWriter,
+		ExitErrHandler: a.ExitErrHandler,
+		Commands: []*cli.Command{
+			a.newBenchCommand(),
+			{
+				Name:  "version",
+				Usage: "Manage OTLP proto versions",
+				Commands: []*cli.Command{
+					a.newVersionAddCommand(),
+				},
+			},
+		},
+	}
+	return root
+}
+
+const (
+	rootVerboseFlag = "verbose"
+)
+
+func (a *App) before(ctx context.Context, cmd *cli.Command) (context.Context, error) {
+	// Init logger
+	if a.Log == nil {
+		handlerOpts := &tint.Options{Level: slog.LevelDebug}
+		if cmd.Bool(rootVerboseFlag) {
+			handlerOpts.Level = slog.LevelDebug
+		}
+		tint.NewHandler(a.ErrWriter, handlerOpts)
+		handler := tint.NewHandler(a.Writer, handlerOpts)
+		logger := slog.New(handler)
+		a.Log = logger
+	}
+
+	// Resolve versions directory
+	if a.VersionDir == "" {
+		a.VersionDir = "versions"
+	}
+	var err error
+	a.VersionDir, err = filepath.Abs(a.VersionDir)
+	if err != nil {
+		return ctx, fmt.Errorf("resolve versions dir: %w", err)
+	}
+	if !strings.HasPrefix(a.VersionDir, goModRoot()) {
+		return ctx, fmt.Errorf("versions dir must be a subdirectory of the go mod root: %s is not a subdirectory of %s", a.VersionDir, goModRoot())
+	}
+
+	return ctx, nil
+}
+
+func (a *App) exec(ctx context.Context, dir string, name string, args ...string) error {
+	command := fmt.Sprintf("%s %s", name, strings.Join(args, " "))
+	a.Log.Info("executing command", "command", command, "dir", dir)
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Dir = dir
+	cmd.Stdout = a.Writer
+	cmd.Stderr = a.ErrWriter
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("execute command %s: %w", command, err)
+	}
+	return nil
+}
+
+func goModRoot() string {
+	_, file, _, ok := runtime.Caller(1)
+	if !ok {
+		return ""
+	}
+	return filepath.Dir(file)
+}

otlp-bench/main_test.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/urfave/cli/v3"
+)
+
+func TestVersionAdd(t *testing.T) {
+	for _, removeBefore := range []bool{true, false} {
+		builder := testAppBuilder{RemoveDirBefore: removeBefore, RemoveDirAfter: !removeBefore}
+		app := builder.Build(t)
+		err := app.Run(t.Context(), []string{"", "version", "add", "dict", "bcbee5a324cae805e296be1bf0e4edc8d6da6585", "https://github.com/florianl/opentelemetry-proto.git", "--verbose"})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestBench(t *testing.T) {
+	builder := testAppBuilder{}
+	app := builder.Build(t)
+	err := app.Run(t.Context(), []string{"", "bench", "testdata/profile.otlp", "--verbose"})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+type testAppBuilder struct {
+	RemoveDirBefore bool
+	RemoveDirAfter  bool
+}
+
+func (b testAppBuilder) Build(t *testing.T) *App {
+	t.Helper()
+	a := &App{
+		Writer:         t.Output(),
+		ErrWriter:      t.Output(),
+		VersionDir:     "testdata/versions",
+		ExitErrHandler: func(ctx context.Context, c *cli.Command, err error) {},
+	}
+	removeDir := func() {
+		if err := os.RemoveAll(a.VersionDir); err != nil {
+			t.Fatal(err)
+		}
+
+	}
+	if b.RemoveDirBefore {
+		removeDir()
+	}
+	if b.RemoveDirAfter {
+		t.Cleanup(func() {
+			if err := os.RemoveAll(a.VersionDir); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+	return a
+}