feat(go): add CSV rendering (#767)

Author: Proximyst

.github/workflows/docker-test.yaml

@@ -27,6 +27,7 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     permissions:
       contents: read # clone the repository
+      id-token: write # required to read secrets
     steps:
       - uses: actions/checkout@v4
         with:
@@ -43,12 +44,33 @@ jobs:
           BUILDKIT_STEP_LOG_MAX_SPEED: -1
           FILE: ${{ matrix.file }}
         run: docker build . -t image-renderer -f "$FILE"
+
+      - name: Read license
+        if: ${{ github.event.repository.fork == false }}
+        id: license_secret
+        uses: grafana/shared-workflows/actions/get-vault-secrets@a37de51f3d713a30a9e4b21bcdfbd38170020593 # get-vault-secrets/v1
+        with:
+          export_env: false
+          repo_secrets: |
+            LICENSE_JWT=grafana-enterprise-license:license-jwt
+      - name: Write license to file system
+        if: ${{ github.event.repository.fork == false }}
+        env:
+          SECRET: ${{ fromJson(steps.license_secret.outputs.secrets).LICENSE_JWT }}
+        run: |
+          set -euo pipefail
+          LPATH="$(mktemp --tmpdir XXXXXXXXXXXX.jwt)"
+          echo "$SECRET" > "$LPATH"
+          echo "LICENSE_JWT=$LPATH" >> "$GITHUB_ENV"
+
       - name: go test ./tests/acceptance/...
         run: go test ./tests/acceptance/... -count=1
         env:
           TERM: linux
           IMAGE: image-renderer
           UPDATE_FIXTURES: 'true' # this will make changes, so we can see them
+          REQUIRE_ACCEPTANCE: 'true' # fail if no image can be found
+          REQUIRE_ENTERPRISE: ${{ github.event.repository.fork == false }}
 
       - name: Tar changed files
         if: failure()

.gitignore

@@ -44,3 +44,6 @@ tests/testdata/diff_*
 cache
 
 /devenv/docker/tracing/tempo-data/
+
+# Grafana Enterprise licence keys
+*.jwt

docs/testing.md

@@ -30,4 +30,14 @@ Or you can also pull a specific image to test, for example:
 
 And then run the tests
 
-`IMAGE=grafana/grafana-image-renderer:v4.0.13 go test ./tests/acceptance/...`
+`IMAGE=grafana/grafana-image-renderer:v4.0.13 go test ./tests/acceptance/...`
+
+### Enterprise tests
+
+Some tests require an active Enterprise licence.
+
+If you're a Grafana Labs employee, you can find one of these in the `grafana-enterprise` repository:
+
+```shell
+$ ln -s ../grafana-enterprise/tools/license.jwt .
+```

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/urfave/cli-altsrc/v3 v3.0.1
 	github.com/urfave/cli/v3 v3.4.1
 	golang.org/x/sync v0.16.0
+	golang.org/x/text v0.28.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -82,7 +83,6 @@ require (
 	golang.org/x/image v0.30.0 // indirect
 	golang.org/x/mod v0.27.0 // indirect
 	golang.org/x/sys v0.35.0 // indirect
-	golang.org/x/text v0.28.0 // indirect
 	golang.org/x/tools v0.36.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/protobuf v1.36.7 // indirect

pkg/api/middleware/logger.go

@@ -0,0 +1,57 @@
+package middleware
+
+import (
+	"log/slog"
+	"net/http"
+	"sync"
+)
+
+func RequestLogger(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		lw := &loggingResponseWriter{w: w}
+		defer func() {
+			slog.DebugContext(r.Context(), "request complete",
+				"method", r.Method,
+				"mux_pattern", r.Pattern,
+				"uri", r.URL,
+				"status", lw.statusCode,
+				"status_text", http.StatusText(lw.statusCode))
+		}()
+		h.ServeHTTP(lw, r)
+	})
+}
+
+type loggingResponseWriter struct {
+	w          http.ResponseWriter
+	once       sync.Once
+	statusCode int
+}
+
+var (
+	_ http.ResponseWriter = (*loggingResponseWriter)(nil)
+	_ http.Flusher        = (*loggingResponseWriter)(nil)
+)
+
+func (l *loggingResponseWriter) Header() http.Header {
+	return l.w.Header()
+}
+
+func (l *loggingResponseWriter) WriteHeader(code int) {
+	l.once.Do(func() {
+		l.statusCode = code
+	})
+	l.w.WriteHeader(code)
+}
+
+func (l *loggingResponseWriter) Write(b []byte) (int, error) {
+	l.once.Do(func() {
+		l.statusCode = http.StatusOK
+	})
+	return l.w.Write(b)
+}
+
+func (l *loggingResponseWriter) Flush() {
+	if flusher, ok := l.w.(http.Flusher); ok {
+		flusher.Flush()
+	}
+}

pkg/api/mux.go

@@ -27,9 +27,11 @@ func NewHandler(
 	mux.Handle("GET /healthz", HandleGetHealthz())
 	mux.Handle("GET /version", HandleGetVersion(versions, browser))
 	mux.Handle("GET /render", middleware.RequireAuthToken(middleware.TrustedURL(HandlePostRender(browser)), string(token)))
+	mux.Handle("GET /render/csv", middleware.RequireAuthToken(middleware.TrustedURL(HandlePostRenderCSV(browser)), string(token)))
 	mux.Handle("GET /render/version", HandleGetRenderVersion(versions))
 
 	handler := middleware.RequestMetrics(mux)
+	handler = middleware.RequestLogger(handler)
 	handler = middleware.Recovery(handler) // must come last!
 	return handler, nil
 }

pkg/api/rendercsv.go

@@ -0,0 +1,75 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/grafana/grafana-image-renderer/pkg/service"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	// This also implicitly gives us a count for each result type, so we can calculate success rate.
+	MetricRenderCSVDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:        "http_render_csv_request_duration",
+		Help:        "How long does a single CSV render take?",
+		ConstLabels: prometheus.Labels{"unit": "seconds"},
+		Buckets:     []float64{0.5, 1, 3, 4, 5, 7, 9, 10, 11, 15, 19, 20, 21, 24, 27, 29, 30, 31, 35, 55, 95, 125, 305, 605},
+	}, []string{"result"})
+)
+
+func HandlePostRenderCSV(browser *service.BrowserService) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		url := r.URL.Query().Get("url")
+		if url == "" {
+			http.Error(w, "missing 'url' query parameter", http.StatusBadRequest)
+			return
+		}
+		if encoding := r.URL.Query().Get("encoding"); encoding != "" && encoding != "csv" {
+			http.Error(w, "invalid 'encoding' query parameter: must be 'csv' or empty/missing", http.StatusBadRequest)
+			return
+		}
+		ctx := r.Context()
+		if timeout := r.URL.Query().Get("timeout"); timeout != "" {
+			if regexpOnlyNumbers.MatchString(timeout) {
+				seconds, err := strconv.Atoi(timeout)
+				if err != nil {
+					http.Error(w, fmt.Sprintf("invalid 'timeout' query parameter: %v", err), http.StatusBadRequest)
+					return
+				}
+				timeoutCtx, cancelTimeout := context.WithTimeout(r.Context(), time.Duration(seconds)*time.Second)
+				defer cancelTimeout()
+				ctx = timeoutCtx
+			} else {
+				timeout, err := time.ParseDuration(timeout)
+				if err != nil {
+					http.Error(w, fmt.Sprintf("invalid 'timeout' query parameter: %v", err), http.StatusBadRequest)
+					return
+				}
+				timeoutCtx, cancelTimeout := context.WithTimeout(r.Context(), timeout)
+				defer cancelTimeout()
+				ctx = timeoutCtx
+			}
+		}
+		renderKey := r.URL.Query().Get("renderKey")
+		domain := r.URL.Query().Get("domain")
+
+		start := time.Now()
+		contents, err := browser.RenderCSV(ctx, url, renderKey, domain)
+		if err != nil {
+			MetricRenderCSVDuration.WithLabelValues("error").Observe(time.Since(start).Seconds())
+			http.Error(w, "CSV rendering failed", http.StatusInternalServerError)
+			slog.ErrorContext(ctx, "failed to render CSV", "err", err)
+			return
+		}
+		MetricRenderCSVDuration.WithLabelValues("success").Observe(time.Since(start).Seconds())
+
+		w.Header().Set("Content-Type", "text/csv")
+		w.Header().Set("Content-Disposition", `attachment; filename="data.csv"`)
+		_, _ = w.Write(contents)
+	})
+}

pkg/metrics/registry.go

@@ -21,6 +21,7 @@ func NewRegistry() *prometheus.Registry {
 		middleware.MetricTrustedURLRequests,
 
 		api.MetricRenderDuration,
+		api.MetricRenderCSVDuration,
 	)
 	return registry
 }

pkg/service/browser.go

@@ -8,10 +8,13 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/chromedp/cdproto/browser"
 	"github.com/chromedp/cdproto/emulation"
 	"github.com/chromedp/cdproto/network"
 	"github.com/chromedp/cdproto/page"
@@ -208,12 +211,8 @@ type renderingOptions struct {
 	landscape          bool
 }
 
-func (s *BrowserService) Render(ctx context.Context, url string, optionFuncs ...RenderingOption) ([]byte, string, error) {
-	if url == "" {
-		return nil, "text/plain", fmt.Errorf("url must not be empty")
-	}
-
-	opts := &renderingOptions{ // set sensible defaults here; we want all values filled in to show explicit intent
+func defaultRenderingOptions() *renderingOptions {
+	return &renderingOptions{ // set sensible defaults here; we want all values filled in to show explicit intent
 		gpu:                false,                 // assume no GPU: this can be heavy, and if it exists, it likely exists for AI/ML/transcoding/... purposes, not for us
 		sandbox:            false,                 // FIXME: enable this; <https://github.com/grafana/grafana-operator-experience-squad/issues/1460>
 		timezone:           time.UTC,              // UTC ensures consistency when it is not configured but the users' servers are in multiple locations
@@ -227,6 +226,14 @@ func (s *BrowserService) Render(ctx context.Context, url string, optionFuncs ...
 		printer:            defaultPDFPrinter(), // print as PDF if no other format is requested
 		landscape:          true,
 	}
+}
+
+func (s *BrowserService) Render(ctx context.Context, url string, optionFuncs ...RenderingOption) ([]byte, string, error) {
+	if url == "" {
+		return nil, "text/plain", fmt.Errorf("url must not be empty")
+	}
+
+	opts := defaultRenderingOptions()
 	for _, f := range s.defaultRenderingOptions {
 		if err := f(opts); err != nil {
 			return nil, "text/plain", fmt.Errorf("failed to apply default rendering option: %w", err)
@@ -284,6 +291,79 @@ func (s *BrowserService) Render(ctx context.Context, url string, optionFuncs ...
 	}
 }
 
+// RenderCSV visits a web page and downloads the CSV inside.
+//
+// You may be thinking: what the hell are we doing? Why are we using a browser for this?
+// The CSV endpoint just returns HTML. The actual query is done by the browser, and then a script _in the webpage_ downloads it as a CSV file.
+// This SHOULD be replaced at some point, such that the Grafana server does all the work; this is just not acceptable behaviour...
+func (s *BrowserService) RenderCSV(ctx context.Context, url, renderKey, domain string) ([]byte, error) {
+	if url == "" {
+		return nil, fmt.Errorf("url must not be empty")
+	}
+
+	traceID, err := getTraceID(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get trace ID: %w", err)
+	}
+	log := s.log.With("trace_id", traceID)
+
+	allocatorOptions, err := s.createAllocatorOptions(defaultRenderingOptions())
+	if err != nil {
+		return nil, fmt.Errorf("failed to create allocator options: %w", err)
+	}
+	allocatorCtx, cancelAllocator := chromedp.NewExecAllocator(ctx, allocatorOptions...)
+	defer cancelAllocator()
+	browserCtx, cancelBrowser := chromedp.NewContext(allocatorCtx, browserLoggers(ctx, log))
+	defer cancelBrowser()
+
+	tmpDir, err := os.MkdirTemp("", "gir-csv-"+traceID+"-*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary directory: %w", err)
+	}
+	defer func() {
+		if err := os.RemoveAll(tmpDir); err != nil {
+			log.WarnContext(ctx, "failed to remove temporary directory", "path", tmpDir, "error", err)
+		}
+	}()
+
+	actions := []chromedp.Action{
+		setCookies([]*network.SetCookieParams{
+			{
+				Name:   "renderKey",
+				Value:  renderKey,
+				Domain: domain,
+			},
+		}),
+		browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorAllow).WithDownloadPath(tmpDir),
+		chromedp.Navigate(url),
+		waitForViz(),
+		waitForDuration(time.Second),
+	}
+	if err := chromedp.Run(browserCtx, actions...); err != nil {
+		return nil, fmt.Errorf("failed to run browser: %w", err)
+	}
+
+	// Wait for the file to be downloaded.
+	var entries []os.DirEntry
+	for {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+
+		entries, err = os.ReadDir(tmpDir)
+		if err == nil && len(entries) > 0 {
+			break // file exists now
+		}
+	}
+
+	fileContents, err := os.ReadFile(filepath.Join(tmpDir, entries[0].Name()))
+	if err != nil {
+		return nil, fmt.Errorf("failed to read temporary file: %w", err)
+	}
+
+	return fileContents, nil
+}
+
 func getTraceID(context.Context) (string, error) {
 	// TODO: Use OTEL trace ID from context
 	id, err := uuid.NewRandom()