Carptenter: multiple renderers, filters, fixes. (#754)

Author: winder

.gitignore

@@ -8,7 +8,7 @@
 
 # Test binary, built with `go test -c`
 *.test
-carpenter
+/cmd/carpenter/carpenter
 
 # Test & linter reports
 *report.xml

cmd/carpenter/Makefile

@@ -14,3 +14,11 @@ lint-fix:
 	golangci-lint run -c ../../.golangci.yml --fix
 
 checks: test lint
+
+$(GOPATH)/bin/go-enum:
+	go get -u github.com/abice/go-enum
+	go install github.com/abice/go-enum
+
+.PHONY: generate
+generate: $(GOPATH)/bin/go-enum
+	go generate ./...

cmd/carpenter/command.go

@@ -5,153 +5,104 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"slices"
 	"strings"
-	"time"
 
-	"github.com/charmbracelet/lipgloss"
 	"github.com/urfave/cli/v3"
 
+	"github.com/smartcontractkit/chainlink-ccip/cmd/carpenter/internal/filter"
 	"github.com/smartcontractkit/chainlink-ccip/cmd/carpenter/internal/parse"
+	"github.com/smartcontractkit/chainlink-ccip/cmd/carpenter/internal/render"
 	"github.com/smartcontractkit/chainlink-ccip/cmd/carpenter/internal/stream"
 )
 
 type arguments struct {
-	files          []string
-	logType        string
-	disableFilters bool
-}
-
-// renderData
-/*
-
-2024-12-04T20:15:35Z | 1.1.1 |       Commit(MerkleRoot) | <processor details>
-                       | | |           |     |-- processor
-                       | | |           |-- OCR Plugin
-                       | | |-- sequence number
-                       | |-- DON ID
-                       -- oracleID
-
-*/
-func renderData(data *parse.Data) {
-	// simple color selection algorithm
-	withColor := func(in interface{}, i int) string {
-		color := fmt.Sprintf("%d", i%7+1)
-		str := fmt.Sprintf("%v", in)
-
-		return lipgloss.NewStyle().Foreground(lipgloss.Color(color)).Render(str)
-	}
-
-	var timeStyle = lipgloss.NewStyle().Width(10).Height(1).MaxHeight(1).
-		Align(lipgloss.Center)
-	var uidStyle = lipgloss.NewStyle().Width(25).Height(1).MaxHeight(1).
-		Align(lipgloss.Left).PaddingLeft(1).Bold(true)
-	var levelStyle = lipgloss.NewStyle().Width(4).Height(1).MaxHeight(1).
-		Align(lipgloss.Left).PaddingLeft(1).Italic(true)
-	var messageStyle = lipgloss.NewStyle().Width(60).Height(1).MaxHeight(1).
-		Align(lipgloss.Left).PaddingLeft(1)
-	var fieldsStyle = lipgloss.NewStyle().Width(100).Height(1).MaxHeight(1).
-		Align(lipgloss.Left).PaddingLeft(1)
+	files        []string
+	logType      parse.LogType
+	rendererName string
 
-	uid := fmt.Sprintf("%s.%s.%s.%s.%s",
-		withColor(data.OracleID, data.OracleID),
-		withColor(data.DONID, data.DONID),
-		withColor(data.SequenceNumber, data.SequenceNumber),
-		withColor(data.Component, 0),
-		withColor(data.OCRPhase, ocrPhaseToColor(data.OCRPhase)),
-	)
-
-	fmt.Printf("%s|%s|%s|%s|%s\n",
-		timeStyle.Render(data.Timestamp.Format(time.TimeOnly)),
-		uidStyle.Render(uid),
-		levelStyle.Render(truncateLevel(data.Level)),
-		messageStyle.Render(data.Message),
-		fieldsStyle.Render(getRelevantFieldsForMessage(data)),
-	)
-}
-
-func ocrPhaseToColor(phase string) int {
-	switch phase {
-	case "qry":
-		return 1
-	case "obs":
-		return 2
-	case "otcm":
-		return 3
-	case "rprt":
-		return 4
-	case "sacc":
-		return 5
-	case "strn":
-		return 6
-	default:
-		return 0
-	}
+	filter.CompiledFilterFields
+	filterOP filter.FilterOP
 }
 
-func truncateLevel(level string) string {
-	switch lv := strings.ToLower(level); lv {
-	case "info":
-		return "ifo"
-	case "debug":
-		return "dbg"
-	case "warn":
-		return "wrn"
-	case "error":
-		return "err"
-	case "critical":
-		return "crt"
-	default:
-		return "unk"
-	}
-}
-
-func getRelevantFieldsForMessage(data *parse.Data) string {
-	var fields string
-
-	if strings.ToLower(data.Level) == "error" {
-		fields = fmt.Sprintf("err=%v", data.RawLoggerFields["err"])
-	}
-
-	if strings.HasPrefix(data.Message, "failed to get token prices outcome") {
-		return fmt.Sprintf("err=%v", data.RawLoggerFields["err"])
-	}
-
-	if strings.HasPrefix(data.Message, "Get consensus observation failed, empty outcome") {
-		return fmt.Sprintf("err=%v", data.RawLoggerFields["err"])
-	}
-
-	if strings.HasPrefix(data.Message, "Sending Outcome") {
-		return fmt.Sprintf("nextState=%v outcome=%v",
-			data.RawLoggerFields["nextState"], data.RawLoggerFields["outcome"])
-	}
-
-	if strings.HasPrefix(data.Message, "sending merkle root processor observation") {
-		return fmt.Sprintf("observation=%v", data.RawLoggerFields["observation"])
-	}
-
-	if strings.HasPrefix(data.Message, "call to MsgsBetweenSeqNums returned unexpected") {
-		return fmt.Sprintf(
-			"%s expected=%v actual=%v chain=%v",
-			fields,
-			data.RawLoggerFields["expected"],
-			data.RawLoggerFields["actual"],
-			data.RawLoggerFields["chain"],
-		)
-	}
-	if strings.HasPrefix(data.Message, "queried messages between sequence numbers") {
-		return fmt.Sprintf("%s numMsgs=%v sourceChain=%v seqNumRange=%v",
-			fields,
-			data.RawLoggerFields["numMsgs"],
-			data.RawLoggerFields["sourceChainSelector"],
-			data.RawLoggerFields["seqNumRange"],
-		)
-	}
-	if strings.HasPrefix(data.Message, "decoded messages between sequence numbers") {
-		return fmt.Sprintf("%s sourceChain=%v seqNumRange=%v",
-			fields, data.RawLoggerFields["sourceChainSelector"], data.RawLoggerFields["seqNumRange"])
+func makeCommand() *cli.Command {
+	var args arguments
+	return &cli.Command{
+		Name:  "carpenter",
+		Usage: "A tool for parsing and displaying logs",
+		Flags: []cli.Flag{
+			&cli.StringSliceFlag{
+				Name:        "filename",
+				Usage:       "Provide one or more files to read. If not provided, reads from stdin.",
+				Destination: &args.files,
+			},
+			&cli.StringFlag{
+				Name:  "logType",
+				Usage: "Specify the type of log to parse, valid options: json, mixed, ci",
+				Value: "json",
+				Validator: func(s string) error {
+					var err error
+					args.logType, err = parse.ParseLogType(s)
+					if err != nil {
+						return fmt.Errorf("expected one of [%s]",
+							strings.Join(parse.LogTypeNames(), ", "))
+					}
+					return nil
+				},
+			},
+			&cli.StringFlag{
+				OnlyOnce:    true,
+				Name:        "renderer",
+				Usage:       fmt.Sprintf("Select which rendering algorithm to use: [%s]", strings.Join(render.GetRenderers(), ", ")),
+				Value:       "basic",
+				Destination: &args.rendererName,
+				Validator: func(s string) error {
+					choices := render.GetRenderers()
+					if !slices.Contains(choices, s) {
+						return fmt.Errorf("expected one of [%s]",
+							s, strings.Join(choices, ", "))
+					}
+					return nil
+				},
+			},
+			&cli.StringSliceFlag{
+				Name: "filter",
+				Usage: fmt.Sprintf(
+					"Line selection filters. Format as 'FieldName:Regexp', valid fields: [%s]",
+					strings.Join(filter.FieldNames(), ", ")),
+				//Destination: &args.FilterFields.Filters,
+				Category: "filters",
+				Validator: func(fields []string) error {
+					var err error
+					args.CompiledFilterFields, err = filter.NewFilterFields(fields)
+					if err != nil {
+						return err
+					}
+					return nil
+				},
+			},
+			&cli.StringFlag{
+				Name: "filter-op",
+				Usage: fmt.Sprintf(
+					"Operation to use when combining filters. Valid options: [%s]",
+					strings.Join(filter.FilterOPNames(), ", ")),
+				Category: "filters",
+				Value:    string(filter.FilterOPAND),
+				Validator: func(s string) error {
+					var err error
+					args.filterOP, err = filter.ParseFilterOP(s)
+					if err != nil {
+						return fmt.Errorf("expected one of %s", err,
+							strings.Join(filter.FilterOPNames(), ", "))
+					}
+					return nil
+				},
+			},
+		},
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			return run(args)
+		},
 	}
-
-	return ""
 }
 
 func run(args arguments) error {
@@ -162,6 +113,11 @@ func run(args arguments) error {
 		io.Filenames = args.files
 	}
 
+	renderer, err := render.GetRenderer(args.rendererName, render.Options{})
+	if err != nil {
+		return fmt.Errorf("failed to get renderer: %w", err)
+	}
+
 	inputStream, err := stream.InitializeInputStream(io)
 	if err != nil {
 		return fmt.Errorf("failed to initialize input stream: %w", err)
@@ -170,59 +126,26 @@ func run(args arguments) error {
 	scanner := bufio.NewScanner(inputStream)
 	for scanner.Scan() {
 		line := scanner.Text()
-		data, err := parse.Filter(line, args.logType, args.disableFilters)
+		data, err := parse.ParseLine(line, args.logType)
+		if err != nil {
+			return fmt.Errorf("ParseLine: %w", err)
+		}
+
+		include, err := filter.Filter(data, args.CompiledFilterFields, args.filterOP)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "Unable to get data: %s\n", err)
+			msg := fmt.Sprintf("Unable to get data: %s\n", err)
+			_, err2 := fmt.Fprintf(os.Stderr, msg)
+			if err2 != nil {
+				panic(msg)
+			}
 			return err
 		}
-		if data == nil {
+		if !include {
 			// no data to display.
 			continue
 		}
 
-		renderData(data)
-		//fmt.Println(data)
+		renderer(data)
 	}
 	return nil
 }
-
-func makeCommand() *cli.Command {
-	var args arguments
-	return &cli.Command{
-		Name:  "carpenter",
-		Usage: "A tool for parsing and displaying logs",
-		Flags: []cli.Flag{
-			&cli.StringSliceFlag{
-				Name:        "filename",
-				Usage:       "Provide one or more files to read. If not provided, reads from stdin.",
-				Destination: &args.files,
-			},
-			&cli.StringFlag{
-				Name:        "logType",
-				Usage:       "Specify the type of log to parse, valid options: json, mixed, ci",
-				Destination: &args.logType,
-				Required:    true,
-				Validator: func(s string) error {
-					if !parse.IsValidLogType(s) {
-						return fmt.Errorf("invalid log type: %s, expected either %s or %s or %s",
-							s,
-							parse.LogTypeJSON,
-							parse.LogTypeMixed,
-							parse.LogTypeMixedGoTestJSON,
-						)
-					}
-					return nil
-				},
-			},
-			&cli.BoolFlag{
-				Name:        "disableFilters",
-				Usage:       "Set to disable filter application on the logs. Defaults to false.",
-				Destination: &args.disableFilters,
-				Required:    false,
-			},
-		},
-		Action: func(ctx context.Context, cmd *cli.Command) error {
-			return run(args)
-		},
-	}
-}

cmd/carpenter/go.mod

@@ -1,6 +1,6 @@
 module github.com/smartcontractkit/chainlink-ccip/cmd/carpenter
 
-go 1.23.3
+go 1.24.0
 
 require (
 	github.com/charmbracelet/lipgloss v1.0.0
@@ -11,6 +11,7 @@ require (
 require (
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/charmbracelet/x/ansi v0.4.2 // indirect
+	github.com/charmbracelet/x/term v0.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect

cmd/carpenter/go.sum

@@ -4,6 +4,8 @@ github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O
 github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo=
 github.com/charmbracelet/x/ansi v0.4.2 h1:0JM6Aj/g/KC154/gOP4vfxun0ff6itogDYk41kof+qk=
 github.com/charmbracelet/x/ansi v0.4.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
+github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
+github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=