chore: add logs, har and config data in html output (#1946)

* chore: add logs, har and config data in html output

* chore: add logs, har and config data in html output

* chore: keep scraper id empty for tests

Author: yashmehrotra

api/context.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/flanksource/commons/har"
 	"github.com/flanksource/commons/logger"
 	v1 "github.com/flanksource/config-db/api/v1"
 	dutyCtx "github.com/flanksource/duty/context"
@@ -20,6 +21,7 @@ type ScrapeContext struct {
 
 	isIncremental bool
 	debugRun      bool
+	harCollector  *har.Collector
 
 	namespace string
 
@@ -111,6 +113,7 @@ func (ctx ScrapeContext) WithValue(key, val any) ScrapeContext {
 		temp:              ctx.temp,
 		isIncremental:     ctx.isIncremental,
 		debugRun:          ctx.debugRun,
+		harCollector:      ctx.harCollector,
 		namespace:         ctx.namespace,
 		jobHistory:        ctx.jobHistory,
 		scrapeConfig:      ctx.scrapeConfig,
@@ -215,3 +218,12 @@ func (ctx ScrapeContext) AsDebugRun(level string) ScrapeContext {
 	}
 	return ctx
 }
+
+func (ctx ScrapeContext) WithHARCollector(collector *har.Collector) ScrapeContext {
+	ctx.harCollector = collector
+	return ctx
+}
+
+func (ctx ScrapeContext) HARCollector() *har.Collector {
+	return ctx.harCollector
+}

api/v1/interface.go

@@ -4,12 +4,15 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"regexp"
 	"strings"
 	"time"
 
 	"github.com/flanksource/clicky"
 	"github.com/flanksource/clicky/api"
 	"github.com/flanksource/commons/collections/set"
+	"github.com/flanksource/commons/har"
+	"gopkg.in/yaml.v3"
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty"
 	"github.com/flanksource/duty/models"
@@ -1002,6 +1005,7 @@ func (s ScrapeResult) Columns() []api.ColumnDef {
 		clicky.Column("Name").Build(),
 		clicky.Column("Type").Build(),
 		clicky.Column("Health").Build(),
+		clicky.Column("Details").Build(),
 		clicky.Column("Error").Build(),
 	}
 
@@ -1012,6 +1016,7 @@ func (s ScrapeResult) Row() map[string]any {
 	row["Name"] = clicky.Text(s.Name)
 	row["Type"] = clicky.Text(s.Type)
 	row["Health"] = clicky.Text(string(s.Health))
+	row["Details"] = s.configDetails()
 	if s.Error != nil {
 		row["Error"] = clicky.Text(s.Error.Error())
 	} else {
@@ -1020,6 +1025,204 @@ func (s ScrapeResult) Row() map[string]any {
 	return row
 }
 
+func (s ScrapeResult) configDetails() api.Collapsed {
+	if s.Config == nil {
+		return clicky.Collapsed("empty", clicky.Text(""))
+	}
+
+	var data any
+	switch v := s.Config.(type) {
+	case string:
+		if json.Unmarshal([]byte(v), &data) != nil {
+			data = v
+		}
+	default:
+		data = v
+	}
+
+	b, err := yaml.Marshal(data)
+	if err != nil {
+		b = []byte(fmt.Sprintf("%v", s.Config))
+	}
+	yamlStr := string(b)
+
+	content := clicky.Text("")
+	if len(s.Labels) > 0 {
+		content = content.Append("Labels: ", "text-gray-500 font-medium").Append(clicky.Map(s.Labels, "badge")).NewLine()
+	}
+	if len(s.Tags) > 0 {
+		content = content.Append("Tags: ", "text-gray-500 font-medium").Append(clicky.Map(s.Tags, "badge")).NewLine()
+	}
+	content = content.Append(clicky.CodeBlock("yaml", yamlStr), "min-w-[600px] block")
+
+	label := fmt.Sprintf("Config (%d bytes)", len(yamlStr))
+	return clicky.Collapsed(label, content)
+}
+
+// CountsGrid renders scrape result counts as a 2-column grid.
+// +kubebuilder:object:generate=false
+type CountsGrid []countEntry
+
+type countEntry struct {
+	label string
+	count int
+}
+
+func (g CountsGrid) HTML() string {
+	var b strings.Builder
+	b.WriteString(`<div class="grid grid-cols-2 gap-x-8 gap-y-2">`)
+	for _, e := range g {
+		fmt.Fprintf(&b,
+			`<div class="flex justify-between px-3 py-1 bg-gray-50 rounded">`+
+				`<span class="text-sm font-medium text-gray-500">%s</span>`+
+				`<span class="text-sm text-gray-900">%d</span>`+
+				`</div>`, e.label, e.count)
+	}
+	b.WriteString(`</div>`)
+	return b.String()
+}
+
+func (g CountsGrid) String() string {
+	var parts []string
+	for _, e := range g {
+		parts = append(parts, fmt.Sprintf("%s: %d", e.label, e.count))
+	}
+	return strings.Join(parts, ", ")
+}
+
+func (g CountsGrid) ANSI() string    { return g.String() }
+func (g CountsGrid) Markdown() string { return g.String() }
+
+// BuildCounts returns scrape result counts as a 2-column grid.
+func BuildCounts(all FullScrapeResults) CountsGrid {
+	return CountsGrid{
+		{"Configs", len(all.Configs)},
+		{"Analysis", len(all.Analysis)},
+		{"Changes", len(all.Changes)},
+		{"Relationships", len(all.Relationships)},
+		{"External Roles", len(all.ExternalRoles)},
+		{"External Users", len(all.ExternalUsers)},
+		{"External Groups", len(all.ExternalGroups)},
+		{"External User Groups", len(all.ExternalUserGroups)},
+		{"Config Access", len(all.ConfigAccess)},
+		{"Config Access Logs", len(all.ConfigAccessLogs)},
+	}
+}
+
+var ansiEscapeRegex = regexp.MustCompile(`\x1b\[[0-9;]*m`)
+
+// LogLine is a single log entry that renders as a table row with colored level prefix.
+// +kubebuilder:object:generate=false
+type LogLine struct {
+	text api.Text
+}
+
+func (l LogLine) Columns() []api.ColumnDef {
+	return []api.ColumnDef{
+		clicky.Column("Line").Build(),
+	}
+}
+
+func (l LogLine) Row() map[string]any {
+	return map[string]any{"Line": l.text}
+}
+
+// BuildLogLines parses raw log text into LogLine rows for table rendering.
+func BuildLogLines(rawLogs string) []LogLine {
+	cleaned := ansiEscapeRegex.ReplaceAllString(rawLogs, "")
+	lines := strings.Split(strings.TrimRight(cleaned, "\n"), "\n")
+	if len(lines) == 0 || (len(lines) == 1 && lines[0] == "") {
+		return nil
+	}
+
+	out := make([]LogLine, 0, len(lines))
+	for _, line := range lines {
+		out = append(out, LogLine{text: colorLogLine(line)})
+	}
+	return out
+}
+
+var logLevelRegex = regexp.MustCompile(`\s(INF|ERR|WRN|DBG\S*|TRC\S*|FTL)\s`)
+
+var logLevelColors = map[string]string{
+	"INF": "text-green-600",
+	"ERR": "text-red-600",
+	"WRN": "text-yellow-600",
+	"DBG": "text-blue-600",
+	"TRC": "text-gray-500",
+	"FTL": "text-red-600",
+}
+
+// colorLogLine highlights the log level prefix with appropriate colors.
+// Matches DBG, DBG-1, TRC-2, etc.
+func colorLogLine(line string) api.Text {
+	loc := logLevelRegex.FindStringIndex(line)
+	if loc == nil {
+		return clicky.Text(line)
+	}
+
+	// loc covers the match including surrounding spaces
+	tag := strings.TrimSpace(line[loc[0]:loc[1]])
+	before := line[:loc[0]+1]
+	after := line[loc[1]-1:]
+
+	// Base level is the prefix before any dash (DBG-1 → DBG)
+	base := tag
+	if i := strings.IndexByte(tag, '-'); i >= 0 {
+		base = tag[:i]
+	}
+	color := logLevelColors[base]
+
+	return clicky.Text(before).Append(tag, color).Append(after)
+}
+
+// HAREntry renders a single HAR request/response as a table row.
+// +kubebuilder:object:generate=false
+type HAREntry struct {
+	Method   string
+	URL      string
+	Status   int
+	Duration string
+}
+
+func (h HAREntry) Columns() []api.ColumnDef {
+	return []api.ColumnDef{
+		clicky.Column("Method").Build(),
+		clicky.Column("URL").Build(),
+		clicky.Column("Status").Build(),
+		clicky.Column("Duration").Build(),
+	}
+}
+
+func (h HAREntry) Row() map[string]any {
+	statusStyle := "text-green-600"
+	if h.Status >= 400 {
+		statusStyle = "text-red-600"
+	} else if h.Status >= 300 {
+		statusStyle = "text-yellow-600"
+	}
+	return map[string]any{
+		"Method":   clicky.Text(h.Method),
+		"URL":      clicky.Text(h.URL),
+		"Status":   clicky.Text(fmt.Sprintf("%d", h.Status)).WithStyles(statusStyle),
+		"Duration": clicky.Text(h.Duration),
+	}
+}
+
+// BuildHAREntries converts HAR entries into table rows.
+func BuildHAREntries(entries []har.Entry) []HAREntry {
+	out := make([]HAREntry, 0, len(entries))
+	for _, e := range entries {
+		out = append(out, HAREntry{
+			Method:   e.Request.Method,
+			URL:      e.Request.URL,
+			Status:   e.Response.Status,
+			Duration: fmt.Sprintf("%.0fms", e.Time),
+		})
+	}
+	return out
+}
+
 func (s ScrapeResult) IsMetadataOnly() bool {
 	return s.Config == nil
 }