From 6b9b688f8595a965849efc6cd1207b2dfa98cb9c Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 8 Jul 2025 16:53:34 +0200 Subject: [PATCH 1/4] port changes from devenv back, make observability stack exportable --- framework/cmd/blockscout.go | 46 --- framework/cmd/ci.go | 362 ------------------ framework/cmd/ci_fake.go | 50 --- framework/cmd/ci_test.go | 258 ------------- framework/cmd/main.go | 133 +------ framework/cmd/observability.go | 82 ---- framework/config.go | 65 +++- framework/observability.go | 150 ++++++++ .../blockscout/docker-compose.yml | 24 +- .../blockscout/envs/common-blockscout.env | 0 .../blockscout/envs/common-frontend.env | 0 .../envs/common-smart-contract-verifier.env | 0 .../blockscout/envs/common-stats.env | 0 .../envs/common-user-ops-indexer.env | 0 .../blockscout/envs/common-visualizer.env | 0 .../blockscout/proxy/default.conf.template | 0 .../proxy/microservices.conf.template | 0 .../blockscout/services/backend.yml | 2 +- .../observability/blockscout/services/db.yml | 0 .../blockscout/services/frontend.yml | 2 +- .../blockscout/services/nginx.yml | 0 .../blockscout/services/redis.yml | 0 .../blockscout/services/sig-provider.yml | 0 .../services/smart-contract-verifier.yml | 2 +- .../blockscout/services/stats.yml | 2 +- .../blockscout/services/user-ops-indexer.yml | 2 +- .../blockscout/services/visualizer.yml | 2 +- .../observability/compose/conf/defaults.ini | 0 .../observability/compose/conf/grafana.ini | 0 .../observability/compose/conf/prometheus.yml | 0 .../provisioning/access-control/sample.yaml | 0 .../dashboards/cadvisor/cadvisor.json | 0 .../dashboards/clnode-errors/errors.json | 0 .../provisioning/dashboards/dashboards.yaml | 0 .../conf/provisioning/dashboards/pg/pg.json | 0 .../provisioning/dashboards/wasp/wasp.json | 0 .../dashboards/workflow-engine/engine.json | 0 .../conf/provisioning/datasources/loki.yaml | 0 .../conf/provisioning/notifiers/sample.yaml | 0 .../conf/provisioning/plugins/sample.yaml | 0 .../compose/conf/provisioning/rules/rules.yml | 0 .../observability/compose/docker-compose.yaml | 0 .../observability/compose/loki-config.yaml | 0 .../{cmd => }/observability/compose/otel.yaml | 0 .../observability/compose/tempo.yaml | 0 45 files changed, 232 insertions(+), 950 deletions(-) delete mode 100644 framework/cmd/blockscout.go delete mode 100644 framework/cmd/ci.go delete mode 100644 framework/cmd/ci_fake.go delete mode 100644 framework/cmd/ci_test.go delete mode 100644 framework/cmd/observability.go create mode 100644 framework/observability.go rename framework/{cmd => }/observability/blockscout/docker-compose.yml (75%) rename framework/{cmd => }/observability/blockscout/envs/common-blockscout.env (100%) rename framework/{cmd => }/observability/blockscout/envs/common-frontend.env (100%) rename framework/{cmd => }/observability/blockscout/envs/common-smart-contract-verifier.env (100%) rename framework/{cmd => }/observability/blockscout/envs/common-stats.env (100%) rename framework/{cmd => }/observability/blockscout/envs/common-user-ops-indexer.env (100%) rename framework/{cmd => }/observability/blockscout/envs/common-visualizer.env (100%) rename framework/{cmd => }/observability/blockscout/proxy/default.conf.template (100%) rename framework/{cmd => }/observability/blockscout/proxy/microservices.conf.template (100%) rename framework/{cmd => }/observability/blockscout/services/backend.yml (91%) rename framework/{cmd => }/observability/blockscout/services/db.yml (100%) rename framework/{cmd => }/observability/blockscout/services/frontend.yml (84%) rename framework/{cmd => }/observability/blockscout/services/nginx.yml (100%) rename framework/{cmd => }/observability/blockscout/services/redis.yml (100%) rename framework/{cmd => }/observability/blockscout/services/sig-provider.yml (100%) rename framework/{cmd => }/observability/blockscout/services/smart-contract-verifier.yml (82%) rename framework/{cmd => }/observability/blockscout/services/stats.yml (97%) rename framework/{cmd => }/observability/blockscout/services/user-ops-indexer.yml (93%) rename framework/{cmd => }/observability/blockscout/services/visualizer.yml (84%) rename framework/{cmd => }/observability/compose/conf/defaults.ini (100%) rename framework/{cmd => }/observability/compose/conf/grafana.ini (100%) rename framework/{cmd => }/observability/compose/conf/prometheus.yml (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/access-control/sample.yaml (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/dashboards/cadvisor/cadvisor.json (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/dashboards/clnode-errors/errors.json (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/dashboards/dashboards.yaml (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/dashboards/pg/pg.json (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/dashboards/wasp/wasp.json (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/dashboards/workflow-engine/engine.json (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/datasources/loki.yaml (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/notifiers/sample.yaml (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/plugins/sample.yaml (100%) rename framework/{cmd => }/observability/compose/conf/provisioning/rules/rules.yml (100%) rename framework/{cmd => }/observability/compose/docker-compose.yaml (100%) rename framework/{cmd => }/observability/compose/loki-config.yaml (100%) rename framework/{cmd => }/observability/compose/otel.yaml (100%) rename framework/{cmd => }/observability/compose/tempo.yaml (100%) diff --git a/framework/cmd/blockscout.go b/framework/cmd/blockscout.go deleted file mode 100644 index 92d55179b..000000000 --- a/framework/cmd/blockscout.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/smartcontractkit/chainlink-testing-framework/framework" -) - -func blockscoutUp(url string) error { - framework.L.Info().Msg("Creating local Blockscout stack") - if err := extractAllFiles("observability"); err != nil { - return err - } - os.Setenv("BLOCKSCOUT_RPC_URL", url) - err := framework.RunCommand("bash", "-c", fmt.Sprintf(` - cd %s && \ - docker compose up -d - `, "blockscout")) - if err != nil { - return err - } - fmt.Println() - framework.L.Info().Msgf("Blockscout is up at: %s", "http://localhost") - return nil -} - -func blockscoutDown(url string) error { - framework.L.Info().Msg("Removing local Blockscout stack") - os.Setenv("BLOCKSCOUT_RPC_URL", url) - err := framework.RunCommand("bash", "-c", fmt.Sprintf(` - cd %s && \ - docker compose down -v - `, "blockscout")) - if err != nil { - return err - } - return framework.RunCommand("bash", "-c", fmt.Sprintf(` - cd %s && \ - rm -rf blockscout-db-data && \ - rm -rf logs && \ - rm -rf redis-data && \ - rm -rf stats-db-data - `, filepath.Join("blockscout", "services"))) -} diff --git a/framework/cmd/ci.go b/framework/cmd/ci.go deleted file mode 100644 index 6f4d74936..000000000 --- a/framework/cmd/ci.go +++ /dev/null @@ -1,362 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" - - "github.com/fatih/color" - "github.com/google/go-github/v50/github" - "github.com/google/uuid" - "go.uber.org/ratelimit" - "golang.org/x/oauth2" - "golang.org/x/sync/errgroup" - - "github.com/smartcontractkit/chainlink-testing-framework/framework" -) - -const ( - JobsRateLimitPerSecond = 20 - GHResultsPerPage = 100 // anything above that won't work, check GitHub docs -) - -const ( - MaxNameLen = 120 - SuccessEmoji = "✅" - FailureEmoji = "❌" - RerunEmoji = "🔄" - CancelledEmoji = "🚫" -) - -var ( - SlowTestThreshold = 5 * time.Minute - ExtremelySlowTestThreshold = 10 * time.Minute - - DebugDirRoot = "ctf-ci-debug" - DebugSubDirWF = filepath.Join(DebugDirRoot, "workflows") - DebugSubDirJobs = filepath.Join(DebugDirRoot, "jobs") - DefaultResultsDir = "." - DefaultResultsFile = "ctf-ci" -) - -type GitHubActionsClient interface { - ListRepositoryWorkflowRuns(ctx context.Context, owner, repo string, opts *github.ListWorkflowRunsOptions) (*github.WorkflowRuns, *github.Response, error) - ListWorkflowJobs(ctx context.Context, owner, repo string, runID int64, opts *github.ListWorkflowJobsOptions) (*github.Jobs, *github.Response, error) -} - -type AnalysisConfig struct { - Debug bool `json:"debug"` - Owner string `json:"owner"` - Repo string `json:"repo"` - WorkflowName string `json:"workflow_name"` - TimeDaysBeforeStart int `json:"time_days_before_start"` - TimeStart time.Time `json:"time_start"` - TimeDaysBeforeEnd int `json:"time_days_before_end"` - TimeEnd time.Time `json:"time_end"` - Typ string `json:"type"` - ResultsFile string `json:"results_file"` -} - -type Stat struct { - Name string `json:"name"` - Successes int `json:"successes"` - Failures int `json:"failures"` - Cancels int `json:"cancels"` - ReRuns int `json:"reRuns"` - P50 time.Duration `json:"p50"` - P95 time.Duration `json:"p95"` - P99 time.Duration `json:"p99"` - TotalDuration time.Duration `json:"totalDuration"` - Durations []time.Duration `json:"-"` -} - -type Stats struct { - Mu *sync.Mutex `json:"-"` - Runs int `json:"runs"` - CancelledRuns int `json:"cancelled_runs"` - IgnoredRuns int `json:"ignored_runs"` - Jobs map[string]*Stat `json:"jobs"` - Steps map[string]*Stat `json:"steps"` -} - -func AnalyzeJobsSteps(ctx context.Context, client GitHubActionsClient, cfg *AnalysisConfig) (*Stats, error) { - framework.L.Info().Time("From", cfg.TimeStart).Time("To", cfg.TimeEnd).Msg("Analyzing workflow runs") - opts := &github.ListWorkflowRunsOptions{ - Created: fmt.Sprintf("%s..%s", cfg.TimeStart.Format(time.DateOnly), cfg.TimeEnd.Format(time.DateOnly)), - ListOptions: github.ListOptions{PerPage: GHResultsPerPage}, - } - rlJobs := ratelimit.New(JobsRateLimitPerSecond) - stats := &Stats{ - Mu: &sync.Mutex{}, - Jobs: make(map[string]*Stat), - Steps: make(map[string]*Stat), - } - refreshDebugDirs() - for { - runs, resp, err := client.ListRepositoryWorkflowRuns(ctx, cfg.Owner, cfg.Repo, opts) - if err != nil { - return nil, fmt.Errorf("failed to fetch workflow runs: %w", err) - } - framework.L.Debug().Int("Runs", len(runs.WorkflowRuns)).Msg("Loading runs") - - eg := &errgroup.Group{} - for _, wr := range runs.WorkflowRuns { - framework.L.Debug().Str("Name", *wr.Name).Msg("Analyzing workflow run") - if !strings.Contains(*wr.Name, cfg.WorkflowName) { - stats.IgnoredRuns++ - continue - } - stats.Runs++ - // analyze workflow - name := *wr.Name - framework.L.Debug().Str("Name", name).Msg("Analyzing workflow run") - _ = dumpResults(cfg.Debug, DebugSubDirWF, name, wr) - eg.Go(func() error { - rlJobs.Take() - jobs, _, err := client.ListWorkflowJobs(ctx, cfg.Owner, cfg.Repo, *wr.ID, &github.ListWorkflowJobsOptions{ - ListOptions: github.ListOptions{PerPage: GHResultsPerPage}, - }) - if err != nil { - return err - } - // analyze jobs - stats.Mu.Lock() - defer stats.Mu.Unlock() - for _, j := range jobs.Jobs { - name := *j.Name - _ = dumpResults(cfg.Debug, DebugSubDirJobs, name, wr) - if skippedOrInProgressJob(j) { - stats.IgnoredRuns++ - continue - } - if j.Status != nil && *j.Status == "cancelled" { - stats.CancelledRuns++ - continue - } - dur := j.CompletedAt.Time.Sub(j.StartedAt.Time) - if _, ok := stats.Jobs[name]; !ok { - stats.Jobs[name] = &Stat{ - Name: name, - Durations: []time.Duration{dur}, - TotalDuration: dur, - } - } else { - stats.Jobs[name].Durations = append(stats.Jobs[*j.Name].Durations, dur) - stats.Jobs[name].TotalDuration += dur - } - if j.RunAttempt != nil && *j.RunAttempt > 1 { - stats.Jobs[name].ReRuns++ - } - if j.Conclusion != nil && *j.Conclusion == "failure" { - stats.Jobs[name].Failures++ - } else { - stats.Jobs[name].Successes++ - } - if j.Conclusion != nil && *j.Conclusion == "cancelled" { - stats.Jobs[name].Cancels++ - } - // analyze steps - for _, s := range j.Steps { - name := *s.Name - if skippedOrInProgressStep(s) { - continue - } - dur := s.CompletedAt.Time.Sub(s.StartedAt.Time) - if _, ok := stats.Steps[name]; !ok { - stats.Steps[name] = &Stat{ - Name: name, - Durations: []time.Duration{dur}, - } - } else { - stats.Steps[name].Durations = append(stats.Steps[name].Durations, dur) - } - if *s.Conclusion == "failure" { - stats.Steps[name].Failures++ - } else { - stats.Steps[name].Successes++ - } - } - } - return nil - }) - } - if err := eg.Wait(); err != nil { - return nil, err - } - if resp.NextPage == 0 { - break - } - opts.Page = resp.NextPage - } - jobs := make([]*Stat, 0) - for _, js := range stats.Jobs { - stat := calculatePercentiles(js) - jobs = append(jobs, stat) - } - sort.Slice(jobs, func(i, j int) bool { return jobs[i].P50 > jobs[j].P50 }) - - steps := make([]*Stat, 0) - for _, js := range stats.Steps { - stat := calculatePercentiles(js) - steps = append(steps, stat) - } - sort.Slice(steps, func(i, j int) bool { return steps[i].P50 > steps[j].P50 }) - - switch cfg.Typ { - case "jobs": - for _, js := range jobs { - printSummary(js.Name, js, cfg.Typ) - } - case "steps": - for _, js := range steps { - printSummary(js.Name, js, cfg.Typ) - } - default: - return nil, errors.New("analytics type is not recognized") - } - framework.L.Info(). - Int("Runs", stats.Runs). - Int("Cancelled", stats.CancelledRuns). - Int("Ignored", stats.IgnoredRuns). - Msg("Total runs analyzed") - return stats, nil -} - -func AnalyzeCIRuns(cfg *AnalysisConfig) (*Stats, error) { - ctx := context.Background() - token := os.Getenv("GITHUB_TOKEN") - if token == "" { - return nil, fmt.Errorf("GITHUB_TOKEN environment variable is not set") - } - framework.L.Info(). - Str("Owner", cfg.Owner). - Str("Repo", cfg.Repo). - Str("Workflow", cfg.WorkflowName). - Msg("Analyzing CI runs") - - ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) - tc := oauth2.NewClient(ctx, ts) - client := github.NewClient(tc) - - // Fetch workflow runs for the last N days - // have GH rate limits in mind, see file constants - timeStart := time.Now().AddDate(0, 0, -cfg.TimeDaysBeforeStart) - cfg.TimeStart = timeStart - timeEnd := time.Now().AddDate(0, 0, -cfg.TimeDaysBeforeEnd) - cfg.TimeEnd = timeEnd - stats, err := AnalyzeJobsSteps(ctx, client.Actions, cfg) - if err != nil { - return nil, fmt.Errorf("failed to fetch workflow runs: %w", err) - } - if cfg.ResultsFile != "" { - _ = dumpResults(true, DefaultResultsDir, DefaultResultsFile, stats) - } - return stats, nil -} - -// getColor returns a color printer based on the duration -func getColor(duration time.Duration) *color.Color { - switch { - case duration < SlowTestThreshold: - return color.New(color.FgGreen) - case duration < ExtremelySlowTestThreshold: - return color.New(color.FgYellow) - default: - return color.New(color.FgRed) - } -} - -func printSummary(name string, s *Stat, typ string) { - cp50 := getColor(s.P50) - cp95 := getColor(s.P95) - cp99 := getColor(s.P99) - var cpFlaky *color.Color - if s.ReRuns > 0 { - cpFlaky = color.New(color.FgRed) - } else { - cpFlaky = color.New(color.FgGreen) - } - if len(name) > MaxNameLen { - name = name[:MaxNameLen] - } - switch typ { - case "jobs": - fmt.Printf("%s 50th:%-10s 95th:%-10s 99th:%-10s Total:%-10s %s %-2s %s %-2s %s %-2s %s %-2s\n", - cp50.Sprintf("%-120s", name), - cp50.Sprintf("%-8s", s.P50), - cp95.Sprintf("%-8s", s.P95), - cp99.Sprintf("%-8s", s.P99), - fmt.Sprintf("%-8s", s.TotalDuration.Round(time.Second)), - RerunEmoji, - cpFlaky.Sprintf("%-2d", s.ReRuns), - FailureEmoji, - cpFlaky.Sprintf("%-2d", s.Failures), - SuccessEmoji, - cpFlaky.Sprintf("%-2d", s.Successes), - CancelledEmoji, - cpFlaky.Sprintf("%-2d", s.Cancels), - ) - case "steps": - fmt.Printf("%s 50th:%-10s 95th:%-10s 99th:%-10s %s %-2s %s %-2s\n", - cp50.Sprintf("%-120s", name), - cp50.Sprintf("%-8s", s.P50), - cp95.Sprintf("%-8s", s.P95), - cp99.Sprintf("%-8s", s.P99), - FailureEmoji, - cpFlaky.Sprintf("%-2d", s.Failures), - SuccessEmoji, - cpFlaky.Sprintf("%-2d", s.Successes), - ) - } -} - -func skippedOrInProgressStep(s *github.TaskStep) bool { - if s.Conclusion == nil || s.CompletedAt == nil || *s.Conclusion == "skipped" { - return true - } - return false -} - -func skippedOrInProgressJob(s *github.WorkflowJob) bool { - if s.Conclusion == nil || s.CompletedAt == nil || (*s.Conclusion == "skipped") { - return true - } - return false -} - -func calculatePercentiles(stat *Stat) *Stat { - sort.Slice(stat.Durations, func(i, j int) bool { return stat.Durations[i] < stat.Durations[j] }) - q := func(d *Stat, quantile float64) int { - return int(float64(len(d.Durations)) * quantile / 100) - } - stat.P50 = stat.Durations[q(stat, 50)].Round(time.Second) - stat.P95 = stat.Durations[q(stat, 95)].Round(time.Second) - stat.P99 = stat.Durations[q(stat, 99)].Round(time.Second) - return stat -} - -func refreshDebugDirs() { - _ = os.RemoveAll(DebugDirRoot) - if _, err := os.Stat(DebugDirRoot); os.IsNotExist(err) { - _ = os.MkdirAll(DebugSubDirWF, os.ModePerm) - _ = os.MkdirAll(DebugSubDirJobs, os.ModePerm) - } -} - -func dumpResults(enabled bool, dir, name string, data interface{}) error { - if enabled { - d, err := json.MarshalIndent(data, "", " ") - if err != nil { - return err - } - return os.WriteFile(fmt.Sprintf("%s/%s-%s.json", dir, name, uuid.NewString()[0:5]), d, os.ModeAppend|os.ModePerm) - } - return nil -} diff --git a/framework/cmd/ci_fake.go b/framework/cmd/ci_fake.go deleted file mode 100644 index 906a259c8..000000000 --- a/framework/cmd/ci_fake.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "context" - "errors" - - "github.com/google/go-github/v50/github" -) - -type FakeGHA struct { - workflowRuns []*github.WorkflowRun - workflowJobs []*github.WorkflowJob -} - -func NewFakeGHA() *FakeGHA { - return &FakeGHA{ - workflowRuns: make([]*github.WorkflowRun, 0), - workflowJobs: make([]*github.WorkflowJob, 0), - } -} - -func (c *FakeGHA) WorkflowRun(run *github.WorkflowRun) { - c.workflowRuns = append(c.workflowRuns, run) -} - -func (c *FakeGHA) Job(jobs ...*github.WorkflowJob) { - c.workflowJobs = append(c.workflowJobs, jobs...) -} - -func (c *FakeGHA) ListRepositoryWorkflowRuns(_ context.Context, _, _ string, _ *github.ListWorkflowRunsOptions) (*github.WorkflowRuns, *github.Response, error) { - total := len(c.workflowRuns) - workflowRuns := &github.WorkflowRuns{ - TotalCount: &total, - WorkflowRuns: c.workflowRuns, - } - return workflowRuns, &github.Response{}, nil -} - -func (c *FakeGHA) ListWorkflowJobs(_ context.Context, _, _ string, runID int64, _ *github.ListWorkflowJobsOptions) (*github.Jobs, *github.Response, error) { - jobs := &github.Jobs{} - for _, job := range c.workflowJobs { - if job.RunID != nil && *job.RunID == runID { - jobs.Jobs = append(jobs.Jobs, job) - } - } - if len(jobs.Jobs) == 0 { - return nil, nil, errors.New("no jobs found for the given run ID") - } - return jobs, &github.Response{}, nil -} diff --git a/framework/cmd/ci_test.go b/framework/cmd/ci_test.go deleted file mode 100644 index ef2932fa3..000000000 --- a/framework/cmd/ci_test.go +++ /dev/null @@ -1,258 +0,0 @@ -package main - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/google/go-github/v50/github" -) - -var ( - DefaultRunID = github.Int64(1337) - DefaultWorkflowRunName = "test-workflow" - DefaultJobName = "job" - DefaultJobConfig = &AnalysisConfig{ - Owner: "test", - Repo: "test", - WorkflowName: DefaultWorkflowRunName, - TimeDaysBeforeStart: 1, - TimeStart: time.Now().Add(-24 * time.Hour), - TimeDaysBeforeEnd: 0, - TimeEnd: time.Now(), - Typ: "jobs", - } - DefaultStepsConfig = &AnalysisConfig{ - Owner: "test", - Repo: "test", - WorkflowName: DefaultWorkflowRunName, - TimeDaysBeforeStart: 1, - TimeStart: time.Now().Add(-24 * time.Hour), - TimeDaysBeforeEnd: 0, - TimeEnd: time.Now(), - Typ: "steps", - } -) - -func TestSmokeCLIGitHubAnalytics(t *testing.T) { - tests := []struct { - name string - setup func() *FakeGHA - config *AnalysisConfig - validate func(t *testing.T, res *Stats, err error) - }{ - { - name: "no jobs found for workflow", - config: DefaultJobConfig, - setup: func() *FakeGHA { - c := NewFakeGHA() - c.WorkflowRun(&github.WorkflowRun{ - ID: DefaultRunID, - Name: github.String(DefaultWorkflowRunName), - }) - return c - }, - validate: func(t *testing.T, res *Stats, err error) { - require.Error(t, err) - require.Contains(t, err.Error(), "no jobs found") - require.Empty(t, res) - }, - }, - { - name: "ignoring skipped or in progress jobs", - config: DefaultJobConfig, - setup: func() *FakeGHA { - c := NewFakeGHA() - runID1 := github.Int64(1337) - runID2 := github.Int64(1338) - c.WorkflowRun(&github.WorkflowRun{ - ID: runID1, - Name: github.String(DefaultWorkflowRunName), - }) - c.WorkflowRun(&github.WorkflowRun{ - ID: runID2, - Name: github.String(DefaultWorkflowRunName), - }) - c.Job(&github.WorkflowJob{ - Name: github.String("job-1"), - RunID: runID1, - StartedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour)}, - CompletedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour + time.Minute)}, - Conclusion: github.String("skipped"), - }) - c.Job(&github.WorkflowJob{ - Name: github.String("job-2"), - RunID: runID2, - StartedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour)}, - CompletedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour + time.Minute)}, - // no conclusion yet - }) - return c - }, - validate: func(t *testing.T, res *Stats, err error) { - require.NoError(t, err) - require.Equal(t, 0, len(res.Jobs)) - }, - }, - { - name: "successful analysis with one job", - config: DefaultJobConfig, - setup: func() *FakeGHA { - c := NewFakeGHA() - c.WorkflowRun(&github.WorkflowRun{ - ID: DefaultRunID, - Name: github.String(DefaultWorkflowRunName), - Conclusion: github.String("failure"), - }) - c.Job(&github.WorkflowJob{ - Name: github.String(DefaultJobName), - RunID: DefaultRunID, - StartedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour)}, - CompletedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour + time.Minute)}, - Conclusion: github.String("failure"), - }) - return c - }, - validate: func(t *testing.T, res *Stats, err error) { - require.NoError(t, err) - require.NotEmpty(t, res) - require.Equal(t, 1, len(res.Jobs)) - require.Equal(t, 1, res.Jobs[DefaultJobName].Failures) - require.Equal(t, 0, res.Jobs[DefaultJobName].Successes) - require.Equal(t, 1*time.Minute, res.Jobs[DefaultJobName].P50) - require.Equal(t, 1*time.Minute, res.Jobs[DefaultJobName].P95) - require.Equal(t, 1*time.Minute, res.Jobs[DefaultJobName].P99) - }, - }, - { - name: "correct time frames filtering", - config: DefaultJobConfig, - setup: func() *FakeGHA { - // https://docs.github.com/en/search-github/getting-started-with-searching-on-github/understanding-the-search-syntax#query-for-dates - // there is no need to re-implement this logic in a mock - // TODO: max runs per page is 100, max pages is 10 - // TODO: for precision of this tool is more than enough and after 10 pages GHA returns page 0, investigate if needed - return NewFakeGHA() - }, - validate: func(t *testing.T, res *Stats, err error) {}, - }, - { - name: "correct job quantiles for multiple workflow runs", - config: DefaultJobConfig, - setup: func() *FakeGHA { - c := NewFakeGHA() - runID1 := github.Int64(1337) - runID2 := github.Int64(1338) - runID3 := github.Int64(1339) - c.WorkflowRun(&github.WorkflowRun{ - ID: runID1, - Name: github.String(DefaultWorkflowRunName), - }) - c.WorkflowRun(&github.WorkflowRun{ - ID: runID2, - Name: github.String(DefaultWorkflowRunName), - }) - c.WorkflowRun(&github.WorkflowRun{ - ID: runID3, - Name: github.String(DefaultWorkflowRunName), - }) - c.Job(&github.WorkflowJob{ - Name: github.String(DefaultJobName), - RunID: runID1, - StartedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour)}, - CompletedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour + 1*time.Minute)}, - Conclusion: github.String("failure"), - }) - c.Job(&github.WorkflowJob{ - Name: github.String(DefaultJobName), - RunID: runID2, - StartedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour)}, - CompletedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour + 3*time.Minute)}, - Conclusion: github.String("success"), - }) - c.Job(&github.WorkflowJob{ - Name: github.String(DefaultJobName), - RunID: runID3, - StartedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour)}, - CompletedAt: &github.Timestamp{Time: time.Now().Add(-time.Hour + 5*time.Minute)}, - Conclusion: github.String("success"), - }) - return c - }, - validate: func(t *testing.T, res *Stats, err error) { - require.NoError(t, err) - require.NotEmpty(t, res) - require.Equal(t, 1, len(res.Jobs)) - require.Equal(t, 1, res.Jobs[DefaultJobName].Failures) - require.Equal(t, 2, res.Jobs[DefaultJobName].Successes) - require.Equal(t, 3*time.Minute, res.Jobs[DefaultJobName].P50) - require.Equal(t, 5*time.Minute, res.Jobs[DefaultJobName].P95) - require.Equal(t, 5*time.Minute, res.Jobs[DefaultJobName].P99) - }, - }, - { - name: "successful analysis for job steps", - config: DefaultStepsConfig, - setup: func() *FakeGHA { - c := NewFakeGHA() - c.WorkflowRun(&github.WorkflowRun{ - ID: DefaultRunID, - Name: github.String("test-workflow"), - Conclusion: github.String("success"), - }) - hago := time.Now() - c.Job(&github.WorkflowJob{ - Name: github.String(DefaultJobName), - RunID: DefaultRunID, - StartedAt: &github.Timestamp{Time: hago}, - CompletedAt: &github.Timestamp{Time: hago.Add(10 * time.Minute)}, - Conclusion: github.String("success"), - Steps: []*github.TaskStep{ - { - Name: github.String("step-1"), - StartedAt: &github.Timestamp{Time: hago.Add(1 * time.Minute)}, - CompletedAt: &github.Timestamp{Time: hago.Add(2 * time.Minute)}, - Conclusion: github.String("failure"), - }, - { - Name: github.String("step-2"), - StartedAt: &github.Timestamp{Time: hago.Add(2 * time.Minute)}, - CompletedAt: &github.Timestamp{Time: hago.Add(4 * time.Minute)}, - Conclusion: github.String("success"), - }, - }, - }) - return c - }, - validate: func(t *testing.T, res *Stats, err error) { - require.NoError(t, err) - require.NotEmpty(t, res) - require.Equal(t, 1, len(res.Jobs)) - require.Equal(t, 10*time.Minute, res.Jobs[DefaultJobName].P50) - require.Equal(t, 10*time.Minute, res.Jobs[DefaultJobName].P95) - require.Equal(t, 10*time.Minute, res.Jobs[DefaultJobName].P99) - // steps - require.Equal(t, 0, res.Steps["step-1"].Successes) - require.Equal(t, 1, res.Steps["step-1"].Failures) - require.Equal(t, 1*time.Minute, res.Steps["step-1"].P50) - require.Equal(t, 1*time.Minute, res.Steps["step-1"].P95) - require.Equal(t, 1*time.Minute, res.Steps["step-1"].P99) - require.Equal(t, 1, res.Steps["step-2"].Successes) - require.Equal(t, 0, res.Steps["step-2"].Failures) - require.Equal(t, 2*time.Minute, res.Steps["step-2"].P50) - require.Equal(t, 2*time.Minute, res.Steps["step-2"].P95) - require.Equal(t, 2*time.Minute, res.Steps["step-2"].P99) - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := tt.setup() - res, err := AnalyzeJobsSteps(context.Background(), c, tt.config) - tt.validate(t, res, err) - }) - } -} diff --git a/framework/cmd/main.go b/framework/cmd/main.go index 83804f170..d999b52ac 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -1,7 +1,6 @@ package main import ( - "embed" "fmt" "io/fs" "log" @@ -15,18 +14,6 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/framework" ) -//go:embed observability/* -var embeddedObservabilityFiles embed.FS - -const ( - LocalCLNodeErrorsURL = "http://localhost:3000/d/a7de535b-3e0f-4066-bed7-d505b6ec9ef1/cl-node-errors?orgId=1" - LocalWorkflowEngineURL = "http://localhost:3000/d/ce589a98-b4be-4f80-bed1-bc62f3e4414a/workflow-engine?orgId=1&refresh=30s" - LocalLogsURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" - LocalPrometheusURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22PBFA97CFB590B2093%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22range%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22PBFA97CFB590B2093%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" - LocalPostgresDebugURL = "http://localhost:3000/d/000000039/postgresql-database?orgId=1&refresh=10s&var-DS_PROMETHEUS=PBFA97CFB590B2093&var-interval=$__auto_interval_interval&var-namespace=&var-release=&var-instance=postgres_exporter_0:9187&var-datname=All&var-mode=All&from=now-5m&to=now" - LocalPyroScopeURL = "http://localhost:4040" -) - func main() { app := &cli.App{ Name: "ctf", @@ -86,14 +73,14 @@ func main() { Usage: "ctf obs up", Aliases: []string{"u"}, Description: "Spins up a local observability stack: Grafana, Loki, Pyroscope", - Action: func(c *cli.Context) error { return observabilityUp() }, + Action: func(c *cli.Context) error { return framework.ObservabilityUp() }, }, { Name: "down", Usage: "ctf obs down", Aliases: []string{"d"}, Description: "Removes local observability stack", - Action: func(c *cli.Context) error { return observabilityDown() }, + Action: func(c *cli.Context) error { return framework.ObservabilityDown() }, }, { Name: "restart", @@ -101,48 +88,10 @@ func main() { Aliases: []string{"r"}, Description: "Restart a local observability stack", Action: func(c *cli.Context) error { - if err := observabilityDown(); err != nil { + if err := framework.ObservabilityDown(); err != nil { return err } - return observabilityUp() - }, - }, - { - Name: "load", - Usage: "ctf obs l", - Aliases: []string{"l"}, - Description: "Loads logs to Loki", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "raw-url", - Aliases: []string{"u"}, - Usage: "URL to GitHub raw log data", - }, - &cli.StringFlag{ - Name: "dir", - Aliases: []string{"d"}, - Usage: "Directory to logs, output of 'gh run download $run_id'", - }, - &cli.IntFlag{ - Name: "rps", - Aliases: []string{"r"}, - Usage: "RPS for uploading log chunks", - Value: 30, - }, - &cli.IntFlag{ - Name: "chunk", - Aliases: []string{"c"}, - Usage: "Amount of chunks the files will be split in", - Value: 100, - }, - }, - Action: func(c *cli.Context) error { - return loadLogs( - c.String("raw-url"), - c.String("dir"), - c.Int("rps"), - c.Int("chunk"), - ) + return framework.ObservabilityUp() }, }, }, @@ -166,7 +115,7 @@ func main() { Aliases: []string{"u"}, Description: "Spins up Blockscout stack", Action: func(c *cli.Context) error { - return blockscoutUp(c.String("rpc")) + return framework.BlockscoutUp(c.String("rpc")) }, }, { @@ -175,7 +124,7 @@ func main() { Aliases: []string{"d"}, Description: "Removes Blockscout stack, wipes all Blockscout databases data", Action: func(c *cli.Context) error { - return blockscoutDown(c.String("rpc")) + return framework.BlockscoutDown(c.String("rpc")) }, }, { @@ -185,76 +134,14 @@ func main() { Description: "Reboots Blockscout stack", Action: func(c *cli.Context) error { rpc := c.String("rpc") - if err := blockscoutDown(rpc); err != nil { + if err := framework.BlockscoutDown(rpc); err != nil { return err } - return blockscoutUp(rpc) + return framework.BlockscoutUp(rpc) }, }, }, }, - - { - Name: "ci", - Usage: "Analyze CI job durations and statistics", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "repository", - Aliases: []string{"r"}, - Usage: "GitHub repository in format owner/repo", - Required: true, - }, - &cli.StringFlag{ - Name: "workflow", - Aliases: []string{"w"}, - Usage: "Name of GitHub workflow to analyze", - Required: true, - }, - &cli.StringFlag{ - Name: "start", - Aliases: []string{"s"}, - Value: "1", - Usage: "How many days to analyze", - }, - &cli.StringFlag{ - Name: "end", - Aliases: []string{"e"}, - Value: "0", - Usage: "How many days to analyze", - }, - &cli.StringFlag{ - Name: "type", - Aliases: []string{"t"}, - Usage: "Analytics type: jobs or steps", - }, - &cli.BoolFlag{ - Name: "debug", - Usage: "Dumps all the workflow/jobs files for debugging purposes", - }, - }, - Action: func(c *cli.Context) error { - repo := c.String("repository") - parts := strings.Split(repo, "/") - if len(parts) != 2 { - return fmt.Errorf("repository must be in format owner/repo, got: %s", repo) - } - typ := c.String("type") - if typ != "jobs" && typ != "steps" { - return fmt.Errorf("type must be 'jobs' or 'steps'") - } - _, err := AnalyzeCIRuns(&AnalysisConfig{ - Debug: c.Bool("debug"), - Owner: parts[0], - Repo: parts[1], - WorkflowName: c.String("workflow"), - TimeDaysBeforeStart: c.Int("start"), - TimeDaysBeforeEnd: c.Int("end"), - Typ: typ, - ResultsFile: "ctf-ci.json", - }) - return err - }, - }, }, } @@ -273,7 +160,7 @@ func extractAllFiles(embeddedDir string) error { } // Walk through the embedded files - err = fs.WalkDir(embeddedObservabilityFiles, embeddedDir, func(path string, d fs.DirEntry, err error) error { + err = fs.WalkDir(framework.EmbeddedObservabilityFiles, embeddedDir, func(path string, d fs.DirEntry, err error) error { if err != nil { return fmt.Errorf("error walking the directory: %w", err) } @@ -287,7 +174,7 @@ func extractAllFiles(embeddedDir string) error { } // Read file content from embedded file system - content, err := embeddedObservabilityFiles.ReadFile(path) + content, err := framework.EmbeddedObservabilityFiles.ReadFile(path) if err != nil { return fmt.Errorf("failed to read file %s: %w", path, err) } diff --git a/framework/cmd/observability.go b/framework/cmd/observability.go deleted file mode 100644 index f780288fe..000000000 --- a/framework/cmd/observability.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "fmt" - "net/http" - - "github.com/google/uuid" - "github.com/pkg/errors" - "go.uber.org/ratelimit" - - "github.com/smartcontractkit/chainlink-testing-framework/framework" -) - -func loadLogs(rawURL, dirPath string, rps, chunks int) error { - if rawURL == "" && dirPath == "" { - return fmt.Errorf("at least one source must be provided, either -u $url or -d $dir") - } - jobID := uuid.New().String()[0:5] - framework.L.Info().Str("JobID", jobID).Msg("Loading logs into Loki") - limiter := ratelimit.New(rps) - if rawURL != "" { - L.Info().Msg("Downloading raw logs from URL") - //nolint:gosec - resp, err := http.Get(rawURL) - if err != nil { - return errors.Wrap(err, "error downloading raw logs") - } - defer resp.Body.Close() - - if resp.StatusCode/100 != 2 { - return fmt.Errorf("non-success response code when downloading raw logs: %s", resp.Status) - } - - if err := processAndUploadLog(rawURL, resp.Body, limiter, chunks, jobID); err != nil { - return errors.Wrap(err, "error processing raw logs") - } - } else if dirPath != "" { - L.Info().Msgf("Processing directory: %s", dirPath) - if err := processAndUploadDir(dirPath, limiter, chunks, jobID); err != nil { - return errors.Wrapf(err, "error processing directory: %s", dirPath) - } - } - framework.L.Info().Str("JobID", jobID).Str("URL", grafanaURL+jobID+grafanaURL2).Msg("Upload complete") - return nil -} - -func observabilityUp() error { - framework.L.Info().Msg("Creating local observability stack") - if err := extractAllFiles("observability"); err != nil { - return err - } - if err := framework.NewPromtail(); err != nil { - return err - } - err := framework.RunCommand("bash", "-c", fmt.Sprintf(` - cd %s && \ - docker compose up -d - `, "compose")) - if err != nil { - return err - } - fmt.Println() - framework.L.Info().Msgf("Loki: %s", LocalLogsURL) - framework.L.Info().Msgf("Prometheus: %s", LocalPrometheusURL) - framework.L.Info().Msgf("PostgreSQL: %s", LocalPostgresDebugURL) - framework.L.Info().Msgf("Pyroscope: %s", LocalPyroScopeURL) - framework.L.Info().Msgf("CL Node Errors: %s", LocalCLNodeErrorsURL) - framework.L.Info().Msgf("Workflow Engine: %s", LocalWorkflowEngineURL) - return nil -} - -func observabilityDown() error { - framework.L.Info().Msg("Removing local observability stack") - err := framework.RunCommand("bash", "-c", fmt.Sprintf(` - cd %s && \ - docker compose down -v && docker rm -f promtail - `, "compose")) - if err != nil { - return err - } - return nil -} diff --git a/framework/config.go b/framework/config.go index 8c414f1a9..223b2983c 100644 --- a/framework/config.go +++ b/framework/config.go @@ -38,21 +38,18 @@ const ( ) const ( - OutputFieldNameTOML = "out" - OutputFieldName = "Out" - OverridesFieldName = "Overrides" + DefaultConfigFilePath = "env.toml" + DefaultOverridesFilePath = "overrides.toml" ) var ( - once = &sync.Once{} + Once = &sync.Once{} // Secrets is a singleton AWS Secrets Manager // Loaded once on start inside Load and is safe to call concurrently Secrets *AWSSecretsManager DefaultNetworkName string - AllowedEmptyConfigurationFields = []string{OutputFieldName, OverridesFieldName} - Validator *validator.Validate = validator.New(validator.WithRequiredStructEnabled()) ValidatorTranslator ut.Translator @@ -74,7 +71,7 @@ type ValidationError struct { func mergeInputs[T any]() (*T, error) { var config T paths := strings.Split(os.Getenv(EnvVarTestConfigs), ",") - _, err := getBaseConfigPath() + _, err := BaseConfigPath() if err != nil { return nil, err } @@ -82,9 +79,13 @@ func mergeInputs[T any]() (*T, error) { L.Info().Str("Path", path).Msg("Loading configuration input") data, err := os.ReadFile(filepath.Join(DefaultConfigDir, path)) if err != nil { + if path == DefaultOverridesFilePath { + L.Info().Str("Path", path).Msg("Overrides file not found or empty") + continue + } return nil, fmt.Errorf("error reading config file %s: %w", path, err) } - if L.GetLevel() == zerolog.DebugLevel { + if L.GetLevel() == zerolog.TraceLevel { fmt.Println(string(data)) } @@ -104,8 +105,8 @@ func mergeInputs[T any]() (*T, error) { return nil, fmt.Errorf("failed to decode TOML config, strict mode: %s", err) } } - if L.GetLevel() == zerolog.DebugLevel { - L.Debug().Msg("Merged inputs") + if L.GetLevel() == zerolog.TraceLevel { + L.Trace().Msg("Merged inputs") spew.Dump(config) } return &config, nil @@ -201,12 +202,54 @@ func Load[X any](t *testing.T) (*X, error) { require.NoError(t, err) }) } - if err = DefaultNetwork(once); err != nil { + if err = DefaultNetwork(Once); err != nil { L.Info().Err(err).Msg("docker network creation failed, either docker is not running or you are running in CRIB mode") } return input, nil } +// LoadCache loads cached config with environment values +func LoadCache[X any](t *testing.T) (*X, error) { + cfgPath := os.Getenv("CTF_CONFIGS") + L.Debug().Str("CTFConfigs", cfgPath).Msg("Loading configuration from cache") + if cfgPath == "" { + return nil, fmt.Errorf("CTF_CONFIGS environment variable not set when loading from cache") + } + firstConfig := strings.Split(cfgPath, ",") + cfgParts := strings.Split(firstConfig[0], ".") + if len(cfgParts) != 2 { + return nil, fmt.Errorf("invalid config path when loading from cache: %s", cfgPath) + } + _ = os.Setenv("CTF_CONFIGS", fmt.Sprintf("%s-cache.%s", cfgParts[0], cfgParts[1])) + return Load[X](t) +} + +func BaseConfigPath() (string, error) { + configs := os.Getenv("CTF_CONFIGS") + if configs == "" { + return "", fmt.Errorf("no %s env var is provided, you should provide at least one test config in TOML", EnvVarTestConfigs) + } + L.Debug().Str("DevEnvConfigs", configs).Msg("Getting base config path") + return strings.Split(configs, ",")[0], nil +} + +func BaseConfigName() (string, error) { + cp, err := BaseConfigPath() + if err != nil { + return "", err + } + return strings.Replace(cp, ".toml", "", -1), nil +} + +func BaseCacheName() (string, error) { + cp, err := BaseConfigPath() + if err != nil { + return "", err + } + name := strings.Replace(cp, ".toml", "", -1) + return fmt.Sprintf("%s-cache.toml", name), nil +} + func DefaultNetwork(once *sync.Once) error { var net *testcontainers.DockerNetwork var err error diff --git a/framework/observability.go b/framework/observability.go new file mode 100644 index 000000000..a51f754ca --- /dev/null +++ b/framework/observability.go @@ -0,0 +1,150 @@ +package framework + +import ( + "embed" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +//go:embed observability/* +var EmbeddedObservabilityFiles embed.FS + +const ( + LocalCLNodeErrorsURL = "http://localhost:3000/d/a7de535b-3e0f-4066-bed7-d505b6ec9ef1/cl-node-errors?orgId=1" + LocalWorkflowEngineURL = "http://localhost:3000/d/ce589a98-b4be-4f80-bed1-bc62f3e4414a/workflow-engine?orgId=1&refresh=30s" + LocalLogsURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" + LocalPrometheusURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22PBFA97CFB590B2093%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22range%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22PBFA97CFB590B2093%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" + LocalPostgresDebugURL = "http://localhost:3000/d/000000039/postgresql-database?orgId=1&refresh=10s&var-DS_PROMETHEUS=PBFA97CFB590B2093&var-interval=$__auto_interval_interval&var-namespace=&var-release=&var-instance=postgres_exporter_0:9187&var-datname=All&var-mode=All&from=now-5m&to=now" + LocalPyroScopeURL = "http://localhost:4040" +) + +// extractAllFiles goes through the embedded directory and extracts all files to the current directory +func extractAllFiles(embeddedDir string) error { + // Get current working directory where CLI is running + currentDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + + // Walk through the embedded files + err = fs.WalkDir(EmbeddedObservabilityFiles, embeddedDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("error walking the directory: %w", err) + } + if strings.Contains(path, "README.md") { + return nil + } + + // Skip directories + if d.IsDir() { + return nil + } + + // Read file content from embedded file system + content, err := EmbeddedObservabilityFiles.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", path, err) + } + + // Determine the target path (strip out the `embeddedDir` part) + relativePath, err := filepath.Rel(embeddedDir, path) + if err != nil { + return fmt.Errorf("failed to determine relative path for %s: %w", path, err) + } + targetPath := filepath.Join(currentDir, relativePath) + + // Create target directories if necessary + targetDir := filepath.Dir(targetPath) + err = os.MkdirAll(targetDir, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create directory %s: %w", targetDir, err) + } + + // Write the file content to the target path + //nolint + err = os.WriteFile(targetPath, content, 0o777) + if err != nil { + return fmt.Errorf("failed to write file %s: %w", targetPath, err) + } + return nil + }) + + return err +} + +func BlockscoutUp(url string) error { + L.Info().Msg("Creating local Blockscout stack") + if err := extractAllFiles("observability"); err != nil { + return err + } + os.Setenv("BLOCKSCOUT_RPC_URL", url) + err := RunCommand("bash", "-c", fmt.Sprintf(` + cd %s && \ + docker compose up -d + `, "blockscout")) + if err != nil { + return err + } + fmt.Println() + L.Info().Msgf("Blockscout is up at: %s", "http://localhost") + return nil +} + +func BlockscoutDown(url string) error { + L.Info().Msg("Removing local Blockscout stack") + os.Setenv("BLOCKSCOUT_RPC_URL", url) + err := RunCommand("bash", "-c", fmt.Sprintf(` + cd %s && \ + docker compose down -v + `, "blockscout")) + if err != nil { + return err + } + return RunCommand("bash", "-c", fmt.Sprintf(` + cd %s && \ + rm -rf blockscout-db-data && \ + rm -rf logs && \ + rm -rf redis-data && \ + rm -rf stats-db-data + `, filepath.Join("blockscout", "services"))) +} + +func ObservabilityUp() error { + L.Info().Msg("Creating local observability stack") + if err := extractAllFiles("observability"); err != nil { + return err + } + if err := NewPromtail(); err != nil { + return err + } + err := RunCommand("bash", "-c", fmt.Sprintf(` + cd %s && \ + docker compose up -d + `, "compose")) + if err != nil { + return err + } + fmt.Println() + L.Info().Msgf("Loki: %s", LocalLogsURL) + L.Info().Msgf("Prometheus: %s", LocalPrometheusURL) + L.Info().Msgf("PostgreSQL: %s", LocalPostgresDebugURL) + L.Info().Msgf("Pyroscope: %s", LocalPyroScopeURL) + L.Info().Msgf("CL Node Errors: %s", LocalCLNodeErrorsURL) + L.Info().Msgf("Workflow Engine: %s", LocalWorkflowEngineURL) + return nil +} + +func ObservabilityDown() error { + L.Info().Msg("Removing local observability stack") + err := RunCommand("bash", "-c", fmt.Sprintf(` + cd %s && \ + docker compose down -v && docker rm -f promtail + `, "compose")) + if err != nil { + return err + } + return nil +} diff --git a/framework/cmd/observability/blockscout/docker-compose.yml b/framework/observability/blockscout/docker-compose.yml similarity index 75% rename from framework/cmd/observability/blockscout/docker-compose.yml rename to framework/observability/blockscout/docker-compose.yml index 975a66aa6..98a99c4db 100644 --- a/framework/cmd/observability/blockscout/docker-compose.yml +++ b/framework/observability/blockscout/docker-compose.yml @@ -1,12 +1,12 @@ services: redis-db: extends: - file: ./services/redis.yml + file: services/redis.yml service: redis-db db-init: extends: - file: ./services/db.yml + file: services/db.yml service: db-init db: @@ -14,7 +14,7 @@ services: db-init: condition: service_completed_successfully extends: - file: ./services/db.yml + file: services/db.yml service: db backend: @@ -22,7 +22,7 @@ services: - db - redis-db extends: - file: ./services/backend.yml + file: services/backend.yml service: backend links: - db:database @@ -34,24 +34,24 @@ services: visualizer: extends: - file: ./services/visualizer.yml + file: services/visualizer.yml service: visualizer sig-provider: extends: - file: ./services/sig-provider.yml + file: services/sig-provider.yml service: sig-provider frontend: depends_on: - backend extends: - file: ./services/frontend.yml + file: services/frontend.yml service: frontend stats-db-init: extends: - file: ./services/stats.yml + file: services/stats.yml service: stats-db-init stats-db: @@ -59,7 +59,7 @@ services: stats-db-init: condition: service_completed_successfully extends: - file: ./services/stats.yml + file: services/stats.yml service: stats-db stats: @@ -67,7 +67,7 @@ services: - stats-db - backend extends: - file: ./services/stats.yml + file: services/stats.yml service: stats user-ops-indexer: @@ -75,7 +75,7 @@ services: - db - backend extends: - file: ./services/user-ops-indexer.yml + file: services/user-ops-indexer.yml service: user-ops-indexer proxy: @@ -84,5 +84,5 @@ services: - frontend - stats extends: - file: ./services/nginx.yml + file: services/nginx.yml service: proxy diff --git a/framework/cmd/observability/blockscout/envs/common-blockscout.env b/framework/observability/blockscout/envs/common-blockscout.env similarity index 100% rename from framework/cmd/observability/blockscout/envs/common-blockscout.env rename to framework/observability/blockscout/envs/common-blockscout.env diff --git a/framework/cmd/observability/blockscout/envs/common-frontend.env b/framework/observability/blockscout/envs/common-frontend.env similarity index 100% rename from framework/cmd/observability/blockscout/envs/common-frontend.env rename to framework/observability/blockscout/envs/common-frontend.env diff --git a/framework/cmd/observability/blockscout/envs/common-smart-contract-verifier.env b/framework/observability/blockscout/envs/common-smart-contract-verifier.env similarity index 100% rename from framework/cmd/observability/blockscout/envs/common-smart-contract-verifier.env rename to framework/observability/blockscout/envs/common-smart-contract-verifier.env diff --git a/framework/cmd/observability/blockscout/envs/common-stats.env b/framework/observability/blockscout/envs/common-stats.env similarity index 100% rename from framework/cmd/observability/blockscout/envs/common-stats.env rename to framework/observability/blockscout/envs/common-stats.env diff --git a/framework/cmd/observability/blockscout/envs/common-user-ops-indexer.env b/framework/observability/blockscout/envs/common-user-ops-indexer.env similarity index 100% rename from framework/cmd/observability/blockscout/envs/common-user-ops-indexer.env rename to framework/observability/blockscout/envs/common-user-ops-indexer.env diff --git a/framework/cmd/observability/blockscout/envs/common-visualizer.env b/framework/observability/blockscout/envs/common-visualizer.env similarity index 100% rename from framework/cmd/observability/blockscout/envs/common-visualizer.env rename to framework/observability/blockscout/envs/common-visualizer.env diff --git a/framework/cmd/observability/blockscout/proxy/default.conf.template b/framework/observability/blockscout/proxy/default.conf.template similarity index 100% rename from framework/cmd/observability/blockscout/proxy/default.conf.template rename to framework/observability/blockscout/proxy/default.conf.template diff --git a/framework/cmd/observability/blockscout/proxy/microservices.conf.template b/framework/observability/blockscout/proxy/microservices.conf.template similarity index 100% rename from framework/cmd/observability/blockscout/proxy/microservices.conf.template rename to framework/observability/blockscout/proxy/microservices.conf.template diff --git a/framework/cmd/observability/blockscout/services/backend.yml b/framework/observability/blockscout/services/backend.yml similarity index 91% rename from framework/cmd/observability/blockscout/services/backend.yml rename to framework/observability/blockscout/services/backend.yml index 0231c3fc0..8ae1b2f6c 100644 --- a/framework/cmd/observability/blockscout/services/backend.yml +++ b/framework/observability/blockscout/services/backend.yml @@ -11,6 +11,6 @@ services: extra_hosts: - 'host.docker.internal:host-gateway' env_file: - - ../envs/common-blockscout.env + - ../envs/common-blockscout.env volumes: - ./logs/:/app/logs/ \ No newline at end of file diff --git a/framework/cmd/observability/blockscout/services/db.yml b/framework/observability/blockscout/services/db.yml similarity index 100% rename from framework/cmd/observability/blockscout/services/db.yml rename to framework/observability/blockscout/services/db.yml diff --git a/framework/cmd/observability/blockscout/services/frontend.yml b/framework/observability/blockscout/services/frontend.yml similarity index 84% rename from framework/cmd/observability/blockscout/services/frontend.yml rename to framework/observability/blockscout/services/frontend.yml index 1d0a23776..3123f8e8e 100644 --- a/framework/cmd/observability/blockscout/services/frontend.yml +++ b/framework/observability/blockscout/services/frontend.yml @@ -8,4 +8,4 @@ services: restart: always container_name: 'frontend' env_file: - - ../envs/common-frontend.env + - ../envs/common-frontend.env diff --git a/framework/cmd/observability/blockscout/services/nginx.yml b/framework/observability/blockscout/services/nginx.yml similarity index 100% rename from framework/cmd/observability/blockscout/services/nginx.yml rename to framework/observability/blockscout/services/nginx.yml diff --git a/framework/cmd/observability/blockscout/services/redis.yml b/framework/observability/blockscout/services/redis.yml similarity index 100% rename from framework/cmd/observability/blockscout/services/redis.yml rename to framework/observability/blockscout/services/redis.yml diff --git a/framework/cmd/observability/blockscout/services/sig-provider.yml b/framework/observability/blockscout/services/sig-provider.yml similarity index 100% rename from framework/cmd/observability/blockscout/services/sig-provider.yml rename to framework/observability/blockscout/services/sig-provider.yml diff --git a/framework/cmd/observability/blockscout/services/smart-contract-verifier.yml b/framework/observability/blockscout/services/smart-contract-verifier.yml similarity index 82% rename from framework/cmd/observability/blockscout/services/smart-contract-verifier.yml rename to framework/observability/blockscout/services/smart-contract-verifier.yml index 1efa66013..5707f5f78 100644 --- a/framework/cmd/observability/blockscout/services/smart-contract-verifier.yml +++ b/framework/observability/blockscout/services/smart-contract-verifier.yml @@ -8,4 +8,4 @@ services: restart: always container_name: 'smart-contract-verifier' env_file: - - ../envs/common-smart-contract-verifier.env + - ../envs/common-smart-contract-verifier.env diff --git a/framework/cmd/observability/blockscout/services/stats.yml b/framework/observability/blockscout/services/stats.yml similarity index 97% rename from framework/cmd/observability/blockscout/services/stats.yml rename to framework/observability/blockscout/services/stats.yml index 286813906..3fff4d0a3 100644 --- a/framework/cmd/observability/blockscout/services/stats.yml +++ b/framework/observability/blockscout/services/stats.yml @@ -43,7 +43,7 @@ services: extra_hosts: - 'host.docker.internal:host-gateway' env_file: - - ../envs/common-stats.env + - ../envs/common-stats.env environment: - STATS__DB_URL=${STATS__DB_URL:-postgres://stats:n0uejXPl61ci6ldCuE2gQU5Y@stats-db:5432/stats} - STATS__BLOCKSCOUT_DB_URL=${STATS__BLOCKSCOUT_DB_URL:-postgresql://blockscout:ceWb1MeLBEeOIfk65gU8EjF8@db:5432/blockscout} diff --git a/framework/cmd/observability/blockscout/services/user-ops-indexer.yml b/framework/observability/blockscout/services/user-ops-indexer.yml similarity index 93% rename from framework/cmd/observability/blockscout/services/user-ops-indexer.yml rename to framework/observability/blockscout/services/user-ops-indexer.yml index 7918aa7ef..5d376a336 100644 --- a/framework/cmd/observability/blockscout/services/user-ops-indexer.yml +++ b/framework/observability/blockscout/services/user-ops-indexer.yml @@ -10,7 +10,7 @@ services: extra_hosts: - 'host.docker.internal:host-gateway' env_file: - - ../envs/common-user-ops-indexer.env + - ../envs/common-user-ops-indexer.env environment: - USER_OPS_INDEXER__INDEXER__RPC_URL=${USER_OPS_INDEXER__INDEXER__RPC_URL:-ws://host.docker.internal:8545/} - USER_OPS_INDEXER__DATABASE__CONNECT__URL=${USER_OPS_INDEXER__DATABASE__CONNECT__URL:-postgresql://blockscout:ceWb1MeLBEeOIfk65gU8EjF8@db:5432/blockscout} diff --git a/framework/cmd/observability/blockscout/services/visualizer.yml b/framework/observability/blockscout/services/visualizer.yml similarity index 84% rename from framework/cmd/observability/blockscout/services/visualizer.yml rename to framework/observability/blockscout/services/visualizer.yml index d58c1b7f7..5361c49a4 100644 --- a/framework/cmd/observability/blockscout/services/visualizer.yml +++ b/framework/observability/blockscout/services/visualizer.yml @@ -8,4 +8,4 @@ services: restart: always container_name: 'visualizer' env_file: - - ../envs/common-visualizer.env + - ../envs/common-visualizer.env diff --git a/framework/cmd/observability/compose/conf/defaults.ini b/framework/observability/compose/conf/defaults.ini similarity index 100% rename from framework/cmd/observability/compose/conf/defaults.ini rename to framework/observability/compose/conf/defaults.ini diff --git a/framework/cmd/observability/compose/conf/grafana.ini b/framework/observability/compose/conf/grafana.ini similarity index 100% rename from framework/cmd/observability/compose/conf/grafana.ini rename to framework/observability/compose/conf/grafana.ini diff --git a/framework/cmd/observability/compose/conf/prometheus.yml b/framework/observability/compose/conf/prometheus.yml similarity index 100% rename from framework/cmd/observability/compose/conf/prometheus.yml rename to framework/observability/compose/conf/prometheus.yml diff --git a/framework/cmd/observability/compose/conf/provisioning/access-control/sample.yaml b/framework/observability/compose/conf/provisioning/access-control/sample.yaml similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/access-control/sample.yaml rename to framework/observability/compose/conf/provisioning/access-control/sample.yaml diff --git a/framework/cmd/observability/compose/conf/provisioning/dashboards/cadvisor/cadvisor.json b/framework/observability/compose/conf/provisioning/dashboards/cadvisor/cadvisor.json similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/dashboards/cadvisor/cadvisor.json rename to framework/observability/compose/conf/provisioning/dashboards/cadvisor/cadvisor.json diff --git a/framework/cmd/observability/compose/conf/provisioning/dashboards/clnode-errors/errors.json b/framework/observability/compose/conf/provisioning/dashboards/clnode-errors/errors.json similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/dashboards/clnode-errors/errors.json rename to framework/observability/compose/conf/provisioning/dashboards/clnode-errors/errors.json diff --git a/framework/cmd/observability/compose/conf/provisioning/dashboards/dashboards.yaml b/framework/observability/compose/conf/provisioning/dashboards/dashboards.yaml similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/dashboards/dashboards.yaml rename to framework/observability/compose/conf/provisioning/dashboards/dashboards.yaml diff --git a/framework/cmd/observability/compose/conf/provisioning/dashboards/pg/pg.json b/framework/observability/compose/conf/provisioning/dashboards/pg/pg.json similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/dashboards/pg/pg.json rename to framework/observability/compose/conf/provisioning/dashboards/pg/pg.json diff --git a/framework/cmd/observability/compose/conf/provisioning/dashboards/wasp/wasp.json b/framework/observability/compose/conf/provisioning/dashboards/wasp/wasp.json similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/dashboards/wasp/wasp.json rename to framework/observability/compose/conf/provisioning/dashboards/wasp/wasp.json diff --git a/framework/cmd/observability/compose/conf/provisioning/dashboards/workflow-engine/engine.json b/framework/observability/compose/conf/provisioning/dashboards/workflow-engine/engine.json similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/dashboards/workflow-engine/engine.json rename to framework/observability/compose/conf/provisioning/dashboards/workflow-engine/engine.json diff --git a/framework/cmd/observability/compose/conf/provisioning/datasources/loki.yaml b/framework/observability/compose/conf/provisioning/datasources/loki.yaml similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/datasources/loki.yaml rename to framework/observability/compose/conf/provisioning/datasources/loki.yaml diff --git a/framework/cmd/observability/compose/conf/provisioning/notifiers/sample.yaml b/framework/observability/compose/conf/provisioning/notifiers/sample.yaml similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/notifiers/sample.yaml rename to framework/observability/compose/conf/provisioning/notifiers/sample.yaml diff --git a/framework/cmd/observability/compose/conf/provisioning/plugins/sample.yaml b/framework/observability/compose/conf/provisioning/plugins/sample.yaml similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/plugins/sample.yaml rename to framework/observability/compose/conf/provisioning/plugins/sample.yaml diff --git a/framework/cmd/observability/compose/conf/provisioning/rules/rules.yml b/framework/observability/compose/conf/provisioning/rules/rules.yml similarity index 100% rename from framework/cmd/observability/compose/conf/provisioning/rules/rules.yml rename to framework/observability/compose/conf/provisioning/rules/rules.yml diff --git a/framework/cmd/observability/compose/docker-compose.yaml b/framework/observability/compose/docker-compose.yaml similarity index 100% rename from framework/cmd/observability/compose/docker-compose.yaml rename to framework/observability/compose/docker-compose.yaml diff --git a/framework/cmd/observability/compose/loki-config.yaml b/framework/observability/compose/loki-config.yaml similarity index 100% rename from framework/cmd/observability/compose/loki-config.yaml rename to framework/observability/compose/loki-config.yaml diff --git a/framework/cmd/observability/compose/otel.yaml b/framework/observability/compose/otel.yaml similarity index 100% rename from framework/cmd/observability/compose/otel.yaml rename to framework/observability/compose/otel.yaml diff --git a/framework/cmd/observability/compose/tempo.yaml b/framework/observability/compose/tempo.yaml similarity index 100% rename from framework/cmd/observability/compose/tempo.yaml rename to framework/observability/compose/tempo.yaml From 7dcf0a51064565bb1115475a5f689ffd02b43f37 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 8 Jul 2025 17:35:40 +0200 Subject: [PATCH 2/4] fix go mod and lint --- framework/cmd/logs.go | 197 ------------------------------------- framework/go.mod | 8 -- framework/go.sum | 26 ----- framework/observability.go | 6 +- 4 files changed, 3 insertions(+), 234 deletions(-) delete mode 100644 framework/cmd/logs.go diff --git a/framework/cmd/logs.go b/framework/cmd/logs.go deleted file mode 100644 index 16158cd73..000000000 --- a/framework/cmd/logs.go +++ /dev/null @@ -1,197 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/pkg/errors" - - "github.com/smartcontractkit/chainlink-testing-framework/framework" - - "go.uber.org/ratelimit" -) - -var L = framework.L - -type LokiPushRequest struct { - Streams []LokiStream `json:"streams"` -} - -type LokiStream struct { - Stream map[string]string `json:"stream"` - Values [][2]string `json:"values"` -} - -const ( - lokiURL = "http://localhost:3030/loki/api/v1/push" - grafanaURL = "http://localhost:3000/explore?panes=%7B%22V0P%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22" - grafanaURL2 = "%5C%22%7D%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" -) - -func processAndUploadDir(dirPath string, limiter ratelimit.Limiter, chunks int, jobID string) error { - return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return errors.Wrapf(err, "error accessing file: %s", path) - } - if info.IsDir() { - return nil - } - L.Info().Msgf("Processing file: %s", path) - f, err := os.Open(path) - if err != nil { - return errors.Wrapf(err, "error opening file: %s", path) - } - defer f.Close() - - if err := processAndUploadLog(path, f, limiter, chunks, jobID); err != nil { - return errors.Wrapf(err, "error processing file: %s", path) - } - return nil - }) -} - -func processAndUploadLog(source string, r io.Reader, limiter ratelimit.Limiter, chunks int, jobID string) error { - scanner := bufio.NewScanner(r) - var values [][2]string - baseTime := time.Now() - - // Read all log lines; each line gets a unique timestamp. - for scanner.Scan() { - line := scanner.Text() - ts := baseTime.UnixNano() - values = append(values, [2]string{fmt.Sprintf("%d", ts), line}) - baseTime = baseTime.Add(time.Nanosecond) - } - if err := scanner.Err(); err != nil { - return fmt.Errorf("error scanning logs from %s: %w", source, err) - } - - totalLines := len(values) - if totalLines == 0 { - L.Info().Msgf("No log lines found in %s", source) - return nil - } - // Some logs may include CL node logs, skip chunking for all that is less - if totalLines <= 10000 { - chunks = 1 - } - if chunks > totalLines { - chunks = totalLines - } - chunkSize := totalLines / chunks - remainder := totalLines % chunks - L.Debug().Int("total_lines", totalLines). - Int("chunks", chunks). - Msgf("Starting chunk processing for %s", source) - var wg sync.WaitGroup - errCh := make(chan error, chunks) - start := 0 - for i := 0; i < chunks; i++ { - extra := 0 - if i < remainder { - extra = 1 - } - end := start + chunkSize + extra - chunkValues := values[start:end] - startLine := start + 1 - endLine := end - start = end - - labels := map[string]string{ - "job": jobID, - "chunk": fmt.Sprintf("%d", i+1), - "source": source, - } - reqBody := LokiPushRequest{ - Streams: []LokiStream{ - { - Stream: labels, - Values: chunkValues, - }, - }, - } - data, err := json.Marshal(reqBody) - if err != nil { - return fmt.Errorf("error marshaling JSON for chunk %d: %w", i+1, err) - } - chunkMB := float64(len(data)) / (1024 * 1024) - L.Debug().Int("chunk", i+1). - Float64("chunk_size_MB", chunkMB). - Int("start_line", startLine). - Int("end_line", endLine). - Msg("Prepared chunk for upload") - - wg.Add(1) - go func(chunkNum, sLine, eLine int, payload []byte, sizeMB float64) { - defer wg.Done() - const maxRetries = 50 - const retryDelay = 1 * time.Second - - var resp *http.Response - var attempt int - var err error - for attempt = 1; attempt <= maxRetries; attempt++ { - limiter.Take() - resp, err = http.Post(lokiURL, "application/json", bytes.NewReader(payload)) - if err != nil { - if strings.Contains(err.Error(), "connection refused") { - L.Fatal().Msg("connection refused, is local Loki up and running? use 'ctf obs u'") - return - } - L.Error().Err(err). - Int("status", resp.StatusCode). - Int("attempt", attempt). - Int("chunk", chunkNum). - Float64("chunk_size_MB", sizeMB). - Msg("Error sending POST request") - time.Sleep(retryDelay) - continue - } - - body, _ := io.ReadAll(resp.Body) - resp.Body.Close() - - if resp.StatusCode == 429 { - L.Debug().Int("attempt", attempt). - Int("chunk", chunkNum). - Float64("chunk_size_MB", sizeMB). - Msg("Received 429, retrying...") - time.Sleep(retryDelay) - continue - } - - if resp.StatusCode/100 != 2 { - err = fmt.Errorf("loki error: %s - %s", resp.Status, body) - L.Error().Err(err).Int("chunk", chunkNum). - Float64("chunk_size_MB", sizeMB). - Msg("Chunk upload failed") - time.Sleep(retryDelay) - continue - } - - L.Info().Int("chunk", chunkNum). - Float64("chunk_size_MB", sizeMB). - Msg("Successfully uploaded chunk") - return - } - errCh <- fmt.Errorf("max retries reached for chunk %d; last error: %v", chunkNum, err) - }(i+1, startLine, endLine, data, chunkMB) - } - - wg.Wait() - close(errCh) - if len(errCh) > 0 { - return <-errCh - } - - return nil -} diff --git a/framework/go.mod b/framework/go.mod index 4fd7b8805..f10f4484d 100644 --- a/framework/go.mod +++ b/framework/go.mod @@ -14,13 +14,11 @@ require ( github.com/docker/docker v28.0.4+incompatible github.com/docker/go-connections v0.5.0 github.com/ethereum/go-ethereum v1.15.0 - github.com/fatih/color v1.16.0 github.com/go-playground/locales v0.14.1 github.com/go-playground/universal-translator v0.18.1 github.com/go-playground/validator/v10 v10.22.1 github.com/go-resty/resty/v2 v2.15.3 github.com/google/go-cmp v0.7.0 - github.com/google/go-github/v50 v50.2.0 github.com/google/uuid v1.6.0 github.com/hashicorp/consul/sdk v0.16.2 github.com/minio/minio-go/v7 v7.0.68 @@ -34,8 +32,6 @@ require ( go.opentelemetry.io/otel v1.35.0 go.opentelemetry.io/otel/sdk v1.34.0 go.opentelemetry.io/otel/trace v1.35.0 - go.uber.org/ratelimit v0.3.1 - golang.org/x/oauth2 v0.25.0 golang.org/x/sync v0.13.0 gopkg.in/guregu/null.v4 v4.0.0 ) @@ -44,7 +40,6 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/aws/aws-sdk-go-v2 v1.31.0 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.37 // indirect @@ -58,10 +53,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.3 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.31.3 // indirect github.com/aws/smithy-go v1.21.0 // indirect - github.com/benbjohnson/clock v1.3.0 // indirect github.com/bits-and-blooms/bitset v1.17.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cloudflare/circl v1.1.0 // indirect github.com/consensys/bavard v0.1.22 // indirect github.com/consensys/gnark-crypto v0.14.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -87,7 +80,6 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/holiman/uint256 v1.3.2 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/framework/go.sum b/framework/go.sum index 422d9f9dc..b7888bd24 100644 --- a/framework/go.sum +++ b/framework/go.sum @@ -10,8 +10,6 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= -github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= @@ -46,15 +44,12 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.31.3 h1:VzudTFrDCIDakXtemR7l6Qzt2+JY github.com/aws/aws-sdk-go-v2/service/sts v1.31.3/go.mod h1:yMWe0F+XG0DkRZK5ODZhG7BEFYhLXi2dqGsv6tX0cgI= github.com/aws/smithy-go v1.21.0 h1:H7L8dtDRk0P1Qm6y0ji7MCYMQObJ5R9CRpyPhRUkLYA= github.com/aws/smithy-go v1.21.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.17.0 h1:1X2TS7aHz1ELcC0yU1y2stUs/0ig5oMU6STFZGrhvHI= github.com/bits-and-blooms/bitset v1.17.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/block-vision/sui-go-sdk v1.0.6 h1:FysCc4TJC8v4BEBbCjJPDR4iR5eKqJT1dxGwsT67etg= github.com/block-vision/sui-go-sdk v1.0.6/go.mod h1:FyK1vGE8lWm9QA1fdQpf1agfXQSMbPT8AV1BICgx6d8= -github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -63,8 +58,6 @@ github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= -github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= @@ -129,8 +122,6 @@ github.com/ethereum/go-ethereum v1.15.0 h1:LLb2jCPsbJZcB4INw+E/MgzUX5wlR6SdwXcv0 github.com/ethereum/go-ethereum v1.15.0/go.mod h1:4q+4t48P2C03sjqGvTXix5lEOplf5dz4CTosbjt5tGs= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -172,14 +163,9 @@ github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgj github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v50 v50.2.0 h1:j2FyongEHlO9nxXLc+LP3wuBSVU9mVxfpdYUexMpIfk= -github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -375,14 +361,9 @@ go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= -go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -402,12 +383,9 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= -golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -421,11 +399,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -436,7 +411,6 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/framework/observability.go b/framework/observability.go index a51f754ca..8c39aadf6 100644 --- a/framework/observability.go +++ b/framework/observability.go @@ -13,9 +13,9 @@ import ( var EmbeddedObservabilityFiles embed.FS const ( - LocalCLNodeErrorsURL = "http://localhost:3000/d/a7de535b-3e0f-4066-bed7-d505b6ec9ef1/cl-node-errors?orgId=1" - LocalWorkflowEngineURL = "http://localhost:3000/d/ce589a98-b4be-4f80-bed1-bc62f3e4414a/workflow-engine?orgId=1&refresh=30s" - LocalLogsURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" + LocalCLNodeErrorsURL = "http://localhost:3000/d/a7de535b-3e0f-4066-bed7-d505b6ec9ef1/cl-node-errors?orgId=1&refresh=5s" + LocalWorkflowEngineURL = "http://localhost:3000/d/ce589a98-b4be-4f80-bed1-bc62f3e4414a/workflow-engine?orgId=1&refresh=5s" + LocalLogsURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-15m%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" LocalPrometheusURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22PBFA97CFB590B2093%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22range%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22PBFA97CFB590B2093%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" LocalPostgresDebugURL = "http://localhost:3000/d/000000039/postgresql-database?orgId=1&refresh=10s&var-DS_PROMETHEUS=PBFA97CFB590B2093&var-interval=$__auto_interval_interval&var-namespace=&var-release=&var-instance=postgres_exporter_0:9187&var-datname=All&var-mode=All&from=now-5m&to=now" LocalPyroScopeURL = "http://localhost:4040" From 13d4bb678bdfb8c00313cfa90eab8fa4548c179b Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 8 Jul 2025 19:10:05 +0200 Subject: [PATCH 3/4] set all URLs to 15m/5s refresh, fix shadowed error in DefaultNetwork --- framework/.changeset/v0.10.1.md | 3 ++ framework/cmd/main.go | 63 +++------------------------------ framework/config.go | 13 ++++--- framework/observability.go | 14 ++++---- 4 files changed, 23 insertions(+), 70 deletions(-) create mode 100644 framework/.changeset/v0.10.1.md diff --git a/framework/.changeset/v0.10.1.md b/framework/.changeset/v0.10.1.md new file mode 100644 index 000000000..88dcd211d --- /dev/null +++ b/framework/.changeset/v0.10.1.md @@ -0,0 +1,3 @@ +- Expose observability stack functions to reuse in devenv repository +- Set all dashboard URLs to 15m/5s +- Add config functions compatible with devenv \ No newline at end of file diff --git a/framework/cmd/main.go b/framework/cmd/main.go index d999b52ac..e08e59906 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "io/fs" "log" "os" "path/filepath" @@ -115,7 +114,7 @@ func main() { Aliases: []string{"u"}, Description: "Spins up Blockscout stack", Action: func(c *cli.Context) error { - return framework.BlockscoutUp(c.String("rpc")) + return framework.BlockScoutUp(c.String("rpc")) }, }, { @@ -124,7 +123,7 @@ func main() { Aliases: []string{"d"}, Description: "Removes Blockscout stack, wipes all Blockscout databases data", Action: func(c *cli.Context) error { - return framework.BlockscoutDown(c.String("rpc")) + return framework.BlockScoutDown(c.String("rpc")) }, }, { @@ -134,10 +133,10 @@ func main() { Description: "Reboots Blockscout stack", Action: func(c *cli.Context) error { rpc := c.String("rpc") - if err := framework.BlockscoutDown(rpc); err != nil { + if err := framework.BlockScoutDown(rpc); err != nil { return err } - return framework.BlockscoutUp(rpc) + return framework.BlockScoutUp(rpc) }, }, }, @@ -151,60 +150,6 @@ func main() { } } -// extractAllFiles goes through the embedded directory and extracts all files to the current directory -func extractAllFiles(embeddedDir string) error { - // Get current working directory where CLI is running - currentDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - - // Walk through the embedded files - err = fs.WalkDir(framework.EmbeddedObservabilityFiles, embeddedDir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("error walking the directory: %w", err) - } - if strings.Contains(path, "README.md") { - return nil - } - - // Skip directories - if d.IsDir() { - return nil - } - - // Read file content from embedded file system - content, err := framework.EmbeddedObservabilityFiles.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read file %s: %w", path, err) - } - - // Determine the target path (strip out the `embeddedDir` part) - relativePath, err := filepath.Rel(embeddedDir, path) - if err != nil { - return fmt.Errorf("failed to determine relative path for %s: %w", path, err) - } - targetPath := filepath.Join(currentDir, relativePath) - - // Create target directories if necessary - targetDir := filepath.Dir(targetPath) - err = os.MkdirAll(targetDir, os.ModePerm) - if err != nil { - return fmt.Errorf("failed to create directory %s: %w", targetDir, err) - } - - // Write the file content to the target path - //nolint - err = os.WriteFile(targetPath, content, 0777) - if err != nil { - return fmt.Errorf("failed to write file %s: %w", targetPath, err) - } - return nil - }) - - return err -} - // PrettyPrintTOML pretty prints TOML func PrettyPrintTOML(inputFile string, outputFile string) error { tomlData, err := os.ReadFile(inputFile) diff --git a/framework/config.go b/framework/config.go index 223b2983c..594b669fe 100644 --- a/framework/config.go +++ b/framework/config.go @@ -224,6 +224,7 @@ func LoadCache[X any](t *testing.T) (*X, error) { return Load[X](t) } +// BaseConfigPath returns base config path, ex. env.toml,overrides.toml -> env.toml func BaseConfigPath() (string, error) { configs := os.Getenv("CTF_CONFIGS") if configs == "" { @@ -233,6 +234,7 @@ func BaseConfigPath() (string, error) { return strings.Split(configs, ",")[0], nil } +// BaseConfigName returns base config name, ex. env.toml -> env func BaseConfigName() (string, error) { cp, err := BaseConfigPath() if err != nil { @@ -241,6 +243,7 @@ func BaseConfigName() (string, error) { return strings.Replace(cp, ".toml", "", -1), nil } +// BaseCacheName returns base cache file name, ex.: env.toml -> env-cache.toml func BaseCacheName() (string, error) { cp, err := BaseConfigPath() if err != nil { @@ -252,15 +255,17 @@ func BaseCacheName() (string, error) { func DefaultNetwork(once *sync.Once) error { var net *testcontainers.DockerNetwork - var err error + var innerErr error once.Do(func() { - net, err = network.New( + net, innerErr = network.New( context.Background(), network.WithLabels(map[string]string{"framework": "ctf"}), ) - DefaultNetworkName = net.Name + if innerErr == nil { + DefaultNetworkName = net.Name + } }) - return err + return innerErr } func RenderTemplate(tmpl string, data interface{}) (string, error) { diff --git a/framework/observability.go b/framework/observability.go index 8c39aadf6..2164cb3b4 100644 --- a/framework/observability.go +++ b/framework/observability.go @@ -14,11 +14,11 @@ var EmbeddedObservabilityFiles embed.FS const ( LocalCLNodeErrorsURL = "http://localhost:3000/d/a7de535b-3e0f-4066-bed7-d505b6ec9ef1/cl-node-errors?orgId=1&refresh=5s" - LocalWorkflowEngineURL = "http://localhost:3000/d/ce589a98-b4be-4f80-bed1-bc62f3e4414a/workflow-engine?orgId=1&refresh=5s" - LocalLogsURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-15m%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" - LocalPrometheusURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22PBFA97CFB590B2093%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22range%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22PBFA97CFB590B2093%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" - LocalPostgresDebugURL = "http://localhost:3000/d/000000039/postgresql-database?orgId=1&refresh=10s&var-DS_PROMETHEUS=PBFA97CFB590B2093&var-interval=$__auto_interval_interval&var-namespace=&var-release=&var-instance=postgres_exporter_0:9187&var-datname=All&var-mode=All&from=now-5m&to=now" - LocalPyroScopeURL = "http://localhost:4040" + LocalWorkflowEngineURL = "http://localhost:3000/d/ce589a98-b4be-4f80-bed1-bc62f3e4414a/workflow-engine?orgId=1&refresh=5s&from=now-15m&to=now" + LocalLogsURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22ctf%5C%22%7D%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-15m%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" + LocalPrometheusURL = "http://localhost:3000/explore?panes=%7B%22qZw%22:%7B%22datasource%22:%22PBFA97CFB590B2093%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%22,%22range%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22PBFA97CFB590B2093%22%7D%7D%5D,%22range%22:%7B%22from%22:%22now-15m%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" + LocalPostgresDebugURL = "http://localhost:3000/d/000000039/postgresql-database?orgId=1&refresh=5s&var-DS_PROMETHEUS=PBFA97CFB590B2093&var-interval=$__auto_interval_interval&var-namespace=&var-release=&var-instance=postgres_exporter_0:9187&var-datname=All&var-mode=All&from=now-15m&to=now" + LocalPyroScopeURL = "http://localhost:4040/?query=process_cpu%3Acpu%3Ananoseconds%3Acpu%3Ananoseconds%7Bservice_name%3D%22chainlink-node%22%7D&from=now-15m" ) // extractAllFiles goes through the embedded directory and extracts all files to the current directory @@ -75,7 +75,7 @@ func extractAllFiles(embeddedDir string) error { return err } -func BlockscoutUp(url string) error { +func BlockScoutUp(url string) error { L.Info().Msg("Creating local Blockscout stack") if err := extractAllFiles("observability"); err != nil { return err @@ -93,7 +93,7 @@ func BlockscoutUp(url string) error { return nil } -func BlockscoutDown(url string) error { +func BlockScoutDown(url string) error { L.Info().Msg("Removing local Blockscout stack") os.Setenv("BLOCKSCOUT_RPC_URL", url) err := RunCommand("bash", "-c", fmt.Sprintf(` From 0da767caeb36c0e1a50d3e454ef95db574727777 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 8 Jul 2025 19:33:30 +0200 Subject: [PATCH 4/4] cleanup docs --- book/src/SUMMARY.md | 2 -- book/src/framework/components/analyze_ci.md | 39 --------------------- book/src/framework/components/debug_ci.md | 29 --------------- 3 files changed, 70 deletions(-) delete mode 100644 book/src/framework/components/analyze_ci.md delete mode 100644 book/src/framework/components/debug_ci.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 7f7346e4f..e94b61e02 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -19,8 +19,6 @@ - [Test Configuration](./framework/test_configuration_overrides.md) - [Exposing Components](framework/components/state.md) - [Debugging Tests](framework/components/debug.md) - - [Debugging CI Runs](framework/components/debug_ci.md) - - [Analyzing CI Runs](framework/components/analyze_ci.md) - [Debugging K8s Chaos Tests](framework/chaos/debug-k8s.md) - [Components Cleanup](framework/components/cleanup.md) - [Components Caching](framework/components/caching.md) diff --git a/book/src/framework/components/analyze_ci.md b/book/src/framework/components/analyze_ci.md deleted file mode 100644 index dc0ff37d4..000000000 --- a/book/src/framework/components/analyze_ci.md +++ /dev/null @@ -1,39 +0,0 @@ -# Analyzing CI Runs - -We offer a straightforward CLI tool designed to analyze CI runs, focusing on Jobs and Steps, to provide deeper insights into system-level tests. - -![img_1.png](img_1.png) - -## What to look for - -Check execution times, results are sorted by `median`. - -We exclude any runs without `conclusion` or if they have `conclusion: "skipped"`. - -🔄 - if you see high numbers here it means your pipeline is flaky, and it does not pass from the first attempt on some commits. It may indicate test flakiness! - -🚫 - if numbers are high there it means your pipeline is constantly `cancelled` so it wastes time without bringing any results/value! - -## Examples -``` -# GITHUB_TOKEN must have access to "actions" API -export GITHUB_TOKEN=... - -# E2E test jobs from core for the last day -ctf ci -r "smartcontractkit/chainlink" -w "CI Core" -t jobs - -# E2E test steps from core for the last day -ctf ci -r "smartcontractkit/chainlink" -w "CI Core" -t steps - -# E2E tests from core for the last week (1k jobs limit!) -ctf ci -r "smartcontractkit/chainlink" -w "CI Core" -t jobs -s 7 - -# E2E tests from core for some specific days from 5 days ago to 3 days ago -ctf ci -r "smartcontractkit/chainlink" -w "CI Core" -t jobs -s 5 -e 3 - -# Last 3 days runs for e2e framework tests -ctf ci -r "smartcontractkit/chainlink-testing-framework" -w "Framework Golden Tests Examples" -t jobs -s 3 -``` -You can also use `-debug` flag to dump all the workflow runs/jobs to `ctf-ci-debug` folder. - -All the results are also saved by default into `ctf-ci-$uuid.json` files so you can use this tool as CI performance linter. diff --git a/book/src/framework/components/debug_ci.md b/book/src/framework/components/debug_ci.md deleted file mode 100644 index b848ed667..000000000 --- a/book/src/framework/components/debug_ci.md +++ /dev/null @@ -1,29 +0,0 @@ -# Debugging CI Runs - -Combining test and container logs for debugging in CI can be cumbersome, so we’ve simplified the process with a command that downloads, unzips, and uploads them to a local Loki instance, enabling you to leverage the full power of LogQL. - -Spin up the stack and upload some data -``` -ctf obs u -``` - -## Raw logs from GitHub step URL - -Go to your test run and get the raw logs URL -![raw-logs-url.png](raw-logs-url.png) - -``` -ctf obs l -u "$your_url" -``` -Click the resulting URL after upload is finished to open the filter - -## Logs from GHA artifacts - -Get the `Run ID` from GitHub UI, ex `actions/runs/$run_id`, download the artifact (in case of `CTFv2` it'd be just one dir), then run -``` -gh run download $run_id -ctf obs l -d $artifact_dir -``` -Click the resulting URL after upload is finished to open the filter - -