Skip to content

Commit 39418a7

Browse files
skudasovskudasov
committed
upload GHA logs to local Loki
1 parent 3892728 commit 39418a7

File tree

6 files changed

+87
-28
lines changed

6 files changed

+87
-28
lines changed

‎book/src/SUMMARY.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
- [Test Configuration](./framework/test_configuration_overrides.md)
2121
- [Exposing Components](framework/components/state.md)
2222
- [Debugging Tests](framework/components/debug.md)
23+
- [Debugging CI Runs](framework/components/debug_ci.md)
2324
- [Debugging K8s Chaos Tests](framework/chaos/debug-k8s.md)
2425
- [Components Cleanup](framework/components/cleanup.md)
2526
- [Components Caching](framework/components/caching.md)

‎book/src/framework/components/raw-logs-url.png‎

88.4 KB
Loading

‎framework/cmd/logs.go‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ import (
55
"bytes"
66
"encoding/json"
77
"fmt"
8+
"github.com/pkg/errors"
89
"github.com/smartcontractkit/chainlink-testing-framework/framework"
910
"io"
1011
"net/http"
1112
"os"
1213
"path/filepath"
14+
"strings"
1315
"sync"
1416
"time"
1517

@@ -40,23 +42,20 @@ const (
4042
func processAndUploadDir(dirPath string, limiter ratelimit.Limiter, chunks int, jobID string) error {
4143
return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
4244
if err != nil {
43-
L.Error().Err(err).Msgf("Error accessing %s", path)
44-
return nil
45+
return errors.Wrapf(err, "error accessing file: %s", path)
4546
}
4647
if info.IsDir() {
4748
return nil
4849
}
4950
L.Info().Msgf("Processing file: %s", path)
5051
f, err := os.Open(path)
5152
if err != nil {
52-
L.Error().Err(err).Msgf("Error opening file %s", path)
53-
return nil
53+
return errors.Wrapf(err, "error opening file: %s", path)
5454
}
5555
defer f.Close()
5656

5757
if err := processAndUploadLog(path, f, limiter, chunks, jobID); err != nil {
58-
L.Error().Err(err).Msgf("Error processing file %s", path)
59-
// Continue processing other files even if one fails.
58+
return errors.Wrapf(err, "error processing file: %s", path)
6059
}
6160
return nil
6261
})
@@ -149,7 +148,13 @@ func processAndUploadLog(source string, r io.Reader, limiter ratelimit.Limiter,
149148
limiter.Take()
150149
resp, err = http.Post(lokiURL, "application/json", bytes.NewReader(payload))
151150
if err != nil {
152-
L.Error().Err(err).Int("attempt", attempt).
151+
if strings.Contains(err.Error(), "connection refused") {
152+
L.Fatal().Msg("connection refused, is local Loki up and running? use 'ctf obs u'")
153+
return
154+
}
155+
L.Error().Err(err).
156+
Int("status", resp.StatusCode).
157+
Int("attempt", attempt).
153158
Int("chunk", chunkNum).
154159
Float64("chunk_size_MB", sizeMB).
155160
Msg("Error sending POST request")

‎framework/cmd/observability.go‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,50 +3,38 @@ package main
33
import (
44
"fmt"
55
"github.com/google/uuid"
6+
"github.com/pkg/errors"
67
"github.com/smartcontractkit/chainlink-testing-framework/framework"
78
"go.uber.org/ratelimit"
89
"net/http"
910
)
1011

1112
func loadLogs(rawURL, dirPath string, rps, chunks int) error {
12-
framework.L.Info().Msg("Loading logs into Loki")
13-
sources := 0
14-
if rawURL != "" {
15-
sources++
16-
}
17-
if dirPath != "" {
18-
sources++
19-
}
20-
if sources != 1 {
21-
L.Error().Msg("Usage: provide exactly one of -raw-url or -dir")
22-
return nil
13+
if rawURL == "" && dirPath == "" {
14+
return fmt.Errorf("at least one source must be provided, either -u $url or -d $dir")
2315
}
2416
jobID := uuid.New().String()[0:5]
25-
L.Info().Msgf("Using unique job identifier: %s", jobID)
17+
framework.L.Info().Str("JobID", jobID).Msg("Loading logs into Loki")
2618
limiter := ratelimit.New(rps)
2719
if rawURL != "" {
2820
L.Info().Msg("Downloading raw logs from URL")
2921
resp, err := http.Get(rawURL)
3022
if err != nil {
31-
L.Error().Err(err).Msg("Error downloading raw logs")
32-
return nil
23+
return errors.Wrap(err, "error downloading raw logs")
3324
}
3425
defer resp.Body.Close()
3526

3627
if resp.StatusCode/100 != 2 {
37-
L.Error().Msgf("Non-success response downloading raw logs: %s", resp.Status)
38-
return nil
28+
return fmt.Errorf("non-success response code when downloading raw logs: %s", resp.Status)
3929
}
4030

4131
if err := processAndUploadLog(rawURL, resp.Body, limiter, chunks, jobID); err != nil {
42-
L.Error().Err(err).Msg("Error processing raw logs")
43-
return nil
32+
return errors.Wrap(err, "error processing raw logs")
4433
}
4534
} else if dirPath != "" {
4635
L.Info().Msgf("Processing directory: %s", dirPath)
4736
if err := processAndUploadDir(dirPath, limiter, chunks, jobID); err != nil {
48-
L.Error().Err(err).Msg("Error processing directory")
49-
return nil
37+
return errors.Wrapf(err, "error processing directory: %s", dirPath)
5038
}
5139
}
5240
framework.L.Info().Str("JobID", jobID).Str("URL", grafanaURL+jobID+grafanaURL2).Msg("Upload complete")

‎framework/examples/myproject/go.mod‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ require (
1818
github.com/go-resty/resty/v2 v2.15.3
1919
github.com/google/uuid v1.6.0
2020
github.com/rs/zerolog v1.33.0
21-
github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.8
21+
github.com/smartcontractkit/chainlink-testing-framework/framework v0.5.2
2222
github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2
2323
github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10
2424
github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2
@@ -44,6 +44,7 @@ require (
4444
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
4545
github.com/armon/go-metrics v0.4.1 // indirect
4646
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
47+
github.com/atotto/clipboard v0.1.4 // indirect
4748
github.com/avast/retry-go v3.0.0+incompatible // indirect
4849
github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect
4950
github.com/aws/aws-sdk-go v1.45.25 // indirect
@@ -61,6 +62,7 @@ require (
6162
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 // indirect
6263
github.com/aws/aws-sdk-go-v2/service/sts v1.33.0 // indirect
6364
github.com/aws/smithy-go v1.22.1 // indirect
65+
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
6466
github.com/benbjohnson/clock v1.3.5 // indirect
6567
github.com/beorn7/perks v1.0.1 // indirect
6668
github.com/bits-and-blooms/bitset v1.13.0 // indirect
@@ -69,9 +71,18 @@ require (
6971
github.com/bytedance/sonic v1.12.3 // indirect
7072
github.com/bytedance/sonic/loader v0.2.0 // indirect
7173
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect
74+
github.com/catppuccin/go v0.2.0 // indirect
7275
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
7376
github.com/cespare/xxhash v1.1.0 // indirect
7477
github.com/cespare/xxhash/v2 v2.3.0 // indirect
78+
github.com/charmbracelet/bubbles v0.20.0 // indirect
79+
github.com/charmbracelet/bubbletea v1.1.1 // indirect
80+
github.com/charmbracelet/huh v0.6.0 // indirect
81+
github.com/charmbracelet/huh/spinner v0.0.0-20241028115900-20a4d21717a8 // indirect
82+
github.com/charmbracelet/lipgloss v0.13.0 // indirect
83+
github.com/charmbracelet/x/ansi v0.2.3 // indirect
84+
github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect
85+
github.com/charmbracelet/x/term v0.2.0 // indirect
7586
github.com/cloudwego/base64x v0.1.4 // indirect
7687
github.com/cloudwego/iasm v0.2.0 // indirect
7788
github.com/coder/websocket v1.8.12 // indirect
@@ -82,6 +93,7 @@ require (
8293
github.com/coreos/go-semver v0.3.1 // indirect
8394
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
8495
github.com/cpuguy83/dockercfg v0.3.2 // indirect
96+
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
8597
github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
8698
github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
8799
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -96,6 +108,7 @@ require (
96108
github.com/dustin/go-humanize v1.0.1 // indirect
97109
github.com/edsrzf/mmap-go v1.1.0 // indirect
98110
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
111+
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
99112
github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
100113
github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect
101114
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
@@ -171,14 +184,18 @@ require (
171184
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
172185
github.com/kylelemons/godebug v1.1.0 // indirect
173186
github.com/leodido/go-urn v1.4.0 // indirect
187+
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
174188
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
175189
github.com/magiconair/properties v1.8.7 // indirect
176190
github.com/mailru/easyjson v0.7.7 // indirect
177191
github.com/mattn/go-colorable v0.1.13 // indirect
178192
github.com/mattn/go-isatty v0.0.20 // indirect
193+
github.com/mattn/go-localereader v0.0.1 // indirect
194+
github.com/mattn/go-runewidth v0.0.16 // indirect
179195
github.com/miekg/dns v1.1.56 // indirect
180196
github.com/mitchellh/copystructure v1.0.0 // indirect
181197
github.com/mitchellh/go-homedir v1.1.0 // indirect
198+
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
182199
github.com/mitchellh/mapstructure v1.5.0 // indirect
183200
github.com/mitchellh/reflectwalk v1.0.1 // indirect
184201
github.com/mmcloughlin/addchain v0.4.0 // indirect
@@ -193,6 +210,9 @@ require (
193210
github.com/montanaflynn/stats v0.7.1 // indirect
194211
github.com/morikuni/aec v1.0.0 // indirect
195212
github.com/mr-tron/base58 v1.2.0 // indirect
213+
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
214+
github.com/muesli/cancelreader v0.2.2 // indirect
215+
github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
196216
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
197217
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
198218
github.com/oklog/ulid v1.3.1 // indirect
@@ -201,6 +221,7 @@ require (
201221
github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
202222
github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
203223
github.com/opentracing/opentracing-go v1.2.0 // indirect
224+
github.com/pelletier/go-toml v1.9.5 // indirect
204225
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
205226
github.com/pierrec/lz4/v4 v4.1.21 // indirect
206227
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
@@ -215,8 +236,10 @@ require (
215236
github.com/prometheus/exporter-toolkit v0.10.1-0.20230714054209-2f4150c63f97 // indirect
216237
github.com/prometheus/procfs v0.15.1 // indirect
217238
github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510 // indirect
239+
github.com/rivo/uniseg v0.4.7 // indirect
218240
github.com/robfig/cron/v3 v3.0.1 // indirect
219241
github.com/rogpeppe/go-internal v1.13.1 // indirect
242+
github.com/russross/blackfriday/v2 v2.1.0 // indirect
220243
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
221244
github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
222245
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
@@ -241,7 +264,9 @@ require (
241264
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
242265
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
243266
github.com/ugorji/go/codec v1.2.12 // indirect
267+
github.com/urfave/cli/v2 v2.27.5 // indirect
244268
github.com/x448/float16 v0.8.4 // indirect
269+
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
245270
github.com/yusufpapurcu/wmi v1.2.3 // indirect
246271
go.etcd.io/etcd/api/v3 v3.5.14 // indirect
247272
go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect

0 commit comments

Comments
 (0)