|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "bufio" |
| 5 | + "bytes" |
| 6 | + "encoding/json" |
| 7 | + "fmt" |
| 8 | + "github.com/pkg/errors" |
| 9 | + "github.com/smartcontractkit/chainlink-testing-framework/framework" |
| 10 | + "io" |
| 11 | + "net/http" |
| 12 | + "os" |
| 13 | + "path/filepath" |
| 14 | + "strings" |
| 15 | + "sync" |
| 16 | + "time" |
| 17 | + |
| 18 | + "go.uber.org/ratelimit" |
| 19 | +) |
| 20 | + |
| 21 | +var L = framework.L |
| 22 | + |
| 23 | +type LokiPushRequest struct { |
| 24 | + Streams []LokiStream `json:"streams"` |
| 25 | +} |
| 26 | + |
| 27 | +type LokiStream struct { |
| 28 | + Stream map[string]string `json:"stream"` |
| 29 | + Values [][2]string `json:"values"` |
| 30 | +} |
| 31 | + |
| 32 | +const ( |
| 33 | + lokiURL = "http://localhost:3030/loki/api/v1/push" |
| 34 | + grafanaURL = "http://localhost:3000/explore?panes=%7B%22V0P%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22" |
| 35 | + grafanaURL2 = "%5C%22%7D%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" |
| 36 | +) |
| 37 | + |
| 38 | +func processAndUploadDir(dirPath string, limiter ratelimit.Limiter, chunks int, jobID string) error { |
| 39 | + return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { |
| 40 | + if err != nil { |
| 41 | + return errors.Wrapf(err, "error accessing file: %s", path) |
| 42 | + } |
| 43 | + if info.IsDir() { |
| 44 | + return nil |
| 45 | + } |
| 46 | + L.Info().Msgf("Processing file: %s", path) |
| 47 | + f, err := os.Open(path) |
| 48 | + if err != nil { |
| 49 | + return errors.Wrapf(err, "error opening file: %s", path) |
| 50 | + } |
| 51 | + defer f.Close() |
| 52 | + |
| 53 | + if err := processAndUploadLog(path, f, limiter, chunks, jobID); err != nil { |
| 54 | + return errors.Wrapf(err, "error processing file: %s", path) |
| 55 | + } |
| 56 | + return nil |
| 57 | + }) |
| 58 | +} |
| 59 | + |
| 60 | +func processAndUploadLog(source string, r io.Reader, limiter ratelimit.Limiter, chunks int, jobID string) error { |
| 61 | + scanner := bufio.NewScanner(r) |
| 62 | + var values [][2]string |
| 63 | + baseTime := time.Now() |
| 64 | + |
| 65 | + // Read all log lines; each line gets a unique timestamp. |
| 66 | + for scanner.Scan() { |
| 67 | + line := scanner.Text() |
| 68 | + ts := baseTime.UnixNano() |
| 69 | + values = append(values, [2]string{fmt.Sprintf("%d", ts), line}) |
| 70 | + baseTime = baseTime.Add(time.Nanosecond) |
| 71 | + } |
| 72 | + if err := scanner.Err(); err != nil { |
| 73 | + return fmt.Errorf("error scanning logs from %s: %w", source, err) |
| 74 | + } |
| 75 | + |
| 76 | + totalLines := len(values) |
| 77 | + if totalLines == 0 { |
| 78 | + L.Info().Msgf("No log lines found in %s", source) |
| 79 | + return nil |
| 80 | + } |
| 81 | + // Some logs may include CL node logs, skip chunking for all that is less |
| 82 | + if totalLines <= 10000 { |
| 83 | + chunks = 1 |
| 84 | + } |
| 85 | + if chunks > totalLines { |
| 86 | + chunks = totalLines |
| 87 | + } |
| 88 | + chunkSize := totalLines / chunks |
| 89 | + remainder := totalLines % chunks |
| 90 | + L.Debug().Int("total_lines", totalLines). |
| 91 | + Int("chunks", chunks). |
| 92 | + Msgf("Starting chunk processing for %s", source) |
| 93 | + var wg sync.WaitGroup |
| 94 | + errCh := make(chan error, chunks) |
| 95 | + start := 0 |
| 96 | + for i := 0; i < chunks; i++ { |
| 97 | + extra := 0 |
| 98 | + if i < remainder { |
| 99 | + extra = 1 |
| 100 | + } |
| 101 | + end := start + chunkSize + extra |
| 102 | + chunkValues := values[start:end] |
| 103 | + startLine := start + 1 |
| 104 | + endLine := end |
| 105 | + start = end |
| 106 | + |
| 107 | + labels := map[string]string{ |
| 108 | + "job": jobID, |
| 109 | + "chunk": fmt.Sprintf("%d", i+1), |
| 110 | + "source": source, |
| 111 | + } |
| 112 | + reqBody := LokiPushRequest{ |
| 113 | + Streams: []LokiStream{ |
| 114 | + { |
| 115 | + Stream: labels, |
| 116 | + Values: chunkValues, |
| 117 | + }, |
| 118 | + }, |
| 119 | + } |
| 120 | + data, err := json.Marshal(reqBody) |
| 121 | + if err != nil { |
| 122 | + return fmt.Errorf("error marshaling JSON for chunk %d: %w", i+1, err) |
| 123 | + } |
| 124 | + chunkMB := float64(len(data)) / (1024 * 1024) |
| 125 | + L.Debug().Int("chunk", i+1). |
| 126 | + Float64("chunk_size_MB", chunkMB). |
| 127 | + Int("start_line", startLine). |
| 128 | + Int("end_line", endLine). |
| 129 | + Msg("Prepared chunk for upload") |
| 130 | + |
| 131 | + wg.Add(1) |
| 132 | + go func(chunkNum, sLine, eLine int, payload []byte, sizeMB float64) { |
| 133 | + defer wg.Done() |
| 134 | + const maxRetries = 50 |
| 135 | + const retryDelay = 1 * time.Second |
| 136 | + |
| 137 | + var resp *http.Response |
| 138 | + var attempt int |
| 139 | + var err error |
| 140 | + for attempt = 1; attempt <= maxRetries; attempt++ { |
| 141 | + limiter.Take() |
| 142 | + resp, err = http.Post(lokiURL, "application/json", bytes.NewReader(payload)) |
| 143 | + if err != nil { |
| 144 | + if strings.Contains(err.Error(), "connection refused") { |
| 145 | + L.Fatal().Msg("connection refused, is local Loki up and running? use 'ctf obs u'") |
| 146 | + return |
| 147 | + } |
| 148 | + L.Error().Err(err). |
| 149 | + Int("status", resp.StatusCode). |
| 150 | + Int("attempt", attempt). |
| 151 | + Int("chunk", chunkNum). |
| 152 | + Float64("chunk_size_MB", sizeMB). |
| 153 | + Msg("Error sending POST request") |
| 154 | + time.Sleep(retryDelay) |
| 155 | + continue |
| 156 | + } |
| 157 | + |
| 158 | + body, _ := io.ReadAll(resp.Body) |
| 159 | + resp.Body.Close() |
| 160 | + |
| 161 | + if resp.StatusCode == 429 { |
| 162 | + L.Debug().Int("attempt", attempt). |
| 163 | + Int("chunk", chunkNum). |
| 164 | + Float64("chunk_size_MB", sizeMB). |
| 165 | + Msg("Received 429, retrying...") |
| 166 | + time.Sleep(retryDelay) |
| 167 | + continue |
| 168 | + } |
| 169 | + |
| 170 | + if resp.StatusCode/100 != 2 { |
| 171 | + err = fmt.Errorf("loki error: %s - %s", resp.Status, body) |
| 172 | + L.Error().Err(err).Int("chunk", chunkNum). |
| 173 | + Float64("chunk_size_MB", sizeMB). |
| 174 | + Msg("Chunk upload failed") |
| 175 | + time.Sleep(retryDelay) |
| 176 | + continue |
| 177 | + } |
| 178 | + |
| 179 | + L.Info().Int("chunk", chunkNum). |
| 180 | + Float64("chunk_size_MB", sizeMB). |
| 181 | + Msg("Successfully uploaded chunk") |
| 182 | + return |
| 183 | + } |
| 184 | + errCh <- fmt.Errorf("max retries reached for chunk %d; last error: %v", chunkNum, err) |
| 185 | + }(i+1, startLine, endLine, data, chunkMB) |
| 186 | + } |
| 187 | + |
| 188 | + wg.Wait() |
| 189 | + close(errCh) |
| 190 | + if len(errCh) > 0 { |
| 191 | + return <-errCh |
| 192 | + } |
| 193 | + |
| 194 | + return nil |
| 195 | +} |
0 commit comments