Skip to content

Commit b8143f6

Browse files
lionellonullfunc
lionello
and
nullfunc
authored
Revert "Revert "fix verbose logs showing on CLI for compose up (#1595)" (#1599)" (#1600)
Co-authored-by: Eric Liu <[email protected]>
1 parent 3115987 commit b8143f6

File tree

10 files changed

+1273
-40
lines changed

10 files changed

+1273
-40
lines changed

‎src/pkg/cli/client/byoc/aws/stream.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package aws
33
import (
44
"encoding/json"
55
"io"
6+
"regexp"
67
"slices"
78
"strings"
89
"time"
@@ -17,6 +18,8 @@ import (
1718
"google.golang.org/protobuf/types/known/timestamppb"
1819
)
1920

21+
var codeBuildPrefixRegex = regexp.MustCompile(`^[a-zA-Z0-9_-]+-image/`)
22+
2023
// byocServerStream is a wrapper around awsecs.EventStream that implements connect-like ServerStream
2124
type byocServerStream struct {
2225
err error
@@ -76,9 +79,11 @@ func (bs *byocServerStream) parseEvents(events []ecs.LogEvent) *defangv1.TailRes
7679
// We could loop around the select, but returning an empty response updates the spinner.
7780
return nil
7881
}
82+
7983
var response defangv1.TailResponse
8084
parseFirelensRecords := false
8185
parseECSEventRecords := false
86+
parseCodeBuildRecords := false
8287
// Get the Etag/Host/Service from the first entry (should be the same for all events in this batch)
8388
first := events[0]
8489
switch {
@@ -90,8 +95,13 @@ func (bs *byocServerStream) parseEvents(events []ecs.LogEvent) *defangv1.TailRes
9095
// LogStreams: "crun/main/0f2a8ccde0374239bdd04f5e07d8c523"
9196
response.Host = "pulumi"
9297
response.Service = "cd"
98+
case strings.HasSuffix(*first.LogGroupIdentifier, "/builds") && codeBuildPrefixRegex.MatchString(*first.LogStreamName):
99+
response.Host = "codebuild"
100+
response.Service = "cd"
101+
parseCodeBuildRecords = true
93102
case strings.HasSuffix(*first.LogGroupIdentifier, "/builds") && strings.Contains(*first.LogStreamName, "-firelens-"):
94103
// These events are from the Firelens sidecar "<service>/<kaniko>-firelens-<taskID>"; try to parse the JSON
104+
// or ""
95105
// LogStreams: "app-image/kaniko-firelens-babe6cdb246b4c10b5b7093bb294e6c7"
96106
var record logs.FirelensMessage
97107
if err := json.Unmarshal([]byte(*first.Message), &record); err == nil {
@@ -155,7 +165,11 @@ func (bs *byocServerStream) parseEvents(events []ecs.LogEvent) *defangv1.TailRes
155165
entry.Host = evt.Host()
156166
entry.Message = evt.Status()
157167
}
158-
} else if response.Service == "cd" && strings.HasPrefix(entry.Message, logs.ErrorPrefix) {
168+
} else if parseCodeBuildRecords {
169+
entry.Service = "cd"
170+
entry.Etag = response.Etag
171+
entry.Host = response.Host
172+
} else if (response.Service == "cd") && (strings.HasPrefix(entry.Message, logs.ErrorPrefix) || strings.Contains(strings.ToLower(entry.Message), "error:")) {
159173
entry.Stderr = true
160174
}
161175
if entry.Etag != "" && bs.etag != "" && entry.Etag != bs.etag {
@@ -164,6 +178,7 @@ func (bs *byocServerStream) parseEvents(events []ecs.LogEvent) *defangv1.TailRes
164178
if entry.Service != "" && len(bs.services) > 0 && !slices.Contains(bs.services, entry.Service) {
165179
continue
166180
}
181+
167182
entries = append(entries, entry)
168183
}
169184
if len(entries) == 0 {

‎src/pkg/cli/client/byoc/aws/stream_test.go‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
package aws
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
"github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs"
8+
defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
9+
"github.com/stretchr/testify/assert"
10+
"google.golang.org/protobuf/types/known/timestamppb"
11+
)
12+
13+
// ptrString returns a pointer to the given string.
14+
func ptrString(s string) *string {
15+
return &s
16+
}
17+
18+
func ptrInt64(i int64) *int64 {
19+
return &i
20+
}
21+
22+
func TestStreamToLogEvent(t *testing.T) {
23+
var testEtag = "hg2xsgvsldqk"
24+
var testdata = []struct {
25+
event *ecs.LogEvent
26+
wantResp *defangv1.TailResponse
27+
}{
28+
{
29+
// cd message
30+
event: &ecs.LogEvent{
31+
IngestionTime: ptrInt64(1761883448306),
32+
LogGroupIdentifier: ptrString("532501343364:defang-cd-LogGroup-8id1W5WpeWRu"),
33+
LogStreamName: ptrString("crun/main/127bb61dd5f746918f578f32cc1d6d01"),
34+
Timestamp: ptrInt64(1761883446012),
35+
Message: ptrString(" ** Updating service \"flask\""),
36+
},
37+
wantResp: &defangv1.TailResponse{
38+
Service: "cd",
39+
Host: "pulumi",
40+
Entries: []*defangv1.LogEntry{
41+
{
42+
Timestamp: timestamppb.New(time.Unix(1761883446, int64(12000000))),
43+
Message: " ** Updating service \"flask\"",
44+
Stderr: true,
45+
},
46+
},
47+
},
48+
},
49+
{
50+
// error message
51+
event: &ecs.LogEvent{
52+
IngestionTime: ptrInt64(1761883448306),
53+
LogGroupIdentifier: ptrString("532501343364:defang-cd-LogGroup-8id1W5WpeWRu"),
54+
LogStreamName: ptrString("crun/main/127bb61dd5f746918f578f32cc1d6d01"),
55+
Timestamp: ptrInt64(1761883446012),
56+
Message: ptrString(" \u001b[1m\u001b[38;5;2m+ \u001b[0m aws:cloudwatch:LogGroup builds \u001b[1m\u001b[38;5;2mcreating (0s)\u001b[0m \u001b[38;5;1merror: \u001b[0m\u001b[0m sdk-v2/provider2.go:520: sdk.helper_schema: creating CloudWatch Logs Log Group (/Defang/flask-railpack/beta/builds): operation error CloudWatch Logs: CreateLogGroup, https response error StatusCode: 400, RequestID: e324d4dd-4d1b-4f3e-9b83-18d5c8bd24b1, ResourceAlreadyExistsException: The specified log group already exists: [email protected]"),
57+
},
58+
wantResp: &defangv1.TailResponse{
59+
Service: "cd",
60+
Host: "pulumi",
61+
Entries: []*defangv1.LogEntry{
62+
{
63+
Timestamp: timestamppb.New(time.Unix(1761883446, int64(12000000))),
64+
Message: " \u001b[1m\u001b[38;5;2m+ \u001b[0m aws:cloudwatch:LogGroup builds \u001b[1m\u001b[38;5;2mcreating (0s)\u001b[0m \u001b[38;5;1merror: \u001b[0m\u001b[0m sdk-v2/provider2.go:520: sdk.helper_schema: creating CloudWatch Logs Log Group (/Defang/flask-railpack/beta/builds): operation error CloudWatch Logs: CreateLogGroup, https response error StatusCode: 400, RequestID: e324d4dd-4d1b-4f3e-9b83-18d5c8bd24b1, ResourceAlreadyExistsException: The specified log group already exists: [email protected]",
65+
Stderr: true,
66+
},
67+
},
68+
},
69+
},
70+
{
71+
// service message
72+
event: &ecs.LogEvent{
73+
IngestionTime: ptrInt64(1761883448306),
74+
LogGroupIdentifier: ptrString("532501343364:/Defang/django/beta/builds"),
75+
LogStreamName: ptrString("django-image/django_hg2xsgvsldqk/fb1d2a8e-9553-497e-85e4-91a57f8b6ba6"),
76+
Timestamp: ptrInt64(1761883446012),
77+
Message: ptrString("#12 [7/7] RUN python manage.py collectstatic --noinput\n"),
78+
},
79+
wantResp: &defangv1.TailResponse{
80+
Service: "cd",
81+
Host: "codebuild",
82+
Entries: []*defangv1.LogEntry{
83+
{
84+
Timestamp: timestamppb.New(time.Unix(1761883446, int64(12000000))),
85+
Message: "#12 [7/7] RUN python manage.py collectstatic --noinput\n",
86+
Stderr: false,
87+
},
88+
},
89+
},
90+
},
91+
{
92+
// ECS message
93+
event: &ecs.LogEvent{
94+
IngestionTime: ptrInt64(1761883448306),
95+
LogGroupIdentifier: ptrString("532501343364:/Defang/django/beta/ecs"),
96+
LogStreamName: ptrString("7127bdd6-6e73-3d4e-8c97-18c3071004af"),
97+
Timestamp: ptrInt64(1761883446012),
98+
Message: ptrString("{\"version\":\"0\",\"id\":\"f3a2b329-b75e-ba8b-9cc7-d4a488abc19f\",\"detail-type\":\"ECS Service Action\",\"source\":\"aws.ecs\",\"account\":\"532501343364\",\"time\":\"2025-10-31T05:35:16Z\",\"region\":\"us-west-2\",\"resources\":[\"arn:aws:ecs:us-west-2:532501343364:service/Defang-django-beta-cluster/django_django-db006d3\"],\"detail\":{\"eventType\":\"INFO\",\"eventName\":\"CAPACITY_PROVIDER_STEADY_STATE\",\"clusterArn\":\"arn:aws:ecs:us-west-2:532501343364:cluster/Defang-django-beta-cluster\",\"capacityProviderArns\":[\"arn:aws:ecs:us-west-2:532501343364:capacity-provider/FARGATE_SPOT\"],\"createdAt\":\"2025-10-31T05:35:16.536Z\"}}"),
99+
},
100+
wantResp: nil,
101+
},
102+
{
103+
// railpack message
104+
event: &ecs.LogEvent{
105+
IngestionTime: ptrInt64(1762148772888),
106+
LogGroupIdentifier: ptrString("532501343364:defang-cd-LogGroup-8id1W5WpeWRu"),
107+
LogStreamName: ptrString("crun/main/a467a0afd56d44baab32bb5cceb10da0"),
108+
Timestamp: ptrInt64(1762148771990),
109+
Message: ptrString(" \u001b[1m\u001b[38;5;2m+ \u001b[0m aws:iam:RolePolicy code-build-role-codebuildPolicy \u001b[1m\u001b[38;5;2mcreating (0s)\u001b[0m"),
110+
},
111+
wantResp: &defangv1.TailResponse{
112+
Service: "cd",
113+
Host: "pulumi",
114+
Entries: []*defangv1.LogEntry{
115+
{
116+
Timestamp: timestamppb.New(time.Unix(1762148771, int64(990000000))),
117+
Message: " \u001b[1m\u001b[38;5;2m+ \u001b[0m aws:iam:RolePolicy code-build-role-codebuildPolicy \u001b[1m\u001b[38;5;2mcreating (0s)\u001b[0m",
118+
Stderr: false,
119+
},
120+
},
121+
},
122+
},
123+
{
124+
// service message
125+
event: &ecs.LogEvent{
126+
IngestionTime: ptrInt64(1762144097682),
127+
LogGroupIdentifier: ptrString("532501343364:/Defang/django6/beta/logs"),
128+
LogStreamName: ptrString("django/django_hg2xsgvsldqk/b89c0f0e35ad4357852f0d7cafd488eb"),
129+
Timestamp: ptrInt64(1762144092418),
130+
Message: ptrString(" Applying admin.0001_initial... OK"),
131+
},
132+
wantResp: &defangv1.TailResponse{
133+
Service: "django",
134+
Host: "b89c0f0e35ad4357852f0d7cafd488eb",
135+
Entries: []*defangv1.LogEntry{
136+
{
137+
Timestamp: timestamppb.New(time.Unix(1762144092, int64(418000000))),
138+
Message: " Applying admin.0001_initial... OK",
139+
Stderr: false,
140+
},
141+
},
142+
},
143+
},
144+
}
145+
146+
var byocServiceStream = newByocServerStream(nil, testEtag, []string{"cd", "app", "django"}, nil)
147+
148+
for _, td := range testdata {
149+
tailResp := byocServiceStream.parseEvents([]ecs.LogEvent{*td.event})
150+
if (td.wantResp == nil) != (tailResp == nil) {
151+
t.Errorf("nil mismatch: expected %v, got %v", td.wantResp, tailResp)
152+
continue
153+
} else if (td.wantResp == nil) && (tailResp == nil) {
154+
// no enties expected
155+
continue
156+
}
157+
158+
assert.Equal(t, td.wantResp.Service, tailResp.Service)
159+
assert.Equal(t, td.wantResp.Host, tailResp.Host)
160+
161+
got := tailResp.Entries[0]
162+
want := td.wantResp.Entries[0]
163+
if !got.Timestamp.AsTime().Equal(want.Timestamp.AsTime()) {
164+
t.Errorf("Timestamp = %v; want %v", got.Timestamp, want.Timestamp)
165+
}
166+
if got.Message != want.Message {
167+
t.Errorf("Message = %v; want %v", got.Message, want.Message)
168+
}
169+
if got.Stderr != want.Stderr {
170+
t.Errorf("Stderr = %v; want %v", got.Stderr, want.Stderr)
171+
}
172+
}
173+
}

‎src/pkg/cli/client/byoc/gcp/stream.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -370,8 +370,9 @@ func getLogEntryParser(ctx context.Context, gcpClient *gcp.Gcp) func(entry *logg
370370
} else if entry.GetJsonPayload() != nil && entry.GetJsonPayload().GetFields()["cos.googleapis.com/stream"] != nil {
371371
stderr = entry.GetJsonPayload().GetFields()["cos.googleapis.com/stream"].GetStringValue() == "stderr"
372372
}
373-
374-
// fmt.Printf("ENTRY: %+v\n", entry)
373+
if strings.Contains(strings.ToLower(msg), "error:") {
374+
stderr = true
375+
}
375376

376377
var serviceName, etag, host string
377378
serviceName = entry.Labels["defang-service"]

‎src/pkg/cli/estimate.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,13 @@ func GeneratePreview(ctx context.Context, project *compose.Project, client clien
7676
Verbose: true,
7777
}
7878

79-
err = streamLogs(ctx, previewProvider, project.Name, tailOptions, func(entry *defangv1.LogEntry, options *TailOptions) error {
79+
err = streamLogs(ctx, previewProvider, project.Name, tailOptions, func(entry *defangv1.LogEntry, options *TailOptions, t *term.Term) error {
8080
if strings.HasPrefix(entry.Message, "Preview succeeded") {
8181
return io.EOF
8282
} else if strings.HasPrefix(entry.Message, "Preview failed") {
8383
return errors.New(entry.Message)
8484
}
85-
term.Debug(entry.Message)
85+
t.Debug(entry.Message)
8686
pulumiPreviewLogLines = append(pulumiPreviewLogLines, entry.Message)
8787
return nil
8888
})

0 commit comments

Comments
 (0)