Skip to content

Commit c16dd94

Browse files
jordanstephensjordanstephens
authored
Build Log Partitioning (#769)
* add target for "make protos" * add --build flag to tail * refactor LogType as a bitfield * Partition Build Logs from Service Logs (#766) * partition aws and do build logs * support both BUILD and RUN logs * refactor to use bitfiled LogType * remove obsolete comment * aws byoc: use service name instead of container name for kaniko * make the linter happy * LogType implements pflag.Value
1 parent 7d97160 commit c16dd94

File tree

8 files changed

+120
-60
lines changed

8 files changed

+120
-60
lines changed

‎src/Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ GOFLAGS:=-ldflags "-X main.version=$(VERSION)"
99
$(BINARY_NAME): test
1010
go build -o $@ $(GOFLAGS) ./cmd/cli
1111

12+
.PHONY: protos
13+
protos: $(PROTOS)
1214
$(PROTOS) &: protos/io/defang/v1/fabric.proto buf.gen.yaml
1315
cd protos && buf lint
1416
buf generate protos

‎src/cmd/cli/command/commands.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/DefangLabs/defang/src/pkg/cli"
2020
cliClient "github.com/DefangLabs/defang/src/pkg/cli/client"
2121
"github.com/DefangLabs/defang/src/pkg/cli/compose"
22+
"github.com/DefangLabs/defang/src/pkg/logs"
2223
"github.com/DefangLabs/defang/src/pkg/scope"
2324
"github.com/DefangLabs/defang/src/pkg/term"
2425
"github.com/DefangLabs/defang/src/pkg/track"
@@ -827,6 +828,7 @@ var deleteCmd = &cobra.Command{
827828
Since: since,
828829
Raw: false,
829830
Verbose: verbose,
831+
LogType: logs.LogTypeAll,
830832
}
831833
return cli.Tail(cmd.Context(), loader, provider, tailParams)
832834
},
@@ -1005,6 +1007,7 @@ var cdPreviewCmd = &cobra.Command{
10051007
return cli.Tail(cmd.Context(), loader, provider, cli.TailOptions{
10061008
Etag: resp.Etag,
10071009
Verbose: verbose,
1010+
LogType: logs.LogTypeAll,
10081011
})
10091012
},
10101013
}

‎src/cmd/cli/command/compose.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010

1111
"github.com/DefangLabs/defang/src/pkg/cli"
1212
"github.com/DefangLabs/defang/src/pkg/cli/compose"
13+
"github.com/DefangLabs/defang/src/pkg/logs"
1314
"github.com/DefangLabs/defang/src/pkg/term"
1415
"github.com/DefangLabs/defang/src/pkg/track"
1516
"github.com/DefangLabs/defang/src/pkg/types"
@@ -138,6 +139,7 @@ func makeComposeUpCmd() *cobra.Command {
138139
Since: since,
139140
Raw: false,
140141
Verbose: verbose,
142+
LogType: logs.LogTypeAll,
141143
}
142144

143145
// blocking call to tail
@@ -288,6 +290,7 @@ func makeComposeDownCmd() *cobra.Command {
288290
Raw: false,
289291
EndEventDetectFunc: endLogDetectFunc,
290292
Verbose: verbose,
293+
LogType: logs.LogTypeAll,
291294
}
292295

293296
err = cli.Tail(cmd.Context(), loader, provider, tailParams)
@@ -378,6 +381,8 @@ func makeComposeLogsCmd() *cobra.Command {
378381
if !cmd.Flags().Changed("verbose") {
379382
verbose = true // default verbose for explicit tail command
380383
}
384+
var logType logs.LogType
385+
cmd.Flags().Var(&logType, "type", fmt.Sprintf(`show logs of type; one of %v`, logs.AllLogTypes))
381386

382387
if utc {
383388
os.Setenv("TZ", "") // used by Go's "time" package, see https://pkg.go.dev/time#Location
@@ -398,12 +403,20 @@ func makeComposeLogsCmd() *cobra.Command {
398403
if len(name) > 0 {
399404
services = strings.Split(name, ",")
400405
}
406+
407+
if logType == logs.LogTypeUnspecified {
408+
logType = logs.LogTypeRun
409+
}
410+
411+
term.Debug("logType", logType)
412+
401413
tailOptions := cli.TailOptions{
402414
Services: services,
403415
Etag: etag,
404416
Since: ts,
405417
Raw: raw,
406418
Verbose: verbose,
419+
LogType: logType,
407420
}
408421

409422
loader := configureLoader(cmd)
@@ -419,6 +432,10 @@ func makeComposeLogsCmd() *cobra.Command {
419432
logsCmd.Flags().BoolP("raw", "r", false, "show raw (unparsed) logs")
420433
logsCmd.Flags().StringP("since", "S", "", "show logs since duration/time")
421434
logsCmd.Flags().Bool("utc", false, "show logs in UTC timezone (ie. TZ=UTC)")
435+
var logType logs.LogType
436+
logsCmd.Flags().Var(&logType, "type", fmt.Sprintf(`show logs of type; one of %v`, logs.AllLogTypes))
437+
logsCmd.Flags().String("pattern", "", "show logs matching the text pattern")
438+
logsCmd.Flags().MarkHidden("pattern")
422439
return logsCmd
423440
}
424441

‎src/pkg/cli/client/byoc/aws/byoc.go‎

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs"
2525
"github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs/cfn"
2626
"github.com/DefangLabs/defang/src/pkg/http"
27+
"github.com/DefangLabs/defang/src/pkg/logs"
2728
"github.com/DefangLabs/defang/src/pkg/term"
2829
"github.com/DefangLabs/defang/src/pkg/track"
2930
"github.com/DefangLabs/defang/src/pkg/types"
@@ -681,7 +682,7 @@ func (b *ByocAws) Query(ctx context.Context, req *defangv1.DebugRequest) error {
681682

682683
// Gather logs from the CD task, kaniko, ECS events, and all services
683684
sb := strings.Builder{}
684-
for _, lgi := range b.getLogGroupInputs(req.Etag, req.Project, service) {
685+
for _, lgi := range b.getLogGroupInputs(req.Etag, req.Project, service, logs.LogTypeAll) {
685686
parseECSEventRecords := strings.HasSuffix(lgi.LogGroupARN, "/ecs")
686687
if err := ecs.Query(ctx, lgi, since, time.Now(), func(logEvents []ecs.LogEvent) {
687688
for _, event := range logEvents {
@@ -734,6 +735,7 @@ func (b *ByocAws) Follow(ctx context.Context, req *defangv1.TailRequest) (client
734735
var taskArn ecs.TaskArn
735736
var eventStream ecs.EventStream
736737
stopWhenCDTaskDone := false
738+
logType := logs.LogType(req.LogType)
737739
if etag != "" && !pkg.IsValidRandomID(etag) { // Assume invalid "etag" is a task ID
738740
eventStream, err = b.driver.TailTaskID(ctx, etag)
739741
taskArn, _ = b.driver.GetTaskArn(etag)
@@ -745,7 +747,7 @@ func (b *ByocAws) Follow(ctx context.Context, req *defangv1.TailRequest) (client
745747
if len(req.Services) == 1 {
746748
service = req.Services[0]
747749
}
748-
eventStream, err = ecs.TailLogGroups(ctx, req.Since.AsTime(), b.getLogGroupInputs(etag, req.Project, service)...)
750+
eventStream, err = ecs.TailLogGroups(ctx, req.Since.AsTime(), b.getLogGroupInputs(etag, req.Project, service, logType)...)
749751
taskArn = b.lastCdTaskArn
750752
}
751753
if err != nil {
@@ -771,25 +773,34 @@ func (b *ByocAws) makeLogGroupARN(name string) string {
771773
return b.driver.MakeARN("logs", "log-group:"+name)
772774
}
773775

774-
func (b *ByocAws) getLogGroupInputs(etag types.ETag, projectName, service string) []ecs.LogGroupInput {
776+
func (b *ByocAws) getLogGroupInputs(etag types.ETag, projectName string, service string, logType logs.LogType) []ecs.LogGroupInput {
777+
var groups []ecs.LogGroupInput
775778
var serviceLogsPrefix string
776779
if service != "" {
777780
serviceLogsPrefix = service + "/" + service + "_" + etag
778781
}
779-
// Tail CD, kaniko, and all services (this requires ProjectName to be set)
780-
kanikoTail := ecs.LogGroupInput{LogGroupARN: b.makeLogGroupARN(b.stackDir(projectName, "builds"))} // must match logic in ecs/common.ts; TODO: filter by etag/service
781-
term.Debug("Query kaniko logs", kanikoTail.LogGroupARN)
782-
servicesTail := ecs.LogGroupInput{LogGroupARN: b.makeLogGroupARN(b.stackDir(projectName, "logs")), LogStreamNamePrefix: serviceLogsPrefix} // must match logic in ecs/common.ts
783-
term.Debug("Query services logs", servicesTail.LogGroupARN, serviceLogsPrefix)
784-
ecsTail := ecs.LogGroupInput{LogGroupARN: b.makeLogGroupARN(b.stackDir(projectName, "ecs"))} // must match logic in ecs/common.ts; TODO: filter by etag/service/deploymentId
785-
term.Debug("Query ecs events logs", ecsTail.LogGroupARN)
786-
cdTail := ecs.LogGroupInput{LogGroupARN: b.driver.LogGroupARN} // TODO: filter by etag
787-
// If we know the CD task ARN, only tail the logstream for that CD task
788-
if b.lastCdTaskArn != nil && b.lastCdEtag == etag {
789-
cdTail.LogStreamNames = []string{ecs.GetCDLogStreamForTaskID(ecs.GetTaskID(b.lastCdTaskArn))}
790-
}
791-
term.Debug("Query CD logs", cdTail.LogGroupARN, cdTail.LogStreamNames)
792-
return []ecs.LogGroupInput{cdTail, kanikoTail, servicesTail, ecsTail} // more or less in chronological order
782+
// Tail CD and kaniko
783+
if logType.Has(logs.LogTypeBuild) {
784+
cdTail := ecs.LogGroupInput{LogGroupARN: b.driver.LogGroupARN} // TODO: filter by etag
785+
// If we know the CD task ARN, only tail the logstream for that CD task
786+
if b.lastCdTaskArn != nil && b.lastCdEtag == etag {
787+
cdTail.LogStreamNames = []string{ecs.GetCDLogStreamForTaskID(ecs.GetTaskID(b.lastCdTaskArn))}
788+
}
789+
groups = append(groups, cdTail)
790+
term.Debug("Query CD logs", cdTail.LogGroupARN, cdTail.LogStreamNames)
791+
kanikoTail := ecs.LogGroupInput{LogGroupARN: b.makeLogGroupARN(b.stackDir(projectName, "builds"))} // must match logic in ecs/common.ts; TODO: filter by etag/service
792+
term.Debug("Query kaniko logs", kanikoTail.LogGroupARN)
793+
groups = append(groups, kanikoTail)
794+
ecsTail := ecs.LogGroupInput{LogGroupARN: b.makeLogGroupARN(b.stackDir(projectName, "ecs"))} // must match logic in ecs/common.ts; TODO: filter by etag/service/deploymentId
795+
term.Debug("Query ecs events logs", ecsTail.LogGroupARN)
796+
groups = append(groups, ecsTail)
797+
}
798+
if logType.Has(logs.LogTypeRun) {
799+
servicesTail := ecs.LogGroupInput{LogGroupARN: b.makeLogGroupARN(b.stackDir(projectName, "logs")), LogStreamNamePrefix: serviceLogsPrefix} // must match logic in ecs/common.ts
800+
term.Debug("Query services logs", servicesTail.LogGroupARN, serviceLogsPrefix)
801+
groups = append(groups, servicesTail)
802+
}
803+
return groups
793804
}
794805

795806
// This function was copied from Fabric controller and slightly modified to work with BYOC

‎src/pkg/cli/client/byoc/aws/stream.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ func (bs *byocServerStream) parseEvents(events []ecs.LogEvent) *defangv1.TailRes
115115
var record logs.FirelensMessage
116116
if err := json.Unmarshal([]byte(*event.Message), &record); err == nil {
117117
response.Etag = record.Etag
118-
response.Host = record.Host // TODO: use "kaniko" for kaniko logs
118+
response.Host = record.Host
119119
response.Service = record.ContainerName // TODO: could be service_etag
120120
parseFirelensRecords = true
121121
}
@@ -146,6 +146,7 @@ func (bs *byocServerStream) parseEvents(events []ecs.LogEvent) *defangv1.TailRes
146146
if err := json.Unmarshal([]byte(entry.Message), &record); err == nil {
147147
entry.Message = record.Log
148148
if record.ContainerName == "kaniko" {
149+
entry.Service = record.Service
149150
entry.Stderr = logs.IsLogrusError(entry.Message)
150151
} else {
151152
entry.Stderr = record.Source == logs.SourceStderr

‎src/pkg/cli/client/byoc/do/byoc.go‎

Lines changed: 47 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"github.com/DefangLabs/defang/src/pkg/cli/client/byoc"
2323
awsbyoc "github.com/DefangLabs/defang/src/pkg/cli/client/byoc/aws"
2424
"github.com/DefangLabs/defang/src/pkg/cli/compose"
25+
"github.com/DefangLabs/defang/src/pkg/logs"
2526

2627
"github.com/DefangLabs/defang/src/pkg/clouds/aws"
2728
"github.com/DefangLabs/defang/src/pkg/clouds/do"
@@ -427,27 +428,33 @@ func (b *ByocDo) Follow(ctx context.Context, req *defangv1.TailRequest) (client.
427428
return nil, err
428429
}
429430

431+
logType := logs.LogType(req.LogType)
432+
430433
term.Debugf("Deployment phase: %s", deploymentInfo.GetPhase())
431434
switch deploymentInfo.GetPhase() {
432435
case godo.DeploymentPhase_PendingBuild, godo.DeploymentPhase_PendingDeploy, godo.DeploymentPhase_Deploying:
433436
// Do nothing; check again in 10 seconds
434437

435438
case godo.DeploymentPhase_Error, godo.DeploymentPhase_Canceled:
436-
logs, _, err := b.client.Apps.GetLogs(ctx, cdApp.ID, deploymentID, "", godo.AppLogTypeDeploy, true, 50)
437-
if err != nil {
438-
return nil, err
439+
if logType.Has(logs.LogTypeBuild) {
440+
logs, _, err := b.client.Apps.GetLogs(ctx, cdApp.ID, deploymentID, "", godo.AppLogTypeDeploy, true, 50)
441+
if err != nil {
442+
return nil, err
443+
}
444+
readHistoricalLogs(ctx, logs.HistoricURLs)
439445
}
440-
readHistoricalLogs(ctx, logs.HistoricURLs)
441446
return nil, errors.New("deployment failed")
442447

443448
case godo.DeploymentPhase_Active:
444-
logs, _, err := b.client.Apps.GetLogs(ctx, cdApp.ID, deploymentID, "", godo.AppLogTypeDeploy, true, 50)
445-
if err != nil {
446-
return nil, err
449+
if logType.Has(logs.LogTypeBuild) {
450+
logs, _, err := b.client.Apps.GetLogs(ctx, cdApp.ID, deploymentID, "", godo.AppLogTypeDeploy, true, 50)
451+
if err != nil {
452+
return nil, err
453+
}
454+
readHistoricalLogs(ctx, logs.HistoricURLs)
447455
}
448-
readHistoricalLogs(ctx, logs.HistoricURLs)
449456

450-
appLiveURL, err := b.processServiceLogs(ctx, req.Project)
457+
appLiveURL, err := b.processServiceLogs(ctx, req.Project, logType)
451458
if err != nil {
452459
return nil, err
453460
}
@@ -733,7 +740,7 @@ func (b *ByocDo) processServiceInfo(service *godo.AppServiceSpec, projectName st
733740
return serviceInfo
734741
}
735742

736-
func (b *ByocDo) processServiceLogs(ctx context.Context, projectName string) (string, error) {
743+
func (b *ByocDo) processServiceLogs(ctx context.Context, projectName string, logType logs.LogType) (string, error) {
737744
appLiveURL := ""
738745

739746
buildAppName := fmt.Sprintf("defang-%s-%s-build", projectName, b.PulumiStack)
@@ -746,49 +753,52 @@ func (b *ByocDo) processServiceLogs(ctx context.Context, projectName string) (st
746753
}
747754

748755
for _, app := range currentApps {
749-
if app.Spec.Name == buildAppName {
756+
if logType.Has(logs.LogTypeBuild) && app.Spec.Name == buildAppName {
750757
buildLogs, _, err := b.client.Apps.GetLogs(ctx, app.ID, "", "", godo.AppLogTypeDeploy, false, 50)
751758
if err != nil {
752759
return "", err
753760
}
754761
readHistoricalLogs(ctx, buildLogs.HistoricURLs)
755762
}
763+
756764
if app.Spec.Name == mainAppName {
757765
deployments, _, err := b.client.Apps.ListDeployments(ctx, app.ID, &godo.ListOptions{})
758766
if err != nil {
759767
return "", err
760768
}
761769

762-
mainDeployLogs, resp, err := b.client.Apps.GetLogs(ctx, app.ID, "", "", godo.AppLogTypeDeploy, true, 50)
763-
if resp.StatusCode != 200 {
764-
// godo has no concept of returning the "last deployment", only "Active", "Pending", etc
765-
// Create our own last deployment and return deployment logs if the deployment failed in the last 2 minutes
766-
if deployments[0].Phase == godo.DeploymentPhase_Error && deployments[0].UpdatedAt.After(time.Now().Add(-2*time.Minute)) {
767-
failDeployLogs, _, err := b.client.Apps.GetLogs(ctx, app.ID, deployments[0].ID, "", godo.AppLogTypeDeploy, true, 50)
768-
if err != nil {
769-
return "", err
770+
if logType.Has(logs.LogTypeBuild) {
771+
mainDeployLogs, resp, err := b.client.Apps.GetLogs(ctx, app.ID, "", "", godo.AppLogTypeDeploy, true, 50)
772+
if resp.StatusCode != 200 {
773+
// godo has no concept of returning the "last deployment", only "Active", "Pending", etc
774+
// Create our own last deployment and return deployment logs if the deployment failed in the last 2 minutes
775+
if deployments[0].Phase == godo.DeploymentPhase_Error && deployments[0].UpdatedAt.After(time.Now().Add(-2*time.Minute)) {
776+
failDeployLogs, _, err := b.client.Apps.GetLogs(ctx, app.ID, deployments[0].ID, "", godo.AppLogTypeDeploy, true, 50)
777+
if err != nil {
778+
return "", err
779+
}
780+
readHistoricalLogs(ctx, failDeployLogs.HistoricURLs)
770781
}
771-
readHistoricalLogs(ctx, failDeployLogs.HistoricURLs)
782+
// Assume no deploy happened, return without an error
783+
return "", nil
772784
}
773-
// Assume no deploy happened, return without an error
774-
return "", nil
775-
}
776-
if err != nil {
777-
return "", err
778-
}
779-
780-
readHistoricalLogs(ctx, mainDeployLogs.HistoricURLs)
781-
782-
mainRunLogs, resp, err := b.client.Apps.GetLogs(ctx, app.ID, "", "", godo.AppLogTypeRun, true, 50)
783-
if resp.StatusCode != 200 {
784-
// Assume no deploy happened, return without an error
785-
return "", nil
785+
if err != nil {
786+
return "", err
787+
}
788+
readHistoricalLogs(ctx, mainDeployLogs.HistoricURLs)
786789
}
787-
if err != nil {
788-
return "", err
790+
if logType.Has(logs.LogTypeRun) {
791+
mainRunLogs, resp, err := b.client.Apps.GetLogs(ctx, app.ID, "", "", godo.AppLogTypeRun, true, 50)
792+
if resp.StatusCode != 200 {
793+
// Assume no deploy happened, return without an error
794+
return "", nil
795+
}
796+
if err != nil {
797+
return "", err
798+
}
799+
readHistoricalLogs(ctx, mainRunLogs.HistoricURLs)
800+
appLiveURL = mainRunLogs.LiveURL
789801
}
790-
readHistoricalLogs(ctx, mainRunLogs.HistoricURLs)
791-
appLiveURL = mainRunLogs.LiveURL
792802
}
793803
}
794804
return appLiveURL, nil

‎src/pkg/cli/composeUp.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ func ComposeUp(ctx context.Context, loader client.Loader, c client.FabricClient,
7878
var resp *defangv1.DeployResponse
7979
if upload == compose.UploadModePreview {
8080
resp, err = p.Preview(ctx, deployRequest)
81+
if err != nil {
82+
return nil, project, err
83+
}
8184
} else {
8285
req := client.PrepareDomainDelegationRequest{Project: project.Name, DelegateDomain: delegateDomain.Zone}
8386
var delegation *client.PrepareDomainDelegationResponse
@@ -94,9 +97,9 @@ func ComposeUp(ctx context.Context, loader client.Loader, c client.FabricClient,
9497
deployRequest.DelegationSetId = delegation.DelegationSetId
9598
}
9699
resp, err = p.Deploy(ctx, deployRequest)
97-
}
98-
if err != nil {
99-
return nil, project, err
100+
if err != nil {
101+
return nil, project, err
102+
}
100103
}
101104

102105
if term.DoDebug() {

0 commit comments

Comments
 (0)