Skip to content

Commit 4dc359e

Browse files
sitolesitole
committed
Implemented builds logs for both local and remote resource providers
1 parent 0696d04 commit 4dc359e

File tree

3 files changed

+154
-84
lines changed

3 files changed

+154
-84
lines changed

‎packages/api/internal/clusters/resources.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,16 @@ import (
44
"context"
55
"time"
66

7+
"go.uber.org/zap"
8+
"google.golang.org/protobuf/types/known/timestamppb"
9+
710
"github.com/e2b-dev/infra/packages/api/internal/api"
11+
templatemanagergrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager"
12+
edgeapi "github.com/e2b-dev/infra/packages/shared/pkg/http/edge"
13+
"github.com/e2b-dev/infra/packages/shared/pkg/logger"
814
"github.com/e2b-dev/infra/packages/shared/pkg/logs"
15+
"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
16+
"github.com/e2b-dev/infra/packages/shared/pkg/utils"
917
)
1018

1119
type ClusterResource interface {
@@ -35,3 +43,67 @@ func logQueryWindow(cursor *time.Time, direction api.LogsDirection) (time.Time,
3543

3644
return start, end
3745
}
46+
47+
func logDirectionToTemplateManagerDirection(direction api.LogsDirection) templatemanagergrpc.LogsDirection {
48+
switch direction {
49+
case api.LogsDirectionForward:
50+
return templatemanagergrpc.LogsDirection_Forward
51+
case api.LogsDirectionBackward:
52+
return templatemanagergrpc.LogsDirection_Backward
53+
default:
54+
return templatemanagergrpc.LogsDirection_Forward
55+
}
56+
}
57+
58+
func logCheckSourceType(source *api.LogsSource, sourceType api.LogsSource) bool {
59+
return source == nil || *source == sourceType
60+
}
61+
62+
func logToEdgeLevel(level *logs.LogLevel) *edgeapi.LogLevel {
63+
if level == nil {
64+
return nil
65+
}
66+
67+
value := edgeapi.LogLevel(logs.LevelToString(*level))
68+
69+
return &value
70+
}
71+
72+
func logsFromBuilderInstance(ctx context.Context, instance *Instance, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) {
73+
var lvlReq *templatemanagergrpc.LogLevel
74+
if level != nil {
75+
lvlReq = templatemanagergrpc.LogLevel(*level).Enum()
76+
}
77+
78+
res, err := instance.GetConnection().Template.TemplateBuildStatus(
79+
ctx, &templatemanagergrpc.TemplateStatusRequest{
80+
TemplateID: templateID,
81+
BuildID: buildID,
82+
Offset: &offset,
83+
Limit: utils.ToPtr(uint32(limit)),
84+
Level: lvlReq,
85+
Start: timestamppb.New(start),
86+
End: timestamppb.New(end),
87+
Direction: utils.ToPtr(logDirectionToTemplateManagerDirection(direction)),
88+
},
89+
)
90+
if err != nil {
91+
telemetry.ReportError(ctx, "error when returning logs for template build", err)
92+
logger.L().Error(ctx, "error when returning logs for template build", zap.Error(err), logger.WithBuildID(buildID))
93+
94+
return nil, err
95+
}
96+
97+
raw := res.GetLogEntries()
98+
entries := make([]logs.LogEntry, len(raw))
99+
for i, entry := range res.GetLogEntries() {
100+
entries[i] = logs.LogEntry{
101+
Timestamp: entry.GetTimestamp().AsTime(),
102+
Message: entry.GetMessage(),
103+
Level: logs.LogLevel(entry.GetLevel()),
104+
Fields: entry.GetFields(),
105+
}
106+
}
107+
108+
return entries, nil
109+
}

‎packages/api/internal/clusters/resources_local.go‎

Lines changed: 36 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,14 @@ import (
77

88
"github.com/grafana/loki/pkg/logproto"
99
"go.uber.org/zap"
10-
"google.golang.org/protobuf/types/known/timestamppb"
1110

1211
"github.com/e2b-dev/infra/packages/api/internal/api"
1312
clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg"
1413
clickhouseutils "github.com/e2b-dev/infra/packages/clickhouse/pkg/utils"
15-
templatemanagergrpc "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager"
1614
"github.com/e2b-dev/infra/packages/shared/pkg/logger"
1715
"github.com/e2b-dev/infra/packages/shared/pkg/logs"
1816
"github.com/e2b-dev/infra/packages/shared/pkg/logs/loki"
1917
"github.com/e2b-dev/infra/packages/shared/pkg/smap"
20-
"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
21-
"github.com/e2b-dev/infra/packages/shared/pkg/utils"
2218
)
2319

2420
type LocalClusterResourceProvider struct {
@@ -144,82 +140,49 @@ func (l *LocalClusterResourceProvider) GetSandboxLogs(ctx context.Context, teamI
144140
return api.SandboxLogs{Logs: ll, LogEntries: le}, nil
145141
}
146142

147-
func (l *LocalClusterResourceProvider) GetBuildLogs(ctx context.Context, nodeID *string, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) ([]logs.LogEntry, error) {
143+
func (l *LocalClusterResourceProvider) GetBuildLogs(
144+
ctx context.Context,
145+
nodeID *string,
146+
templateID string,
147+
buildID string,
148+
offset int32,
149+
limit int32,
150+
level *logs.LogLevel,
151+
cursor *time.Time,
152+
direction api.LogsDirection,
153+
source *api.LogsSource,
154+
) ([]logs.LogEntry, error) {
148155
start, end := logQueryWindow(cursor, direction)
149156

150-
lokiDirection := defaultDirection
151-
if direction == api.LogsDirectionBackward {
152-
lokiDirection = logproto.BACKWARD
153-
}
154-
155-
// todo
156-
if source == nil {
157-
// try node and then default to Loki
158-
} else if *source == api.LogsSourcePersistent {
159-
// force to node
160-
} else if *source == api.LogsSourceTemporary {
161-
// force to loki
162-
}
163-
164-
if nodeID != nil {
157+
// Fetch logs directly from template builder instance
158+
if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) {
165159
instance, found := l.instances.Get(*nodeID)
166-
if found {
167-
var lvlReq *templatemanagergrpc.LogLevel
168-
if level != nil {
169-
lvlReq = templatemanagergrpc.LogLevel(*level).Enum()
170-
}
171-
172-
res, err := instance.grpc.Template.TemplateBuildStatus(
173-
ctx, &templatemanagergrpc.TemplateStatusRequest{
174-
TemplateID: templateID,
175-
BuildID: buildID,
176-
Offset: &offset,
177-
Limit: utils.ToPtr(uint32(limit)),
178-
Level: lvlReq,
179-
Start: timestamppb.New(start),
180-
End: timestamppb.New(end),
181-
Direction: utils.ToPtr(logDirectionToTemplateManagerDirection(direction)),
182-
},
183-
)
184-
if err != nil {
185-
telemetry.ReportError(ctx, "error when returning logs for template build", err)
186-
logger.L().Error(ctx, "error when returning logs for template build", zap.Error(err), logger.WithBuildID(buildID))
187-
188-
return nil, err
189-
}
190-
191-
raw := res.GetLogEntries()
192-
193-
// Add an extra newline to each log entry to ensure proper formatting in the CLI
194-
entries := make([]logs.LogEntry, len(raw))
195-
for i, entry := range raw {
196-
entries[i] = logs.LogEntry{
197-
Timestamp: entry.GetTimestamp().AsTime(),
198-
Message: entry.GetMessage(),
199-
Level: logs.LogLevel(entry.GetLevel()),
200-
Fields: entry.GetFields(),
201-
}
202-
}
203-
204-
return entries, nil
160+
if !found {
161+
return nil, fmt.Errorf("node instance not found for id '%s'", *nodeID)
205162
}
206-
}
207163

208-
logs, err := l.queryLogsProvider.QueryBuildLogs(ctx, templateID, buildID, start, end, int(limit), offset, level, lokiDirection)
209-
if err != nil {
210-
return nil, fmt.Errorf("error when fetching build logs: %w", err)
164+
entries, err := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction)
165+
if err != nil {
166+
return nil, fmt.Errorf("error getting build logs from node: %w", err)
167+
}
168+
169+
return entries, nil
211170
}
212171

213-
return logs, nil
214-
}
172+
// Fetch logs from Loki backend
173+
if logCheckSourceType(source, api.LogsSourcePersistent) {
174+
lokiDirection := defaultDirection
175+
if direction == api.LogsDirectionBackward {
176+
lokiDirection = logproto.BACKWARD
177+
}
215178

216-
func logDirectionToTemplateManagerDirection(direction api.LogsDirection) templatemanagergrpc.LogsDirection {
217-
switch direction {
218-
case api.LogsDirectionForward:
219-
return templatemanagergrpc.LogsDirection_Forward
220-
case api.LogsDirectionBackward:
221-
return templatemanagergrpc.LogsDirection_Backward
222-
default:
223-
return templatemanagergrpc.LogsDirection_Forward
179+
entries, err := l.queryLogsProvider.QueryBuildLogs(ctx, templateID, buildID, start, end, int(limit), offset, level, lokiDirection)
180+
if err != nil {
181+
return nil, fmt.Errorf("error when fetching build logs: %w", err)
182+
}
183+
184+
return entries, nil
224185
}
186+
187+
return nil, nil
225188
}

‎packages/api/internal/clusters/resources_remote.go‎

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,53 @@ func (r *ClusterResourceProviderImpl) GetSandboxLogs(ctx context.Context, teamID
123123
return api.SandboxLogs{Logs: l, LogEntries: le}, nil
124124
}
125125

126-
func (r *ClusterResourceProviderImpl) GetBuildLogs(ctx context.Context, _ *string, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, cursor *time.Time, direction api.LogsDirection, source *api.LogsSource) ([]logs.LogEntry, error) {
126+
func (r *ClusterResourceProviderImpl) GetBuildLogs(
127+
ctx context.Context,
128+
nodeID *string,
129+
templateID string,
130+
buildID string,
131+
offset int32,
132+
limit int32,
133+
level *logs.LogLevel,
134+
cursor *time.Time,
135+
direction api.LogsDirection,
136+
source *api.LogsSource,
137+
) ([]logs.LogEntry, error) {
138+
ctx, span := tracer.Start(ctx, "get build-logs")
139+
defer span.End()
140+
127141
start, end := logQueryWindow(cursor, direction)
142+
143+
// Fetch logs directly from template builder instance
144+
if nodeID != nil && logCheckSourceType(source, api.LogsSourceTemporary) {
145+
instance, found := r.instances.Get(*nodeID)
146+
if !found {
147+
return nil, fmt.Errorf("node instance not found for id '%s'", *nodeID)
148+
}
149+
150+
entries, err := logsFromBuilderInstance(ctx, instance, templateID, buildID, offset, limit, level, start, end, direction)
151+
if err != nil {
152+
return nil, fmt.Errorf("error getting build logs from node: %w", err)
153+
}
154+
155+
return entries, nil
156+
}
157+
158+
// Fetch logs from edge API
159+
if logCheckSourceType(source, api.LogsSourcePersistent) {
160+
// Fetch logs from edge API
161+
entries, err := r.getBuildLogsFromEdge(ctx, templateID, buildID, offset, limit, level, start, end, direction)
162+
if err != nil {
163+
return nil, fmt.Errorf("error getting build logs from edge API: %w", err)
164+
}
165+
166+
return entries, nil
167+
}
168+
169+
return nil, nil
170+
}
171+
172+
func (r *ClusterResourceProviderImpl) getBuildLogsFromEdge(ctx context.Context, templateID string, buildID string, offset int32, limit int32, level *logs.LogLevel, start time.Time, end time.Time, direction api.LogsDirection) ([]logs.LogEntry, error) {
128173
res, err := r.client.V1TemplateBuildLogsWithResponse(
129174
ctx, buildID, &edgeapi.V1TemplateBuildLogsParams{
130175
TemplateID: templateID,
@@ -161,13 +206,3 @@ func (r *ClusterResourceProviderImpl) GetBuildLogs(ctx context.Context, _ *strin
161206

162207
return l, nil
163208
}
164-
165-
func logToEdgeLevel(level *logs.LogLevel) *edgeapi.LogLevel {
166-
if level == nil {
167-
return nil
168-
}
169-
170-
value := edgeapi.LogLevel(logs.LevelToString(*level))
171-
172-
return &value
173-
}

0 commit comments

Comments
 (0)