From 08c13fa67e32dfae384b281a90c7dc2947d7c5da Mon Sep 17 00:00:00 2001 From: Matt Brockman Date: Fri, 27 Mar 2026 17:58:21 -0700 Subject: [PATCH 1/4] log when we add sandboxes to catalog --- packages/shared/pkg/sandbox-catalog/catalog_memory.go | 4 ++++ packages/shared/pkg/sandbox-catalog/catalog_redis.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/packages/shared/pkg/sandbox-catalog/catalog_memory.go b/packages/shared/pkg/sandbox-catalog/catalog_memory.go index b922117bc4..a5d6740dbd 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_memory.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_memory.go @@ -6,6 +6,8 @@ import ( "time" "github.com/jellydator/ttlcache/v3" + + "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) type MemorySandboxCatalog struct { @@ -49,6 +51,7 @@ func (c *MemorySandboxCatalog) StoreSandbox(ctx context.Context, sandboxID strin defer c.mtx.Unlock() c.cache.Set(sandboxID, sandboxInfo, expiration) + logger.L().Info(ctx, "stored sandbox in memory catalog", logger.WithSandboxID(sandboxID)) return nil } @@ -76,6 +79,7 @@ func (c *MemorySandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID stri } c.cache.Delete(sandboxID) + logger.L().Info(ctx, "deleted sandbox from memory catalog", logger.WithSandboxID(sandboxID)) return nil } diff --git a/packages/shared/pkg/sandbox-catalog/catalog_redis.go b/packages/shared/pkg/sandbox-catalog/catalog_redis.go index a22d2a5eec..412a75e1c8 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_redis.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_redis.go @@ -105,6 +105,8 @@ func (c *RedisSandboxCatalog) StoreSandbox(ctx context.Context, sandboxID string c.cache.Set(sandboxID, sandboxInfo, catalogRedisLocalCacheTtl) } + logger.L().Info(ctx, "stored sandbox in redis catalog", logger.WithSandboxID(sandboxID)) + return nil } @@ -134,6 +136,7 @@ func (c *RedisSandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID strin c.redisClient.Del(ctx, c.getCatalogKey(sandboxID)) c.cache.Delete(sandboxID) + logger.L().Info(ctx, "deleted sandbox from redis catalog", logger.WithSandboxID(sandboxID)) return nil } From a3175699d90cbc7c150349869b979381f1500a7e Mon Sep 17 00:00:00 2001 From: Matt Brockman Date: Fri, 27 Mar 2026 18:04:44 -0700 Subject: [PATCH 2/4] debug log for the sandbox catalog and sync reconcilliation --- .../api/internal/sandbox/storage/memory/sync.go | 16 ++++++++++++++++ packages/api/internal/sandbox/store.go | 7 +++++++ 2 files changed, 23 insertions(+) diff --git a/packages/api/internal/sandbox/storage/memory/sync.go b/packages/api/internal/sandbox/storage/memory/sync.go index 019855535d..4436f45b8b 100644 --- a/packages/api/internal/sandbox/storage/memory/sync.go +++ b/packages/api/internal/sandbox/storage/memory/sync.go @@ -1,9 +1,11 @@ package memory import ( + "context" "time" "github.com/e2b-dev/infra/packages/api/internal/sandbox" + "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) // TODO: this should be removed once we have a better way to handle node sync @@ -37,6 +39,13 @@ func (s *Storage) Sync(sandboxes []sandbox.Sandbox, nodeID string) []sandbox.San _, found := sandboxMap[data.SandboxID] if !found { + logger.L().Info( + context.Background(), + "sync expiring sandbox missing from node report", + logger.WithSandboxID(data.SandboxID), + logger.WithTeamID(data.TeamID.String()), + logger.WithNodeID(nodeID), + ) item.SetExpired() } }) @@ -48,6 +57,13 @@ func (s *Storage) Sync(sandboxes []sandbox.Sandbox, nodeID string) []sandbox.San continue } + logger.L().Info( + context.Background(), + "sync discovered sandbox missing from cache", + logger.WithSandboxID(sandbox.SandboxID), + logger.WithTeamID(sandbox.TeamID.String()), + logger.WithNodeID(nodeID), + ) toBeAdded = append(toBeAdded, sandbox) } diff --git a/packages/api/internal/sandbox/store.go b/packages/api/internal/sandbox/store.go index 3302ca29fc..268ae9efa1 100644 --- a/packages/api/internal/sandbox/store.go +++ b/packages/api/internal/sandbox/store.go @@ -158,6 +158,13 @@ func (s *Store) WaitForStateChange(ctx context.Context, teamID uuid.UUID, sandbo func (s *Store) Sync(ctx context.Context, sandboxes []Sandbox, nodeID string) { sbxs := s.storage.Sync(sandboxes, nodeID) for _, sbx := range sbxs { + logger.L().Info( + ctx, + "re-adding sandbox during sync", + logger.WithSandboxID(sbx.SandboxID), + logger.WithTeamID(sbx.TeamID.String()), + logger.WithNodeID(nodeID), + ) err := s.Add(ctx, sbx, false) if err != nil { logger.L().Error(ctx, "Failed to re-add sandbox during sync", zap.Error(err), logger.WithSandboxID(sbx.SandboxID)) From 98802ce259df2e421ed5b8f9533950f4ada7241a Mon Sep 17 00:00:00 2001 From: Matt Brockman Date: Fri, 27 Mar 2026 18:15:11 -0700 Subject: [PATCH 3/4] switch from info to debug --- packages/api/internal/sandbox/storage/memory/sync.go | 4 ++-- packages/api/internal/sandbox/store.go | 2 +- packages/shared/pkg/sandbox-catalog/catalog_memory.go | 4 ++-- packages/shared/pkg/sandbox-catalog/catalog_redis.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/api/internal/sandbox/storage/memory/sync.go b/packages/api/internal/sandbox/storage/memory/sync.go index 4436f45b8b..98ae936bd3 100644 --- a/packages/api/internal/sandbox/storage/memory/sync.go +++ b/packages/api/internal/sandbox/storage/memory/sync.go @@ -39,7 +39,7 @@ func (s *Storage) Sync(sandboxes []sandbox.Sandbox, nodeID string) []sandbox.San _, found := sandboxMap[data.SandboxID] if !found { - logger.L().Info( + logger.L().Debug( context.Background(), "sync expiring sandbox missing from node report", logger.WithSandboxID(data.SandboxID), @@ -57,7 +57,7 @@ func (s *Storage) Sync(sandboxes []sandbox.Sandbox, nodeID string) []sandbox.San continue } - logger.L().Info( + logger.L().Debug( context.Background(), "sync discovered sandbox missing from cache", logger.WithSandboxID(sandbox.SandboxID), diff --git a/packages/api/internal/sandbox/store.go b/packages/api/internal/sandbox/store.go index 268ae9efa1..cd6ddc7a57 100644 --- a/packages/api/internal/sandbox/store.go +++ b/packages/api/internal/sandbox/store.go @@ -158,7 +158,7 @@ func (s *Store) WaitForStateChange(ctx context.Context, teamID uuid.UUID, sandbo func (s *Store) Sync(ctx context.Context, sandboxes []Sandbox, nodeID string) { sbxs := s.storage.Sync(sandboxes, nodeID) for _, sbx := range sbxs { - logger.L().Info( + logger.L().Debug( ctx, "re-adding sandbox during sync", logger.WithSandboxID(sbx.SandboxID), diff --git a/packages/shared/pkg/sandbox-catalog/catalog_memory.go b/packages/shared/pkg/sandbox-catalog/catalog_memory.go index a5d6740dbd..601145b082 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_memory.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_memory.go @@ -51,7 +51,7 @@ func (c *MemorySandboxCatalog) StoreSandbox(ctx context.Context, sandboxID strin defer c.mtx.Unlock() c.cache.Set(sandboxID, sandboxInfo, expiration) - logger.L().Info(ctx, "stored sandbox in memory catalog", logger.WithSandboxID(sandboxID)) + logger.L().Debug(ctx, "stored sandbox in memory catalog", logger.WithSandboxID(sandboxID)) return nil } @@ -79,7 +79,7 @@ func (c *MemorySandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID stri } c.cache.Delete(sandboxID) - logger.L().Info(ctx, "deleted sandbox from memory catalog", logger.WithSandboxID(sandboxID)) + logger.L().Debug(ctx, "deleted sandbox from memory catalog", logger.WithSandboxID(sandboxID)) return nil } diff --git a/packages/shared/pkg/sandbox-catalog/catalog_redis.go b/packages/shared/pkg/sandbox-catalog/catalog_redis.go index 412a75e1c8..63d8dd8468 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_redis.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_redis.go @@ -105,7 +105,7 @@ func (c *RedisSandboxCatalog) StoreSandbox(ctx context.Context, sandboxID string c.cache.Set(sandboxID, sandboxInfo, catalogRedisLocalCacheTtl) } - logger.L().Info(ctx, "stored sandbox in redis catalog", logger.WithSandboxID(sandboxID)) + logger.L().Debug(ctx, "stored sandbox in redis catalog", logger.WithSandboxID(sandboxID)) return nil } @@ -136,7 +136,7 @@ func (c *RedisSandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID strin c.redisClient.Del(ctx, c.getCatalogKey(sandboxID)) c.cache.Delete(sandboxID) - logger.L().Info(ctx, "deleted sandbox from redis catalog", logger.WithSandboxID(sandboxID)) + logger.L().Debug(ctx, "deleted sandbox from redis catalog", logger.WithSandboxID(sandboxID)) return nil } From b6f881537d8d062f189d96b6dd71b8d93d087fd5 Mon Sep 17 00:00:00 2001 From: Matt Brockman Date: Fri, 27 Mar 2026 18:27:41 -0700 Subject: [PATCH 4/4] deletion also is before the mutex lock --- packages/shared/pkg/sandbox-catalog/catalog_memory.go | 5 +++-- packages/shared/pkg/sandbox-catalog/catalog_redis.go | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/shared/pkg/sandbox-catalog/catalog_memory.go b/packages/shared/pkg/sandbox-catalog/catalog_memory.go index 601145b082..aec8ba6e0c 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_memory.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_memory.go @@ -47,11 +47,12 @@ func (c *MemorySandboxCatalog) StoreSandbox(ctx context.Context, sandboxID strin _, span := tracer.Start(ctx, "sandbox-catalog-store") defer span.End() + logger.L().Debug(ctx, "storing sandbox in memory catalog", logger.WithSandboxID(sandboxID)) + c.mtx.Lock() defer c.mtx.Unlock() c.cache.Set(sandboxID, sandboxInfo, expiration) - logger.L().Debug(ctx, "stored sandbox in memory catalog", logger.WithSandboxID(sandboxID)) return nil } @@ -78,8 +79,8 @@ func (c *MemorySandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID stri return nil } + logger.L().Debug(ctx, "deleting sandbox from memory catalog", logger.WithSandboxID(sandboxID)) c.cache.Delete(sandboxID) - logger.L().Debug(ctx, "deleted sandbox from memory catalog", logger.WithSandboxID(sandboxID)) return nil } diff --git a/packages/shared/pkg/sandbox-catalog/catalog_redis.go b/packages/shared/pkg/sandbox-catalog/catalog_redis.go index 63d8dd8468..f59d395976 100644 --- a/packages/shared/pkg/sandbox-catalog/catalog_redis.go +++ b/packages/shared/pkg/sandbox-catalog/catalog_redis.go @@ -86,6 +86,8 @@ func (c *RedisSandboxCatalog) StoreSandbox(ctx context.Context, sandboxID string spanCtx, span := tracer.Start(ctx, "sandbox-catalog-store") defer span.End() + logger.L().Debug(ctx, "storing sandbox in redis catalog", logger.WithSandboxID(sandboxID)) + ctx, ctxCancel := context.WithTimeout(spanCtx, catalogRedisTimeout) defer ctxCancel() @@ -105,8 +107,6 @@ func (c *RedisSandboxCatalog) StoreSandbox(ctx context.Context, sandboxID string c.cache.Set(sandboxID, sandboxInfo, catalogRedisLocalCacheTtl) } - logger.L().Debug(ctx, "stored sandbox in redis catalog", logger.WithSandboxID(sandboxID)) - return nil } @@ -134,9 +134,9 @@ func (c *RedisSandboxCatalog) DeleteSandbox(ctx context.Context, sandboxID strin return nil } + logger.L().Debug(ctx, "deleting sandbox from redis catalog", logger.WithSandboxID(sandboxID)) c.redisClient.Del(ctx, c.getCatalogKey(sandboxID)) c.cache.Delete(sandboxID) - logger.L().Debug(ctx, "deleted sandbox from redis catalog", logger.WithSandboxID(sandboxID)) return nil }