fix: suppress excessive reaper logging and update reaper query (#832)

This change contains two fixes related to the gram functions reaper.

## fix: suppress logging to user deployment logs

This change modifies the function reaping process to reduce noise in
user deployment logs by suppressing routine informational messages.

## fix: scope app reaper query to project id

This change updates the database query to list reapable fly apps so that
it can be scoped to a specific project ID. This allows project-scoped
reaping. Previously, the project-scoped reaper was not passing the
project ID to the query and it was acting as a global reaper.

Author: disintegrator

.changeset/lucky-corners-tie.md

@@ -0,0 +1,5 @@
+---
+"server": patch
+---
+
+Modified the function reaping process to reduce noise in user deployment logs by suppressing routine informational messages.

.changeset/weak-beans-retire.md

@@ -0,0 +1,5 @@
+---
+"server": patch
+---
+
+Updated the database query to list reapable fly apps so that it can be scoped to a specific project ID. This allows project-scoped reaping. Previously, the project-scoped reaper was not passing the project ID to the query and it was acting as a global reaper.

server/internal/attr/conventions.go

@@ -153,6 +153,7 @@ const (
 	ResourceURIKey                 = attribute.Key("gram.resource.uri")
 	ToolsetIDKey                   = attribute.Key("gram.toolset.id")
 	ToolsetSlugKey                 = attribute.Key("gram.toolset.slug")
+	VisibilityKey                  = attribute.Key("gram.visibility")
 
 	PaginationTsStartKey     = attribute.Key("gram.pagination.ts_start")
 	PaginationTsEndKey       = attribute.Key("gram.pagination.ts_end")
@@ -164,6 +165,10 @@ const (
 	ClickhouseQueryDurationMsKey = attribute.Key("gram.clickhouse.query_duration_ms")
 )
 
+const (
+	VisibilityInternalValue = "internal"
+)
+
 func Error(v error) attribute.KeyValue { return ErrorMessageKey.String(v.Error()) }
 func SlogError(v error) slog.Attr      { return slog.String(string(ErrorMessageKey), v.Error()) }
 
@@ -627,6 +632,13 @@ func SlogToolCallDuration(v time.Duration) slog.Attr {
 	return slog.Float64(string(ToolCallDurationKey), v.Seconds())
 }
 
+func VisibilityInternal() attribute.KeyValue {
+	return VisibilityKey.String(VisibilityInternalValue)
+}
+func SlogVisibilityInternal() slog.Attr {
+	return slog.String(string(VisibilityKey), VisibilityInternalValue)
+}
+
 func PaginationTsStart(v time.Time) attribute.KeyValue {
 	return PaginationTsStartKey.String(v.Format(time.RFC3339))
 }

server/internal/background/activities/reap_functions.go

@@ -75,13 +75,14 @@ func (r *ReapFlyApps) Do(ctx context.Context, req ReapFlyAppsRequest) (*ReapFlyA
 		// Starting with a small batch size for now and we'll increase later on
 		// after some observation.
 		BatchSize: pgtype.Int8{Int64: 50, Valid: true},
+
+		ProjectID: req.ProjectID,
 	})
 	if err != nil {
 		return nil, oops.E(oops.CodeUnexpected, err, "failed to query apps to reap").Log(ctx, logger)
 	}
 
 	if len(appsToReap) == 0 {
-		logger.InfoContext(ctx, "no apps to reap")
 		return &ReapFlyAppsResult{
 			Reaped: 0,
 			Errors: 0,
@@ -103,8 +104,6 @@ func (r *ReapFlyApps) Do(ctx context.Context, req ReapFlyAppsRequest) (*ReapFlyA
 			attr.SlogDeploymentFunctionsID(app.FunctionID.String()),
 		)
 
-		appLogger.InfoContext(ctx, "reaping fly app")
-
 		if err := r.deployer.Reap(ctx, functions.ReapRequest{
 			ProjectID:    app.ProjectID,
 			DeploymentID: app.DeploymentID,
@@ -116,7 +115,7 @@ func (r *ReapFlyApps) Do(ctx context.Context, req ReapFlyAppsRequest) (*ReapFlyA
 		}
 
 		result.Reaped++
-		appLogger.InfoContext(ctx, "successfully reaped fly app")
+		appLogger.InfoContext(ctx, "successfully reaped fly app", attr.SlogVisibilityInternal())
 	}
 
 	r.metrics.RecordFlyAppReaperReapCount(ctx, int64(result.Reaped), int64(result.Errors))

server/internal/deployments/events/log.go

@@ -53,13 +53,16 @@ func (l *LogHandler) Enabled(ctx context.Context, level slog.Level) bool {
 }
 
 func (l *LogHandler) Handle(ctx context.Context, record slog.Record) error {
+	internal := false
 	event := l.attrEvent
 	projectID := l.attrProjectID
 	deploymentID := l.attrDeploymentID
 	openAPIAssetID := l.attrOpenAPIAssetID
 
 	record.Attrs(func(a slog.Attr) bool {
 		switch {
+		case a.Key == string(attr.VisibilityKey) && a.Value.Kind() == slog.KindString:
+			internal = a.Value.String() == attr.VisibilityInternalValue
 		case a.Key == string(attr.EventKey) && a.Value.Kind() == slog.KindString:
 			event = a.Value.String()
 		case a.Key == string(attr.ProjectIDKey) && a.Value.Kind() == slog.KindString:
@@ -79,6 +82,10 @@ func (l *LogHandler) Handle(ctx context.Context, record slog.Record) error {
 		return true
 	})
 
+	if internal {
+		return nil
+	}
+
 	if event == "" {
 		event = fmt.Sprintf("log:%s", strings.ToLower(record.Level.String()))
 	}

server/internal/functions/deploy_fly.go

@@ -57,6 +57,7 @@ type FlyRunnerOptions struct {
 
 type FlyRunner struct {
 	logger          *slog.Logger
+	tracer          trace.Tracer
 	db              *pgxpool.Pool
 	assetStorage    assets.BlobStore
 	client          *fly.Client
@@ -116,6 +117,7 @@ func NewFlyRunner(
 
 	return &FlyRunner{
 		logger:          logger.With(attr.SlogComponent("flyio-orchestrator")),
+		tracer:          tracerProvider.Tracer("github.com/speakeasy-api/gram/server/internal/functions"),
 		db:              db,
 		assetStorage:    assetStorage,
 		client:          c,
@@ -507,7 +509,11 @@ func (f *FlyRunner) Deploy(ctx context.Context, req RunnerDeployRequest) (res *R
 }
 
 func (f *FlyRunner) Reap(ctx context.Context, req ReapRequest) error {
+	ctx, span := f.tracer.Start(ctx, "FlyRunner.Reap")
+	defer span.End()
+
 	logger := f.logger.With(
+		attr.SlogVisibilityInternal(),
 		attr.SlogProjectID(req.ProjectID.String()),
 		attr.SlogDeploymentID(req.DeploymentID.String()),
 		attr.SlogDeploymentFunctionsID(req.FunctionID.String()),
@@ -530,7 +536,6 @@ func (f *FlyRunner) reap(ctx context.Context, logger *slog.Logger, appsRepo *rep
 	})
 	switch {
 	case errors.Is(err, sql.ErrNoRows):
-		logger.InfoContext(ctx, "no existing app found")
 		return nil
 	case err != nil:
 		return fmt.Errorf("get existing app name: %w", err)

server/internal/functions/queries.sql

@@ -102,6 +102,7 @@ WITH ranked_deployments AS (
   INNER JOIN deployments d ON d.id = fa.deployment_id
   WHERE
     fa.status = 'ready'
+    AND (@project_id::uuid IS NULL OR fa.project_id = @project_id)
     AND fa.reaped_at IS NULL
 )
 SELECT