Skip to content

Commit 5129ed0

Browse files
aonnikovaonnikov
authored
fix: log more stats in datalake (#9624)
Signed-off-by: Alexander Onnikov <[email protected]>
1 parent dbee2f3 commit 5129ed0

File tree

5 files changed

+59
-33
lines changed

5 files changed

+59
-33
lines changed

‎services/datalake/pod-datalake/src/datalake/datalake.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ export class DatalakeImpl implements Datalake {
115115
const events = Array.isArray(name) ? name.map((n) => blobEvents.deleted(n)) : [blobEvents.deleted(name)]
116116
await this.producer.send(workspace, events)
117117
} catch (err) {
118-
ctx.error('failed to send blob deleted event', { err })
118+
ctx.error('failed to send blob deleted event', { workspace, name, err })
119119
}
120120
}
121121

@@ -155,7 +155,7 @@ export class DatalakeImpl implements Datalake {
155155
: blobEvents.created(name, { contentType, lastModified, size, etag: hash })
156156
await this.producer.send(workspace, [event])
157157
} catch (err) {
158-
ctx.error('failed to send blob created event', { err })
158+
ctx.error('failed to send blob created event', { workspace, name, err })
159159
}
160160

161161
return { name, size, contentType, lastModified, etag: hash }
@@ -176,7 +176,7 @@ export class DatalakeImpl implements Datalake {
176176
: blobEvents.created(name, { contentType, lastModified, size, etag: hash })
177177
await this.producer.send(workspace, [event])
178178
} catch (err) {
179-
ctx.error('failed to send blob created event', { err })
179+
ctx.error('failed to send blob created event', { workspace, name, err })
180180
}
181181

182182
return { name, size, contentType, lastModified, etag: hash }
@@ -215,7 +215,7 @@ export class DatalakeImpl implements Datalake {
215215
: blobEvents.created(name, { contentType, lastModified, size, etag: hash })
216216
await this.producer.send(workspace, [event])
217217
} catch (err) {
218-
ctx.error('failed to send blob created event', { err })
218+
ctx.error('failed to send blob created event', { workspace, name, err })
219219
}
220220

221221
return { name, size, contentType, lastModified, etag: hash }

‎services/datalake/pod-datalake/src/datalake/db.ts

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -471,55 +471,67 @@ export class LoggedDB implements BlobDB {
471471
) {}
472472

473473
async getData (ctx: MeasureContext, dataId: BlobDataId): Promise<BlobDataRecord | null> {
474-
return await ctx.with('db.getData', {}, () => this.db.getData(this.ctx, dataId))
474+
const params = { location: dataId.location }
475+
return await ctx.with('db.getData', {}, () => this.db.getData(this.ctx, dataId), params)
475476
}
476477

477478
async getBlob (ctx: MeasureContext, blobId: BlobId): Promise<BlobWithDataRecord | null> {
478-
return await ctx.with('db.getBlob', {}, () => this.db.getBlob(this.ctx, blobId))
479+
const params = { workspace: blobId.workspace }
480+
return await ctx.with('db.getBlob', {}, () => this.db.getBlob(this.ctx, blobId), params)
479481
}
480482

481483
async listBlobs (ctx: MeasureContext, workspace: string, options: ListBlobOptions): Promise<ListBlobResult> {
482-
return await ctx.with('db.listBlobs', {}, () => this.db.listBlobs(this.ctx, workspace, options))
484+
const params = { workspace }
485+
return await ctx.with('db.listBlobs', {}, () => this.db.listBlobs(this.ctx, workspace, options), params)
483486
}
484487

485488
async createData (ctx: MeasureContext, data: BlobDataRecord): Promise<void> {
486-
await ctx.with('db.createData', {}, () => this.db.createData(this.ctx, data))
489+
const params = { type: data.type }
490+
await ctx.with('db.createData', {}, () => this.db.createData(this.ctx, data), params)
487491
}
488492

489493
async createBlob (ctx: MeasureContext, blob: Omit<BlobRecord, 'filename'>): Promise<void> {
490-
await ctx.with('db.createBlob', {}, () => this.db.createBlob(this.ctx, blob))
494+
const params = { workspace: blob.workspace, location: blob.location }
495+
await ctx.with('db.createBlob', {}, () => this.db.createBlob(this.ctx, blob), params)
491496
}
492497

493498
async createBlobData (ctx: MeasureContext, data: BlobWithDataRecord): Promise<void> {
494-
await ctx.with('db.createBlobData', {}, () => this.db.createBlobData(this.ctx, data))
499+
const params = { workspace: data.workspace, location: data.location, type: data.type }
500+
await ctx.with('db.createBlobData', {}, () => this.db.createBlobData(this.ctx, data), params)
495501
}
496502

497503
async deleteBlobList (ctx: MeasureContext, blobs: BlobIds): Promise<void> {
498-
await ctx.with('db.deleteBlobList', {}, () => this.db.deleteBlobList(this.ctx, blobs))
504+
const params = { workspace: blobs.workspace }
505+
await ctx.with('db.deleteBlobList', {}, () => this.db.deleteBlobList(this.ctx, blobs), params)
499506
}
500507

501508
async deleteBlob (ctx: MeasureContext, blob: BlobId): Promise<void> {
502-
await ctx.with('db.deleteBlob', {}, () => this.db.deleteBlob(this.ctx, blob))
509+
const params = { workspace: blob.workspace }
510+
await ctx.with('db.deleteBlob', {}, () => this.db.deleteBlob(this.ctx, blob), params)
503511
}
504512

505513
async getMeta (ctx: MeasureContext, blobId: BlobId): Promise<BlobMeta | null> {
506-
return await this.ctx.with('db.getMeta', {}, () => this.db.getMeta(ctx, blobId))
514+
const params = { workspace: blobId.workspace }
515+
return await this.ctx.with('db.getMeta', {}, () => this.db.getMeta(ctx, blobId), params)
507516
}
508517

509518
async setMeta (ctx: MeasureContext, blobId: BlobId, meta: BlobMeta): Promise<void> {
510-
await this.ctx.with('db.setMeta', {}, () => this.db.setMeta(ctx, blobId, meta))
519+
const params = { workspace: blobId.workspace }
520+
await this.ctx.with('db.setMeta', {}, () => this.db.setMeta(ctx, blobId, meta), params)
511521
}
512522

513523
async setParent (ctx: MeasureContext, blob: BlobId, parent: BlobId | null): Promise<void> {
514-
await ctx.with('db.setParent', {}, () => this.db.setParent(this.ctx, blob, parent))
524+
const params = { workspace: blob.workspace }
525+
await ctx.with('db.setParent', {}, () => this.db.setParent(this.ctx, blob, parent), params)
515526
}
516527

517528
async getStats (ctx: MeasureContext): Promise<StatsResult> {
518529
return await ctx.with('db.getStats', {}, () => this.db.getStats(this.ctx))
519530
}
520531

521532
async getWorkspaceStats (ctx: MeasureContext, workspace: string): Promise<WorkspaceStatsResult> {
522-
return await ctx.with('db.getWorkspaceStats', {}, () => this.db.getWorkspaceStats(this.ctx, workspace))
533+
const params = { workspace }
534+
return await ctx.with('db.getWorkspaceStats', {}, () => this.db.getWorkspaceStats(this.ctx, workspace), params)
523535
}
524536
}
525537

‎services/datalake/pod-datalake/src/handlers/blob.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ export async function handleBlobDelete (
161161
res.status(204).send()
162162
} catch (error: any) {
163163
Analytics.handleError(error)
164-
ctx.error('failed to delete blob', { error })
164+
ctx.error('failed to delete blob', { workspace, name, error })
165165
res.status(500).send()
166166
}
167167
}
@@ -182,7 +182,7 @@ export async function handleBlobDeleteList (
182182
res.status(204).send()
183183
} catch (error: any) {
184184
Analytics.handleError(error)
185-
ctx.error('failed to delete blobs', { error })
185+
ctx.error('failed to delete blobs', { workspace, names: body.names, error })
186186
res.status(500).send()
187187
}
188188
}
@@ -224,7 +224,7 @@ export async function handleBlobSetParent (
224224
res.status(204).send()
225225
} catch (error: any) {
226226
Analytics.handleError(error)
227-
ctx.error('failed to delete blob', { error })
227+
ctx.error('failed to delete blob', { workspace, name, error })
228228
res.status(500).send()
229229
}
230230
}

‎services/datalake/pod-datalake/src/handlers/image.ts

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,9 +132,14 @@ export async function handleImageGet (
132132
await writeTempFile(tmpFile, blob.body)
133133

134134
try {
135-
const { contentType } = await ctx.with('sharp', {}, () => {
136-
return runPipeline(tmpFile, outFile, { format, width, height, fit })
137-
})
135+
const { contentType } = await ctx.with(
136+
'sharp',
137+
{ format },
138+
() => {
139+
return runPipeline(tmpFile, outFile, { format, width, height, fit })
140+
},
141+
{ fit, width, height, size: blob.size }
142+
)
138143

139144
res.setHeader('Content-Type', contentType)
140145
res.setHeader('Cache-Control', cacheControl)

‎services/datalake/pod-datalake/src/s3/bucket.ts

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,11 @@ class S3BucketImpl implements S3Bucket {
6363

6464
async head (ctx: MeasureContext, key: string): Promise<S3Object | null> {
6565
try {
66-
const result = await ctx.with('s3.headObject', {}, () =>
67-
this.client.headObject({ Bucket: this.bucket, Key: key })
66+
const result = await ctx.with(
67+
's3.headObject',
68+
{},
69+
() => this.client.headObject({ Bucket: this.bucket, Key: key }),
70+
{ bucket: this.bucket }
6871
)
6972

7073
return {
@@ -77,7 +80,7 @@ class S3BucketImpl implements S3Bucket {
7780
}
7881
} catch (err: any) {
7982
if (err?.$metadata?.httpStatusCode !== 404) {
80-
ctx.warn('no object found', { error: err, key })
83+
ctx.warn('no object found', { error: err, bucket: this.bucket, key })
8184
}
8285
return null
8386
}
@@ -87,7 +90,7 @@ class S3BucketImpl implements S3Bucket {
8790
try {
8891
const command = { Bucket: this.bucket, Key: key, Range: options?.range }
8992

90-
const result = await ctx.with('s3.getObject', {}, () => this.client.getObject(command))
93+
const result = await ctx.with('s3.getObject', {}, () => this.client.getObject(command), { bucket: this.bucket })
9194

9295
if (result.Body === undefined) {
9396
return null
@@ -115,7 +118,7 @@ class S3BucketImpl implements S3Bucket {
115118
}
116119
} catch (err: any) {
117120
if (err?.$metadata?.httpStatusCode !== 404) {
118-
ctx.warn('no object found', { error: err, key })
121+
ctx.warn('no object found', { error: err, bucket: this.bucket, key })
119122
}
120123
return null
121124
}
@@ -140,7 +143,7 @@ class S3BucketImpl implements S3Bucket {
140143
}
141144

142145
if (Buffer.isBuffer(body)) {
143-
const result = await ctx.with('s3.putObject', {}, () => this.client.putObject(command))
146+
const result = await ctx.with('s3.putObject', {}, () => this.client.putObject(command), { bucket: this.bucket })
144147

145148
return {
146149
key,
@@ -158,7 +161,7 @@ class S3BucketImpl implements S3Bucket {
158161
leavePartsOnError: false
159162
})
160163

161-
const result = await ctx.with('s3.upload', {}, () => upload.done())
164+
const result = await ctx.with('s3.upload', {}, () => upload.done(), { bucket: this.bucket })
162165

163166
return {
164167
key,
@@ -190,7 +193,9 @@ class S3BucketImpl implements S3Bucket {
190193
}
191194
}
192195

193-
const result = await ctx.with('s3.createMultipartUpload', {}, () => this.client.createMultipartUpload(command))
196+
const result = await ctx.with('s3.createMultipartUpload', {}, () => this.client.createMultipartUpload(command), {
197+
bucket: this.bucket
198+
})
194199
if (result.UploadId === undefined) {
195200
throw new Error('failed to create multipart upload')
196201
}
@@ -215,7 +220,7 @@ class S3BucketImpl implements S3Bucket {
215220
UploadId: multipart.uploadId,
216221
PartNumber: options.partNumber
217222
}
218-
const result = await ctx.with('s3.uploadPart', {}, () => this.client.uploadPart(command))
223+
const result = await ctx.with('s3.uploadPart', {}, () => this.client.uploadPart(command), { bucket: this.bucket })
219224
return {
220225
etag: result.ETag ?? '',
221226
partNumber: options.partNumber
@@ -241,7 +246,9 @@ class S3BucketImpl implements S3Bucket {
241246
})
242247
}
243248
}
244-
await ctx.with('s3.completeMultipartUpload', {}, () => this.client.completeMultipartUpload(command))
249+
await ctx.with('s3.completeMultipartUpload', {}, () => this.client.completeMultipartUpload(command), {
250+
bucket: this.bucket
251+
})
245252
}
246253

247254
async abortMultipartUpload (ctx: MeasureContext, key: string, multipart: S3MultipartUpload): Promise<void> {
@@ -250,6 +257,8 @@ class S3BucketImpl implements S3Bucket {
250257
Key: key,
251258
UploadId: multipart.uploadId
252259
}
253-
await ctx.with('s3.abortMultipartUpload', {}, () => this.client.abortMultipartUpload(command))
260+
await ctx.with('s3.abortMultipartUpload', {}, () => this.client.abortMultipartUpload(command), {
261+
bucket: this.bucket
262+
})
254263
}
255264
}

0 commit comments

Comments
 (0)