fix: add X-Robots-Tag header to prevent robot indexing and following

src/http/routes/object/createObject.ts

@@ -32,6 +32,7 @@ interface createObjectRequestInterface extends RequestGenericInterface {
     'content-type': string
     'cache-control'?: string
     'x-upsert'?: string
+    'x-robots-tag'?: string
   }
 }
 

src/http/routes/object/getObject.ts

@@ -71,17 +71,21 @@ async function requestHandler(
 
   if (bucket.public) {
     // request is authenticated but we still use the superUser as we don't need to check RLS
-    obj = await request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id, version')
+    obj = await request.storage
+      .asSuperUser()
+      .from(bucketName)
+      .findObject(objectName, 'id, version, metadata')
   } else {
     // request is authenticated use RLS
-    obj = await request.storage.from(bucketName).findObject(objectName, 'id, version')
+    obj = await request.storage.from(bucketName).findObject(objectName, 'id, version, metadata')
   }
 
   return request.storage.renderer('asset').render(request, response, {
     bucket: storageS3Bucket,
     key: s3Key,
     version: obj.version,
     download,
+    xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
     signal: request.signals.disconnect.signal,
   })
 }
@@ -95,6 +99,7 @@ export default async function routes(fastify: FastifyInstance) {
       // @todo add success response schema here
       schema: {
         params: getObjectParamsSchema,
+        querystring: getObjectQuerySchema,
         headers: { $ref: 'authSchema#' },
         summary,
         response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } },

src/http/routes/object/getPublicObject.ts

@@ -38,6 +38,7 @@ export default async function routes(fastify: FastifyInstance) {
       exposeHeadRoute: false,
       schema: {
         params: getPublicObjectParamsSchema,
+        querystring: getObjectQuerySchema,
         summary,
         response: { '4xx': { $ref: 'errorSchema#', description: 'Error response' } },
         tags: ['object'],
@@ -55,7 +56,10 @@ export default async function routes(fastify: FastifyInstance) {
         request.storage.asSuperUser().findBucket(bucketName, 'id,public', {
           isPublic: true,
         }),
-        request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id,version'),
+        request.storage
+          .asSuperUser()
+          .from(bucketName)
+          .findObject(objectName, 'id,version,metadata'),
       ])
 
       // send the object from s3
@@ -70,6 +74,7 @@ export default async function routes(fastify: FastifyInstance) {
         key: s3Key,
         version: obj.version,
         download,
+        xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
         signal: request.signals.disconnect.signal,
       })
     }

src/http/routes/object/getSignedObject.ts

@@ -83,14 +83,15 @@ export default async function routes(fastify: FastifyInstance) {
       const obj = await request.storage
         .asSuperUser()
         .from(bucketName)
-        .findObject(objParts.join('/'), 'id,version')
+        .findObject(objParts.join('/'), 'id,version,metadata')
 
       return request.storage.renderer('asset').render(request, response, {
         bucket: storageS3Bucket,
         key: s3Key,
         version: obj.version,
         download,
         expires: new Date(exp * 1000).toUTCString(),
+        xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
         signal: request.signals.disconnect.signal,
       })
     }

src/http/routes/object/updateObject.ts

@@ -29,6 +29,7 @@ interface updateObjectRequestInterface extends RequestGenericInterface {
     'content-type': string
     'cache-control'?: string
     'x-upsert'?: string
+    'x-robots-tag'?: string
   }
 }
 

src/http/routes/render/renderAuthenticatedImage.ts

@@ -51,7 +51,9 @@ export default async function routes(fastify: FastifyInstance) {
       const { bucketName } = request.params
       const objectName = request.params['*']
 
-      const obj = await request.storage.from(bucketName).findObject(objectName, 'id,version')
+      const obj = await request.storage
+        .from(bucketName)
+        .findObject(objectName, 'id,version,metadata')
 
       const s3Key = request.storage.location.getKeyLocation({
         tenantId: request.tenantId,
@@ -73,6 +75,7 @@ export default async function routes(fastify: FastifyInstance) {
         key: s3Key,
         version: obj.version,
         download,
+        xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
         signal: request.signals.disconnect.signal,
       })
     }

src/http/routes/render/renderPublicImage.ts

@@ -55,7 +55,10 @@ export default async function routes(fastify: FastifyInstance) {
         request.storage.asSuperUser().findBucket(bucketName, 'id,public', {
           isPublic: true,
         }),
-        request.storage.asSuperUser().from(bucketName).findObject(objectName, 'id,version'),
+        request.storage
+          .asSuperUser()
+          .from(bucketName)
+          .findObject(objectName, 'id,version,metadata'),
       ])
 
       const s3Key = `${request.tenantId}/${bucketName}/${objectName}`
@@ -74,6 +77,7 @@ export default async function routes(fastify: FastifyInstance) {
         key: s3Key,
         version: obj.version,
         download,
+        xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
         signal: request.signals.disconnect.signal,
       })
     }

src/http/routes/render/renderSignedImage.ts

@@ -83,7 +83,7 @@ export default async function routes(fastify: FastifyInstance) {
       const obj = await request.storage
         .asSuperUser()
         .from(bucketName)
-        .findObject(objParts.join('/'), 'id,version')
+        .findObject(objParts.join('/'), 'id,version,metadata')
 
       const renderer = request.storage.renderer('image') as ImageRenderer
 
@@ -102,6 +102,7 @@ export default async function routes(fastify: FastifyInstance) {
           version: obj.version,
           download,
           expires: new Date(exp * 1000).toUTCString(),
+          xRobotsTag: obj.metadata?.['xRobotsTag'] as string | undefined,
           signal: request.signals.disconnect.signal,
         })
     }

src/storage/backend/adapter.ts

@@ -31,6 +31,7 @@ export type ObjectMetadata = {
   eTag: string
   contentRange?: string
   httpStatusCode?: number
+  xRobotsTag?: string
 }
 
 export type UploadPart = {

src/storage/renderer/renderer.ts

@@ -10,6 +10,7 @@ export interface RenderOptions {
   version: string | undefined
   download?: string
   expires?: string
+  xRobotsTag?: string
   object?: Obj
   signal?: AbortSignal
 }
@@ -80,6 +81,7 @@ export abstract class Renderer {
       .header('ETag', data.metadata.eTag)
       .header('Content-Length', data.metadata.contentLength)
       .header('Last-Modified', data.metadata.lastModified?.toUTCString())
+      .header('X-Robots-Tag', options.xRobotsTag || 'none')
 
     if (options.expires) {
       response.header('Expires', options.expires)

src/storage/uploader.ts

@@ -20,7 +20,8 @@ interface FileUpload {
   mimeType: string
   cacheControl: string
   isTruncated: () => boolean
-  userMetadata?: Record<string, any>
+  xRobotsTag?: string
+  userMetadata?: Record<string, unknown>
 }
 
 export interface UploadRequest {
@@ -112,6 +113,10 @@ export class Uploader {
         request.signal
       )
 
+      if (request.file.xRobotsTag) {
+        objectMetadata.xRobotsTag = request.file.xRobotsTag
+      }
+
       if (file.isTruncated()) {
         throw ERRORS.EntityTooLarge()
       }
@@ -301,9 +306,10 @@ export async function fileUploadFromRequest(
   }
 ): Promise<FileUpload & { maxFileSize: number }> {
   const contentType = request.headers['content-type']
+  const xRobotsTag = request.headers['x-robots-tag'] as string | undefined
 
   let body: Readable
-  let userMetadata: Record<string, any> | undefined
+  let userMetadata: Record<string, unknown> | undefined
   let mimeType: string
   let isTruncated: () => boolean
   let maxFileSize = 0
@@ -349,7 +355,7 @@ export async function fileUploadFromRequest(
 
         try {
           userMetadata = JSON.parse(customMd)
-        } catch (e) {
+        } catch {
           // no-op
         }
       }
@@ -388,14 +394,15 @@ export async function fileUploadFromRequest(
     isTruncated,
     userMetadata,
     maxFileSize,
+    xRobotsTag,
   }
 }
 
 export function parseUserMetadata(metadata: string) {
   try {
     const json = Buffer.from(metadata, 'base64').toString('utf8')
     return JSON.parse(json) as Record<string, string>
-  } catch (e) {
+  } catch {
     // no-op
     return undefined
   }

src/test/bucket.test.ts

@@ -361,6 +361,7 @@ describe('testing public bucket functionality', () => {
       url: `/object/public/public-bucket/favicon.ico`,
     })
     expect(publicResponse.statusCode).toBe(200)
+    expect(publicResponse.headers['x-robots-tag']).toBe('none')
     expect(publicResponse.headers['etag']).toBe('abc')
     expect(publicResponse.headers['last-modified']).toBe('Thu, 12 Aug 2021 16:00:00 GMT')
 

src/test/object.test.ts

@@ -61,6 +61,7 @@ describe('testing GET object', () => {
     })
     expect(response.statusCode).toBe(200)
     expect(response.headers['etag']).toBe('abc')
+    expect(response.headers['x-robots-tag']).toBe('none')
     expect(response.headers['last-modified']).toBe('Thu, 12 Aug 2021 16:00:00 GMT')
     expect(S3Backend.prototype.getObject).toBeCalled()
   })
@@ -2119,6 +2120,7 @@ describe('testing retrieving signed URL', () => {
       url: `/object/sign/${urlToSign}?token=${jwtToken}`,
     })
     expect(response.statusCode).toBe(200)
+    expect(response.headers['x-robots-tag']).toBe('none')
     expect(response.headers['etag']).toBe('abc')
     expect(response.headers['last-modified']).toBe('Thu, 12 Aug 2021 16:00:00 GMT')
   })
@@ -2521,3 +2523,128 @@ describe('testing list objects', () => {
     expect(responseJSON[1].name).toBe('sadcat-upload23.png')
   })
 })
+
+describe('x-robots-tag header', () => {
+  const X_ROBOTS_TEST_BUCKET = 'X_ROBOTS_TEST_BUCKET'
+  beforeAll(async () => {
+    appInstance = app()
+    await appInstance.inject({
+      method: 'POST',
+      url: `/bucket`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+      payload: {
+        name: X_ROBOTS_TEST_BUCKET,
+      },
+    })
+    await appInstance.close()
+  })
+
+  afterAll(async () => {
+    appInstance = app()
+    await appInstance.inject({
+      method: 'POST',
+      url: `/bucket/${X_ROBOTS_TEST_BUCKET}/empty`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    await appInstance.inject({
+      method: 'DELETE',
+      url: `/bucket/${X_ROBOTS_TEST_BUCKET}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    await appInstance.close()
+  })
+
+  test('defaults x-robots-tag header to none if not specified', async () => {
+    const objPath = `${X_ROBOTS_TEST_BUCKET}/test-file-1.txt`
+
+    await appInstance.inject({
+      method: 'POST',
+      url: `/object/${objPath}`,
+      payload: new File(['test'], 'file.txt'),
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+
+    const response = await appInstance.inject({
+      method: 'GET',
+      url: `/object/authenticated/${objPath}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    expect(response.headers['x-robots-tag']).toBe('none')
+  })
+
+  test('uses provided x-robots-tag header if set', async () => {
+    const objPath = `${X_ROBOTS_TEST_BUCKET}/test-file-2.txt`
+
+    await appInstance.inject({
+      method: 'POST',
+      url: `/object/${objPath}`,
+      payload: new File(['test'], 'file.txt'),
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+        'x-robots-tag': 'all',
+      },
+    })
+
+    const response = await appInstance.inject({
+      method: 'GET',
+      url: `/object/authenticated/${objPath}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    expect(response.headers['x-robots-tag']).toBe('all')
+  })
+
+  test('updates x-robots-tag header on upsert', async () => {
+    const objPath = `${X_ROBOTS_TEST_BUCKET}/test-file-3.txt`
+
+    await appInstance.inject({
+      method: 'POST',
+      url: `/object/${objPath}`,
+      payload: new File(['test'], 'file.txt'),
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+        'x-robots-tag': 'max-snippet: 10, notranslate',
+      },
+    })
+
+    const response = await appInstance.inject({
+      method: 'GET',
+      url: `/object/authenticated/${objPath}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    expect(response.headers['x-robots-tag']).toBe('max-snippet: 10, notranslate')
+
+    await appInstance.inject({
+      method: 'POST',
+      url: `/object/${objPath}`,
+      payload: new File(['test'], 'file.txt'),
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+        'x-upsert': 'true',
+        'x-robots-tag': 'nofollow',
+      },
+    })
+
+    const response2 = await appInstance.inject({
+      method: 'GET',
+      url: `/object/authenticated/${objPath}`,
+      headers: {
+        authorization: `Bearer ${await serviceKeyAsync}`,
+      },
+    })
+    expect(response2.headers['x-robots-tag']).toBe('nofollow')
+  })
+})