[DOP-2715] bucket polices and job filter (#688)

Author: rkhullar-mongodb

api/controllers/v1/jobs.ts

@@ -249,6 +249,14 @@ async function SubmitArchiveJob(jobId: string) {
   const job = await models.jobs.getJobById(jobId);
   const repo = await models.branches.getRepo(job.payload.repoName);
 
+  /* NOTE
+   * we don't archive landing for two reasons:
+   * - we can't unless we add efs to batch for extra storage; or https://github.com/aws/containers-roadmap/issues/1383
+   * - other properties like realm are nested under s3
+   */
+  const archiveExclusions = ['docs-landing'];
+  if (archiveExclusions.includes(repo.repoName)) return;
+
   const response = await new Batch(environment).submitArchiveJob(
     repo.bucket[environment],
     `docs-archive-${environment}-mongodb`,

infrastructure/ecs-main/buckets.yml

@@ -79,9 +79,12 @@ Resources:
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-archive-job-${self:provider.stage}-batch
-            Action: 's3:GetObject'
+            Action:
+              - 's3:GetObject'
+              - 's3:ListBucket'
             Resource:
               - Fn::Join: ['', [{ "Fn::GetAtt": ["DocsBucket", "Arn" ] }, '/*']]
+              - Fn::Join: ['', [{ "Fn::GetAtt": ["DocsBucket", "Arn" ] }]]
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-deploy-job-${self:provider.stage}-batch
@@ -106,9 +109,12 @@ Resources:
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-archive-job-${self:provider.stage}-batch
-            Action: 's3:GetObject'
+            Action:
+              - 's3:GetObject'
+              - 's3:ListBucket'
             Resource:
               - Fn::Join: ['', [{ "Fn::GetAtt": ["DocAtlasBucket", "Arn" ] }, '/*']]
+              - Fn::Join: ['', [{ "Fn::GetAtt": ["DocAtlasBucket", "Arn" ] }]]
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-deploy-job-${self:provider.stage}-batch
@@ -133,9 +139,12 @@ Resources:
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-archive-job-${self:provider.stage}-batch
-            Action: 's3:GetObject'
+            Action:
+              - 's3:GetObject'
+              - 's3:ListBucket'
             Resource:
               - Fn::Join: ['', [{ "Fn::GetAtt": ["CloudManagerBucket", "Arn" ] }, '/*']]
+              - Fn::Join: ['', [{ "Fn::GetAtt": ["CloudManagerBucket", "Arn" ] }]]
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-deploy-job-${self:provider.stage}-batch
@@ -160,9 +169,12 @@ Resources:
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-archive-job-${self:provider.stage}-batch
-            Action: 's3:GetObject'
+            Action:
+              - 's3:GetObject'
+              - 's3:ListBucket'
             Resource:
               - Fn::Join: ['', [{ "Fn::GetAtt": ["OpsManagerBucket", "Arn" ] }, '/*']]
+              - Fn::Join: ['', [{ "Fn::GetAtt": ["OpsManagerBucket", "Arn" ] }]]
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-deploy-job-${self:provider.stage}-batch
@@ -187,9 +199,12 @@ Resources:
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-archive-job-${self:provider.stage}-batch
-            Action: 's3:GetObject'
+            Action:
+              - 's3:GetObject'
+              - 's3:ListBucket'
             Resource:
               - Fn::Join: ['', [{ "Fn::GetAtt": ["JavaBucket", "Arn" ] }, '/*']]
+              - Fn::Join: ['', [{ "Fn::GetAtt": ["JavaBucket", "Arn" ] }]]
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-deploy-job-${self:provider.stage}-batch
@@ -214,9 +229,12 @@ Resources:
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-archive-job-${self:provider.stage}-batch
-            Action: 's3:GetObject'
+            Action:
+              - 's3:GetObject'
+              - 's3:ListBucket'
             Resource:
               - Fn::Join: ['', [{ "Fn::GetAtt": ["GoBucket", "Arn" ] }, '/*']]
+              - Fn::Join: ['', [{ "Fn::GetAtt": ["GoBucket", "Arn" ] }]]
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-deploy-job-${self:provider.stage}-batch
@@ -241,9 +259,12 @@ Resources:
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-archive-job-${self:provider.stage}-batch
-            Action: 's3:GetObject'
+            Action:
+              - 's3:GetObject'
+              - 's3:ListBucket'
             Resource:
               - Fn::Join: ['', [{ "Fn::GetAtt": ["NodeBucket", "Arn" ] }, '/*']]
+              - Fn::Join: ['', [{ "Fn::GetAtt": ["NodeBucket", "Arn" ] }]]
           - Effect: Allow
             Principal:
               AWS: arn:aws:iam::${aws:accountId}:role/docs-deploy-job-${self:provider.stage}-batch