diff --git a/platform-cloud/docs/compute-envs/aws-batch.md b/platform-cloud/docs/compute-envs/aws-batch.md index 4fc731a56..1bd88146c 100644 --- a/platform-cloud/docs/compute-envs/aws-batch.md +++ b/platform-cloud/docs/compute-envs/aws-batch.md @@ -270,6 +270,26 @@ A permissive and broad policy with all the required permissions is provided here "ec2:GetConsoleOutput" ], "Resource": "*" + }, + { + "Sid": "OptionalLineageIntegrationSQSAndS3", + "Effect": "Allow", + "Action": [ + "sqs:CreateQueue", + "sqs:GetQueueAttributes", + "sqs:SetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "s3:CreateBucket", + "s3:GetBucketNotificationConfiguration", + "s3:PutBucketNotificationConfiguration", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:sqs:*:*:seqera-lineage-*", + "arn:aws:s3:::seqera-lineage-*" + ] } ] } @@ -601,6 +621,39 @@ Platform can retrieve the EC2 instance console output to detect errors in the us } ``` +### Data lineage (optional) + +If you enable [data lineage](../data/data-lineage) in your workspace, add the following permissions to your Platform integration credentials to create the queue infrastructure and bucket notifications used by the lineage service: + +```json +{ + "Sid": "LineageIntegrationSQS", + "Effect": "Allow", + "Action": [ + "sqs:CreateQueue", + "sqs:GetQueueAttributes", + "sqs:SetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:DeleteMessage" + ], + "Resource": "arn:aws:sqs:::seqera-lineage-*" +}, +{ + "Sid": "LineageIntegrationS3", + "Effect": "Allow", + "Action": [ + "s3:CreateBucket", + "s3:GetBucketNotificationConfiguration", + "s3:PutBucketNotificationConfiguration", + "s3:GetBucketLocation" + ], + "Resource": "arn:aws:s3:::seqera-lineage-*" +} +``` + +If you manage your own EC2 instance role or head job role (rather than letting Seqera create them with Batch Forge), see [Manual AWS Batch configuration](../enterprise/advanced-topics/manual-aws-batch-setup#create-an-ec2-instance-role) for additional S3 permissions to add to those roles. + ## Create the IAM policy The policy above must be created in the AWS account where the AWS Batch resources need to be created. diff --git a/platform-cloud/docs/compute-envs/aws-cloud.md b/platform-cloud/docs/compute-envs/aws-cloud.md index ccad58165..7a25aa97a 100644 --- a/platform-cloud/docs/compute-envs/aws-cloud.md +++ b/platform-cloud/docs/compute-envs/aws-cloud.md @@ -163,6 +163,26 @@ A permissive and broad policy with all the required permissions is provided here "ec2:GetConsoleOutput" ], "Resource": "*" + }, + { + "Sid": "OptionalLineageIntegrationSQSAndS3", + "Effect": "Allow", + "Action": [ + "sqs:CreateQueue", + "sqs:GetQueueAttributes", + "sqs:SetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "s3:CreateBucket", + "s3:GetBucketNotificationConfiguration", + "s3:PutBucketNotificationConfiguration", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:sqs:*:*:seqera-lineage-*", + "arn:aws:s3:::seqera-lineage-*" + ] } ] } @@ -316,6 +336,39 @@ Platform can retrieve the EC2 instance console output to detect errors in the us } ``` +### Data lineage (optional) + +If you enable [data lineage](../data/data-lineage) in your workspace, add the following permissions to your Platform integration credentials to create the queue infrastructure and bucket notifications used by the lineage service: + +```json +{ + "Sid": "LineageIntegrationSQS", + "Effect": "Allow", + "Action": [ + "sqs:CreateQueue", + "sqs:GetQueueAttributes", + "sqs:SetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:DeleteMessage" + ], + "Resource": "arn:aws:sqs:::seqera-lineage-*" +}, +{ + "Sid": "LineageIntegrationS3", + "Effect": "Allow", + "Action": [ + "s3:CreateBucket", + "s3:GetBucketNotificationConfiguration", + "s3:PutBucketNotificationConfiguration", + "s3:GetBucketLocation" + ], + "Resource": "arn:aws:s3:::seqera-lineage-*" +} +``` + +If you manage your own EC2 instance role (rather than letting Seqera create it automatically), see [Manual AWS Batch configuration](../enterprise/advanced-topics/manual-aws-batch-setup#create-an-ec2-instance-role) for the additional S3 policy to attach to that role. + ## Create the IAM policy The policy above must be created in the AWS account where the AWS Batch resources need to be created. diff --git a/platform-cloud/docs/data/data-lineage.md b/platform-cloud/docs/data/data-lineage.md index ef43cffbc..755fe4193 100644 --- a/platform-cloud/docs/data/data-lineage.md +++ b/platform-cloud/docs/data/data-lineage.md @@ -54,72 +54,12 @@ Changing the lineage storage bucket path after lineage data is generated will re When launching a pipeline in a data-lineage enabled workspace, the **Enable lineage** toggle in the pipeline **Run setup** reflects the **Enable lineage by default** workspace setting. This can be turned off to _explicitly exclude_ data lineage creation for the pipeline run. -### Additional IAM permissions required - -If using existing AWS Batch or AWS Cloud compute environments with custom IAM roles, the following service role policies are required: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "ListObjectsInBucket", - "Effect": "Allow", - "Action": [ - "s3:ListBucket" - ], - "Resource": "arn:aws:s3:::seqera-lineage-" - }, - { - "Sid": "AllObjectActions", - "Effect": "Allow", - "Action": "s3:*Object", - "Resource": "arn:aws:s3:::seqera-lineage-/*" - }, - { - "Sid": "AllowObjectTagging", - "Effect": "Allow", - "Action": [ - "s3:PutObjectTagging", - "s3:GetObjectTagging" - ], - "Resource": "arn:aws:s3:::seqera-lineage-/*" - } - ] -} -``` +### IAM permissions required -Platform integration credentials require the following additional permissions: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "sqs:CreateQueue", - "sqs:GetQueueAttributes", - "sqs:SetQueueAttributes", - "sqs:GetQueueUrl", - "sqs:ReceiveMessage", - "sqs:DeleteMessage" - ], - "Resource": "arn:aws:sqs:*:*:seqera-lineage-*" - }, - { - "Effect": "Allow", - "Action": [ - "s3:CreateBucket", - "s3:GetBucketNotificationConfiguration", - "s3:PutBucketNotificationConfiguration", - "s3:GetBucketLocation" - ], - "Resource": "arn:aws:s3:::seqera-lineage-*" - } - ] -} -``` +Data lineage requires additional AWS IAM permissions. The permissions required depend on the role: + +- **Platform integration credentials** (IAM user): see [AWS Batch — Data lineage](../compute-envs/aws-batch#data-lineage-optional) or [AWS Cloud — Data lineage](../compute-envs/aws-cloud#data-lineage-optional) +- **EC2 instance role / head job role** (manually managed): see [Manual AWS Batch configuration](../enterprise/advanced-topics/manual-aws-batch-setup#create-an-ec2-instance-role) ### Advanced: Experimenting with data lineage diff --git a/platform-cloud/docs/enterprise/advanced-topics/manual-aws-batch-setup.mdx b/platform-cloud/docs/enterprise/advanced-topics/manual-aws-batch-setup.mdx index f59c31ef6..0e0203966 100644 --- a/platform-cloud/docs/enterprise/advanced-topics/manual-aws-batch-setup.mdx +++ b/platform-cloud/docs/enterprise/advanced-topics/manual-aws-batch-setup.mdx @@ -95,6 +95,39 @@ Create a role that controls which AWS resources the EC2 instances launched by AW - `seqera-batchjob` (the instance role policy created above) 1. Enter `seqera-instancerole` as the role name and add an optional description and tags if needed, then select **Create**. +If you enable [data lineage](../data/data-lineage) in your workspace, attach the following additional policy to this role to allow access to the lineage S3 bucket: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "LineageListBucket", + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::seqera-lineage-" + }, + { + "Sid": "LineageObjectAccess", + "Effect": "Allow", + "Action": "s3:*Object", + "Resource": "arn:aws:s3:::seqera-lineage-/*" + }, + { + "Sid": "LineageObjectTagging", + "Effect": "Allow", + "Action": [ + "s3:PutObjectTagging", + "s3:GetObjectTagging" + ], + "Resource": "arn:aws:s3:::seqera-lineage-/*" + } + ] +} +``` + ### Create a Nextflow head job role Create an IAM role for the Nextflow head job. This role is attached to the Nextflow head job container and grants it the permissions needed to orchestrate workflow tasks and retrieve task logs from CloudWatch. You specify this role in the **Head Job role** field when creating a manual compute environment in Seqera Platform.