Skip to content

Commit 2caffcf

Browse files
dforsberdforsber
committed
fix missing sandbox IAM role registration
1 parent f7b3db5 commit 2caffcf

15 files changed

+321
-54
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ dataSource:
5454

5555
% bdcli aws iam -c datasource_config.yaml --region eu-west-1 --create-role-only
5656
✔ Authenticating: success
57-
✔ Creating IAM Role: arn:aws:iam::589434896614:role/boilingdata/bd-euw1-noenv-notmplname-21346bf26c314caf8e7e9832205ffdee
57+
✔ Creating IAM Role: arn:aws:iam::589434896614:role/boilingdata/bd-euw1-noenv-notmpl-21346bf26c314caf8e7e9832205ffdee
5858

5959
% echo "Now you can verify the generated IAM role"
6060
Now you can verify the generated IAM role

src/bdcli/commands/account/bdcli-account-tap-token.ts renamed to src/bdcli/commands/account/bdcli-account-tap-client-token.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { BDAccount } from "../../../integration/boilingdata/account.js";
77
import { combineOptsWithSettings } from "../../utils/config_util.js";
88
import { outputResults } from "../../utils/output_util.js";
99

10-
const logger = getLogger("bdcli-account-token");
10+
const logger = getLogger("bdcli-account-tap-client-token");
1111

1212
async function show(options: any, _command: cmd.Command): Promise<void> {
1313
try {
@@ -19,27 +19,27 @@ async function show(options: any, _command: cmd.Command): Promise<void> {
1919
updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`);
2020
spinnerSuccess();
2121

22-
updateSpinnerText(`Getting BoilingData TAP token`);
22+
updateSpinnerText(`Getting BoilingData Client TAP token`);
2323
if (!region) throw new Error("Pass --region parameter or set AWS_REGION env");
2424
const bdAccount = new BDAccount({ logger, authToken: token });
2525
const {
2626
bdTapToken,
2727
cached: tapCached,
2828
...rest
2929
} = await bdAccount.getTapToken(options.lifetime ?? "24h", options.sharingUser);
30-
updateSpinnerText(`Getting BoilingData TAP token: ${tapCached ? "cached" : "success"}`);
30+
updateSpinnerText(`Getting BoilingData Client TAP token: ${tapCached ? "cached" : "success"}`);
3131
spinnerSuccess();
3232
await outputResults({ bdTapToken, cached: tapCached, ...rest }, options.disableSpinner);
3333
} catch (err: any) {
3434
spinnerError(err?.message);
3535
}
3636
}
3737

38-
const program = new cmd.Command("bdcli account tap-token")
38+
const program = new cmd.Command("bdcli account tap-client-token")
3939
.addOption(
4040
new cmd.Option(
4141
"--lifetime <lifetime>",
42-
"Expiration lifetime for the token, in string format, like '1h' (see https://github.com/vercel/ms)",
42+
"Expiration lifetime for the token, in string format, like '24h' (see https://github.com/vercel/ms)",
4343
),
4444
)
4545
.addOption(

‎src/bdcli/commands/account/bdcli-account-tap-master-secret.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import * as cmd from "commander";
2+
import { getLogger } from "../../utils/logger_util.js";
3+
import { spinnerError, spinnerSuccess, updateSpinnerText } from "../../utils/spinner_util.js";
4+
import { addGlobalOptions } from "../../utils/options_util.js";
5+
import { authSpinnerWithConfigCheck, getIdToken, validateTokenLifetime } from "../../utils/auth_util.js";
6+
import { BDAccount } from "../../../integration/boilingdata/account.js";
7+
import { combineOptsWithSettings } from "../../utils/config_util.js";
8+
import { outputResults } from "../../utils/output_util.js";
9+
10+
const logger = getLogger("bdcli-account-tap-master-secret");
11+
12+
async function show(options: any, _command: cmd.Command): Promise<void> {
13+
try {
14+
options = await combineOptsWithSettings(options, logger);
15+
if (options.lifetime) await validateTokenLifetime(options.lifetime);
16+
17+
if (!authSpinnerWithConfigCheck()) return;
18+
const { idToken: token, cached: idCached, region } = await getIdToken(logger);
19+
updateSpinnerText(`Authenticating: ${idCached ? "cached" : "success"}`);
20+
spinnerSuccess();
21+
22+
updateSpinnerText(`Getting BoilingData Master TAP secret`);
23+
if (!region) throw new Error("Pass --region parameter or set AWS_REGION env");
24+
const bdAccount = new BDAccount({ logger, authToken: token });
25+
const { bdTapMasterSecret, cached: tapCached, ...rest } = await bdAccount.getTapMasterSecret();
26+
updateSpinnerText(`Getting BoilingData Master TAP secret: ${tapCached ? "cached" : "success"}`);
27+
spinnerSuccess();
28+
await outputResults({ bdTapMasterSecret, cached: tapCached, ...rest }, options.disableSpinner);
29+
} catch (err: any) {
30+
spinnerError(err?.message);
31+
}
32+
}
33+
34+
const program = new cmd.Command("bdcli account tap-master-secret")
35+
.addOption(
36+
new cmd.Option(
37+
"--sharing-user <emailOfTapSharingUser>",
38+
"A user has shared Tap for you so that you can write to it.",
39+
),
40+
)
41+
.action(async (options, command) => await show(options, command));
42+
43+
(async () => {
44+
await addGlobalOptions(program, logger);
45+
await program.parseAsync(process.argv);
46+
})();

src/bdcli/commands/aws/bdcli-aws-iam.ts renamed to src/bdcli/commands/aws/bdcli-aws-s3-iam.ts

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,13 @@ import { ELogLevel, getLogger } from "../../utils/logger_util.js";
55
import { spinnerError, spinnerSuccess, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js";
66
import { addGlobalOptions } from "../../utils/options_util.js";
77
import { getIdToken } from "../../utils/auth_util.js";
8-
import { BDIamRole } from "../../../integration/aws/iam_roles.js";
8+
import { BDIamRole, ERoleType } from "../../../integration/aws/iam_role.js";
99
import { BDAccount } from "../../../integration/boilingdata/account.js";
1010
import { BDDataSourceConfig } from "../../../integration/boilingdata/dataset.js";
1111
import { BDIntegration } from "../../../integration/bdIntegration.js";
1212
import { combineOptsWithSettings } from "../../utils/config_util.js";
1313

14-
const logger = getLogger("bdcli-aws");
14+
const logger = getLogger("bdcli-aws-iam");
1515
logger.setLogLevel(ELogLevel.WARN);
1616

1717
async function iamrole(options: any, _command: cmd.Command): Promise<void> {
@@ -29,30 +29,32 @@ async function iamrole(options: any, _command: cmd.Command): Promise<void> {
2929
updateSpinnerText(cached ? "Authenticating: cached" : "Authenticating: success");
3030
spinnerSuccess();
3131

32-
updateSpinnerText("Creating IAM Role");
32+
updateSpinnerText("Creating S3 IAM Role");
3333
if (!region) throw new Error("Pass --region parameter or set AWS_REGION env");
3434
const bdAccount = new BDAccount({ logger, authToken: token });
3535
const bdDataSources = new BDDataSourceConfig({ logger });
3636
await bdDataSources.readConfig(options.config);
37+
const stsClient = new sts.STSClient({ region });
3738
const bdRole = new BDIamRole({
3839
...options,
3940
logger,
41+
roleType: ERoleType.S3,
4042
iamClient: new iam.IAMClient({ region }),
41-
stsClient: new sts.STSClient({ region }),
43+
stsClient,
4244
username: await bdAccount.getUsername(),
4345
assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
4446
assumeCondExternalId: await bdAccount.getExtId(),
4547
});
46-
const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources });
47-
const policyDocument = await bdIntegration.getPolicyDocument();
48+
const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources, stsClient });
49+
const policyDocument = await bdIntegration.getS3PolicyDocument();
4850
const iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument));
49-
updateSpinnerText(`Creating IAM Role: ${iamRoleArn}`);
51+
updateSpinnerText(`Creating S3 IAM Role: ${iamRoleArn}`);
5052
spinnerSuccess();
5153

5254
if (!options.createRoleOnly) {
53-
updateSpinnerText(`Registering IAM Role: ${iamRoleArn}`);
55+
updateSpinnerText(`Registering S3 IAM Role: ${iamRoleArn}`);
5456
const datasourcesConfig = bdDataSources.getDatasourcesConfig();
55-
await bdAccount.setIamRoleWithPayload(iamRoleArn, { datasourcesConfig });
57+
await bdAccount.setS3IamRoleWithPayload(iamRoleArn, { datasourcesConfig });
5658
spinnerSuccess();
5759
}
5860
} catch (err: any) {

‎src/bdcli/commands/aws/bdcli-aws-taps-iam.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import * as iam from "@aws-sdk/client-iam";
2+
import * as sts from "@aws-sdk/client-sts";
3+
import * as cmd from "commander";
4+
import { ELogLevel, getLogger } from "../../utils/logger_util.js";
5+
import { spinnerError, spinnerSuccess, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js";
6+
import { addGlobalOptions } from "../../utils/options_util.js";
7+
import { getIdToken } from "../../utils/auth_util.js";
8+
import { BDIamRole, ERoleType } from "../../../integration/aws/iam_role.js";
9+
import { BDAccount } from "../../../integration/boilingdata/account.js";
10+
import { BDIntegration } from "../../../integration/bdIntegration.js";
11+
import { combineOptsWithSettings } from "../../utils/config_util.js";
12+
13+
const logger = getLogger("bdcli-aws-taps-iam");
14+
logger.setLogLevel(ELogLevel.WARN);
15+
16+
async function iamrole(options: any, _command: cmd.Command): Promise<void> {
17+
try {
18+
options = await combineOptsWithSettings(options, logger);
19+
20+
if (options.delete) {
21+
updateSpinnerText("Not implemented yet. Please delete the IAM Role from AWS Console");
22+
spinnerWarn("Not implemented yet. Please delete the IAM Role from AWS Console");
23+
return;
24+
}
25+
26+
updateSpinnerText("Authenticating");
27+
const { idToken: token, cached, region } = await getIdToken(logger);
28+
updateSpinnerText(cached ? "Authenticating: cached" : "Authenticating: success");
29+
spinnerSuccess();
30+
31+
updateSpinnerText("Creating TAPS IAM Role");
32+
if (!region) throw new Error("Pass --region parameter or set AWS_REGION env");
33+
const bdAccount = new BDAccount({ logger, authToken: token });
34+
const stsClient = new sts.STSClient({ region });
35+
const bdRole = new BDIamRole({
36+
...options,
37+
logger,
38+
roleType: ERoleType.TAP,
39+
iamClient: new iam.IAMClient({ region }),
40+
stsClient,
41+
username: await bdAccount.getUsername(),
42+
assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
43+
assumeCondExternalId: await bdAccount.getExtId(),
44+
});
45+
const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, stsClient });
46+
const policyDocument = await bdIntegration.getTapsPolicyDocument();
47+
const iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument));
48+
updateSpinnerText(`Creating TAPS IAM Role: ${iamRoleArn}`);
49+
spinnerSuccess();
50+
51+
if (!options.createRoleOnly) {
52+
updateSpinnerText(`Registering TAPS IAM Role: ${iamRoleArn}`);
53+
await bdAccount.setTapsIamRoleWithPayload(iamRoleArn);
54+
spinnerSuccess();
55+
}
56+
} catch (err: any) {
57+
spinnerError(err?.message);
58+
}
59+
}
60+
61+
const program = new cmd.Command("bdcli aws taps-iam")
62+
.addHelpText(
63+
"beforeAll",
64+
"If you have an AWS account, you can use this command to create BoilingData assumable AWS IAM Role into " +
65+
"your AWS account. It is fully owned and controlled by you. The IAM Policy allows deploying Data Taps " +
66+
"(Lambda Functions with URL) into your account and creating the needed IAM Role for the Data Tap itself" +
67+
" (service role). \n\nSee the README.md in https://github.com/boilingdata/boilingdata-bdcli " +
68+
"for more information.\n",
69+
)
70+
.addOption(new cmd.Option("-r, --region <region>", "AWS region"))
71+
.addOption(new cmd.Option("--delete", "Delete the IAM role"))
72+
.addOption(new cmd.Option("--create-role-only", "Create the IAM role only and do not update BoilingData"))
73+
.action(async (options, command) => await iamrole(options, command));
74+
75+
(async () => {
76+
await addGlobalOptions(program, logger);
77+
await program.parseAsync(process.argv);
78+
})();

‎src/bdcli/commands/bdcli-account.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ const program = new Command("bdcli account")
88
.command("mfa", "Enable MFA")
99
.command("migrate", "Migrate your account to another AWS region (not-yet-implemented)")
1010
.command("sts-token", "Exchange Cognito ID Token into shared or your own BoilingData Short-Term-Session (STS) token")
11-
.command("tap-token", "Exchange Cognito ID Token into shared or your own BoilingData Stream Tap auth token")
11+
.command("tap-client-token", "Exchange Cognito ID Token into BoilingData Data Tap client auth token")
12+
.command("tap-master-secret", "Get your BoilingData Data Tap master secret")
1213
.command("token-share", "Share data sets via access tokens to other Boiling users")
1314
.command("token-unshare", "Unshare access tokens")
1415
.command("token-list-shares", "List shared data sets (access tokens) from and to you")

‎src/bdcli/commands/bdcli-aws.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { Command } from "commander";
22

33
const program = new Command("bdcli aws")
44
.executableDir("aws")
5-
.command("iam", "Setup IAM Role into your AWS account for controlled BoilingData access");
5+
.command("s3-iam", "Setup IAM Role into your AWS account for accessing S3")
6+
.command("taps-iam", "Setup deployer and service IAM Roles into your AWS account for creating and running Data Taps");
67

78
program.parse(process.argv);

‎src/bdcli/commands/domain/bdcli-domain-setup.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { ELogLevel, getLogger } from "../../utils/logger_util.js";
55
import { spinnerError, spinnerSuccess, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js";
66
import { addGlobalOptions } from "../../utils/options_util.js";
77
import { authSpinnerWithConfigCheck, getIdToken } from "../../utils/auth_util.js";
8-
import { BDIamRole } from "../../../integration/aws/iam_roles.js";
8+
import { BDIamRole } from "../../../integration/aws/iam_role.js";
99
import { BDAccount } from "../../../integration/boilingdata/account.js";
1010
import { BDDataSourceConfig } from "../../../integration/boilingdata/dataset.js";
1111
import { BDIntegration } from "../../../integration/bdIntegration.js";
@@ -34,6 +34,7 @@ async function iamrole(options: any, _command: cmd.Command): Promise<void> {
3434
const bdAccount = new BDAccount({ logger, authToken: token });
3535
const bdDataSources = new BDDataSourceConfig({ logger });
3636
await bdDataSources.readConfig(options.config);
37+
const stsClient = new sts.STSClient({ region });
3738
const bdRole = new BDIamRole({
3839
...options,
3940
logger,
@@ -42,16 +43,16 @@ async function iamrole(options: any, _command: cmd.Command): Promise<void> {
4243
assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
4344
assumeCondExternalId: await bdAccount.getExtId(),
4445
});
45-
const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources });
46-
const policyDocument = await bdIntegration.getPolicyDocument();
46+
const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources, stsClient });
47+
const policyDocument = await bdIntegration.getS3PolicyDocument();
4748
const iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument));
4849
updateSpinnerText(`Creating IAM Role: ${iamRoleArn}`);
4950
spinnerSuccess();
5051

5152
if (!options.createRoleOnly) {
5253
updateSpinnerText(`Registering IAM Role: ${iamRoleArn}`);
5354
const datasourcesConfig = bdDataSources.getDatasourcesConfig();
54-
await bdAccount.setIamRoleWithPayload(iamRoleArn, { datasourcesConfig });
55+
await bdAccount.setS3IamRoleWithPayload(iamRoleArn, { datasourcesConfig });
5556
spinnerSuccess();
5657
}
5758
} catch (err: any) {

‎src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { addGlobalOptions } from "../../utils/options_util.js";
77
import { combineOptsWithSettings } from "../../utils/config_util.js";
88
import { authSpinnerWithConfigCheck, getIdToken } from "../../utils/auth_util.js";
99
import { BDSandbox } from "../../../integration/boilingdata/sandbox.js";
10-
import { BDIamRole } from "../../../integration/aws/iam_roles.js";
10+
import { BDIamRole } from "../../../integration/aws/iam_role.js";
1111
import { BDAccount } from "../../../integration/boilingdata/account.js";
1212
import { BDIntegration } from "../../../integration/bdIntegration.js";
1313
import { BDDataSourceConfig } from "../../../integration/boilingdata/dataset.js";
@@ -35,21 +35,30 @@ async function show(options: any, _command: cmd.Command): Promise<void> {
3535
const bdAccount = new BDAccount({ logger, authToken: token });
3636
const bdDataSources = new BDDataSourceConfig({ logger });
3737
bdDataSources.withConfig({ dataSource: bdSandbox.tmpl.resources.storage });
38+
const stsClient = new sts.STSClient({ region });
3839
const bdRole = new BDIamRole({
3940
...options,
4041
logger,
4142
iamClient: new iam.IAMClient({ region }),
42-
stsClient: new sts.STSClient({ region }),
43+
stsClient,
4344
templateName: bdSandbox.tmpl.id,
4445
username: await bdAccount.getUsername(),
4546
assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
4647
assumeCondExternalId: await bdAccount.getExtId(),
4748
});
48-
const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources });
49-
const policyDocument = await bdIntegration.getPolicyDocument(options.listBucketsPermission);
50-
let iamRoleArn;
51-
if (!options.dryRun) iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument));
52-
updateSpinnerText(`Creating IAM Role: ${iamRoleArn}` + (options.dryRun ? "(dry-run)" : ""));
49+
const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources, stsClient });
50+
const policyDocument = await bdIntegration.getS3PolicyDocument(options.listBucketsPermission);
51+
if (options.dryRun) {
52+
updateSpinnerText(`Creating IAM Role (dry-run)`);
53+
spinnerSuccess();
54+
return;
55+
}
56+
const iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument));
57+
spinnerSuccess();
58+
59+
updateSpinnerText(`Registering S3 IAM Role: ${iamRoleArn}`);
60+
const datasourcesConfig = bdDataSources.getDatasourcesConfig();
61+
await bdAccount.setS3IamRoleWithPayload(iamRoleArn, { datasourcesConfig });
5362
spinnerSuccess();
5463
} catch (err: any) {
5564
// try to decode the message

src/integration/aws/iam_roles.ts renamed to src/integration/aws/iam_role.ts

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,18 @@ enum ENameType {
1313
"POLICY" = "policy",
1414
}
1515

16+
export enum ERoleType {
17+
S3 = "s3",
18+
TAP = "tap",
19+
}
20+
1621
export interface IBDIamRole {
1722
logger: ILogger;
1823
iamClient: iam.IAMClient;
1924
stsClient: sts.STSClient;
2025
region: string;
2126
username: string;
27+
roleType: ERoleType;
2228
templateName?: string;
2329
assumeCondExternalId: string;
2430
assumeAwsAccount: string;
@@ -36,13 +42,15 @@ export class BDIamRole {
3642
private _iamManagedPolicyName?: string;
3743
private boilingDataTags: Tag[];
3844
private path: string;
45+
private type: ERoleType;
3946
private awsAccountId?: string;
4047
private policyArn?: string;
4148

4249
constructor(private params: IBDIamRole) {
4350
this.iamClient = this.params.iamClient;
4451
this.stsClient = this.params.stsClient;
4552
this.logger = this.params.logger;
53+
this.type = this.params.roleType;
4654
this.path = this.params.path ?? "/boilingdata/";
4755
if (!this.path.startsWith("/") || !this.path.endsWith("/")) throw new Error("path must start and end with /");
4856
this.boilingDataTags = [{ Key: "service", Value: "boilingdata" }];
@@ -51,9 +59,9 @@ export class BDIamRole {
5159
private getName(type: string): string {
5260
const prefix = this.params.roleNamePrefix ?? "bd";
5361
const regionShort = getAwsRegionShortName(this.params.region ?? process.env["AWS_REGION"] ?? "eu-west-1");
54-
const tmplName = this.params.templateName ?? "notmplname";
62+
const tmplName = this.params.templateName ?? "notmpl";
5563
const username = this.params.username.replaceAll("-", "");
56-
const name = [prefix, regionShort, tmplName, username].join("-");
64+
const name = [prefix, regionShort, tmplName, username, this.type].join("-");
5765
if (name.length > 64) {
5866
throw new Error(
5967
`${type} name (${name}) too long (${name.length}), reduce prefix/tmplname lengths (roomLeft ${

0 commit comments

Comments
 (0)