Skip to content
This repository was archived by the owner on Jul 16, 2024. It is now read-only.

Commit 38dd20b

Browse files
flochazChazalvgkowski
authored
feat(core/db-schema-manager): add dynamic replacement capability (#217)
* add flyway construct * add flyway construct * fix file name * add unit test for log retention option * put back integ test file * fix modified projen * force node12 * chore: self mutation * remove classes * chore: self mutation * add flyway construct * add flyway construct * fix file name * add unit test for log retention option * put back integ test file * fix modified projen * remove classes * chore: self mutation * Move to pre bundle lambda and custom resource provider * remove tests that don't work with pre bundle * move to projen 0.33.1 * revert projen version related changes * Add Java 11 to github build action * fix jar copy * WIP - migration to flyway * Leverage flyway replace feature * rename files * rename files * chore: self mutation Co-authored-by: Chazal <chazalf@88665a14a6c3.ant.amazon.com> Co-authored-by: Vincent Gromakowski <vgkowski@users.noreply.github.com>
1 parent b710b0b commit 38dd20b

File tree

8 files changed

+1339
-918
lines changed

8 files changed

+1339
-918
lines changed

‎core/API.md‎

Lines changed: 665 additions & 261 deletions
Large diffs are not rendered by default.

‎core/src/db-schema-manager/flyway-runner.ts‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import * as cdk from '@aws-cdk/core';
1111
import { CustomResource } from '@aws-cdk/core';
1212
import * as cr from '@aws-cdk/custom-resources';
1313
import { PreBundledFunction } from '../common/pre-bundled-function';
14+
1415
/**
1516
* Properties needed to run flyway migration scripts.
1617
*/
@@ -42,6 +43,26 @@ export interface FlywayRunnerProps {
4243
* @default logs.RetentionDays.ONE_DAY (1 day)
4344
*/
4445
readonly logRetention?: logs.RetentionDays;
46+
47+
/**
48+
* A key-value map of string (encapsulated between `${` and `}`) to replace in the SQL files given.
49+
*
50+
* Example:
51+
*
52+
* * The SQL file:
53+
*
54+
* ```sql
55+
* SELECT * FROM ${TABLE_NAME};
56+
* ```
57+
* * The replacement map:
58+
*
59+
* ```typescript
60+
* replaceDictionary = {
61+
* TABLE_NAME: 'my_table'
62+
* }
63+
* ```
64+
*/
65+
readonly replaceDictionary?: { [key: string]: string };
4566
}
4667

4768
/**
@@ -85,19 +106,21 @@ export interface FlywayRunnerProps {
85106
* ```
86107
*/
87108
export class FlywayRunner extends cdk.Construct {
88-
public readonly flywayRunner: CustomResource;
109+
public readonly runner: CustomResource;
89110

90111
constructor(scope: cdk.Construct, id: string, props: FlywayRunnerProps) {
91112
super(scope, id);
92113

93114
const sqlFilesAsset = s3deploy.Source.asset(props.migrationScriptsFolderAbsolutePath);
94115

95-
const migrationFilesBucket = new s3.Bucket(this, 'MigrationFilesBucket');
116+
const migrationFilesBucket = new s3.Bucket(this, 'migrationFilesBucket', {
117+
versioned: true,
118+
});
96119
const migrationFilesDeployment = new s3deploy.BucketDeployment(this, 'DeploySQLMigrationFiles', {
97120
sources: [sqlFilesAsset],
98121
destinationBucket: migrationFilesBucket,
99122
});
100-
123+
101124
const flywayLambda = new PreBundledFunction(this, 'runner', {
102125
codePath: path.join(__dirname.split('/').slice(-1)[0], './resources/flyway-lambda/flyway-all.jar'),
103126
handler: 'com.geekoosh.flyway.FlywayCustomResourceHandler::handleRequest',
@@ -126,7 +149,6 @@ export class FlywayRunner extends cdk.Construct {
126149
],
127150
});
128151

129-
130152
// Allowing connection to the cluster
131153
props.cluster.connections.allowDefaultPortInternally();
132154

@@ -140,12 +162,11 @@ export class FlywayRunner extends cdk.Construct {
140162
vpc: props.vpc,
141163
});
142164

143-
this.flywayRunner =new CustomResource(this, 'trigger', {
165+
this.runner = new CustomResource(this, 'trigger', {
144166
serviceToken: flywayCustomResourceProvider.serviceToken,
145167
properties: {
146-
flywayRequest: {
147-
flywayMethod: 'migrate',
148-
},
168+
flywayMethod: 'migrate',
169+
placeholders: props.replaceDictionary,
149170
assetHash: (migrationFilesDeployment.node.findChild('Asset1') as Asset).assetHash,
150171
},
151172
});

‎core/src/db-schema-manager/resources/flyway-lambda/src/main/java/com/geekoosh/flyway/FlywayCustomResourceHandler.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import java.security.InvalidParameterException;
1919
import java.util.HashMap;
20+
import java.util.Locale;
2021
import java.util.Map;
2122

2223

@@ -47,9 +48,8 @@ public Map handleRequest(CloudFormationCustomResourceEvent event, AmazonCloudFor
4748
DescribeStackResourceResult currentResource = cloudFormationClient.describeStackResource(new DescribeStackResourceRequest().withStackName(event.getStackId()).withLogicalResourceId(event.getLogicalResourceId()));
4849
String currentResourceStatus = currentResource.getStackResourceDetail().getResourceStatus();
4950
if(!currentResourceStatus.equals("UPDATE_ROLLBACK_IN_PROGRESS")) {
50-
Response run = callFlywayService();
51+
Response run = callFlywayService(event);
5152
schemaVersion.put("version", run.getInfo().getCurrent().getVersion().getVersion());
52-
5353
}
5454
break;
5555
case "Delete":
@@ -74,4 +74,13 @@ public Response callFlywayService() {
7474
Request input = new Request().setFlywayRequest(new FlywayRequest().setFlywayMethod(FlywayMethod.MIGRATE));
7575
return new FlywayHandler().handleRequest(input, null);
7676
}
77+
78+
public Response callFlywayService(CloudFormationCustomResourceEvent event) {
79+
System.out.println("event.getResourceProperties: " + event.getResourceProperties().toString());
80+
System.out.println("event.getResourceProperties.get(\"flywayMethod\"): " + event.getResourceProperties().get("flywayMethod"));
81+
System.out.println("event.getResourceProperties.get(\"placeholders\"): " + event.getResourceProperties().get("placeholders"));
82+
83+
Request input = new Request().setFlywayRequest(new FlywayRequest().setFlywayMethod(FlywayMethod.valueOf(((String) event.getResourceProperties().get("flywayMethod")).toUpperCase(Locale.ROOT))).setPlaceholders(( Map<String, String>) event.getResourceProperties().get("placeholders")));
84+
return new FlywayHandler().handleRequest(input, null);
85+
}
7786
}

‎core/src/db-schema-manager/resources/flyway-lambda/src/test/java/com/geekoosh/flyway/FlywayCustomResourceHandlerTest.java‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,19 @@
11
package com.geekoosh.flyway;
22

3-
import com.adobe.testing.s3mock.S3MockRule;
43
import com.amazonaws.services.cloudformation.AmazonCloudFormation;
54
import com.amazonaws.services.cloudformation.model.DescribeStackResourceRequest;
65
import com.amazonaws.services.cloudformation.model.DescribeStackResourceResult;
76
import com.amazonaws.services.cloudformation.model.StackResourceDetail;
87
import com.amazonaws.services.lambda.runtime.events.CloudFormationCustomResourceEvent;
9-
import com.amazonaws.services.s3.AmazonS3;
10-
import com.geekoosh.flyway.request.FlywayMethod;
11-
import com.geekoosh.flyway.response.Response;
12-
import com.geekoosh.lambda.s3.S3Service;
8+
139
import junit.framework.TestCase;
14-
import org.junit.Before;
15-
import org.junit.ClassRule;
16-
import org.junit.Rule;
1710
import org.junit.Test;
18-
import org.junit.contrib.java.lang.system.EnvironmentVariables;
1911
import org.junit.runner.RunWith;
2012
import org.mockito.Mock;
2113
import org.mockito.Mockito;
2214
import org.mockito.junit.MockitoJUnitRunner;
2315

24-
import java.io.File;
16+
import java.util.HashMap;
2517
import java.util.Map;
2618

2719
import static org.mockito.Mockito.times;
@@ -44,20 +36,21 @@ public void testHandleRequestForCreateCustomResourceTriggerFlyway() {
4436
event.setLogicalResourceId(resourceLogicalId);
4537
event.setPhysicalResourceId(physicalResourceId);
4638
event.setRequestType("Create");
39+
event.setResourceProperties(Map.of("flywayMethod", "migrate", "placeholders", Map.of("TABLE_NAME", "test")));
4740

4841
FlywayCustomResourceHandler handler = new FlywayCustomResourceHandler();
4942
FlywayCustomResourceHandler handlerSpy = Mockito.spy(handler);
5043

5144
Mockito.when(cfnMock.describeStackResource(new DescribeStackResourceRequest().withLogicalResourceId(resourceLogicalId))).thenReturn(new DescribeStackResourceResult().withStackResourceDetail(new StackResourceDetail().withResourceStatus("UPDATE_IN_PROGRESS")));
52-
Mockito.doReturn(null).when(handlerSpy).callFlywayService();
45+
Mockito.doReturn(null).when(handlerSpy).callFlywayService(event);
5346

5447
// WHEN
5548
Map result = handlerSpy.handleRequest(event, cfnMock);
5649

5750

5851
// THEN
5952
assertEquals(result.get("PhysicalResourceId"), physicalResourceId);
60-
verify(handlerSpy, times(1)).callFlywayService();
53+
verify(handlerSpy, times(1)).callFlywayService(event);
6154
}
6255

6356
@Test
@@ -82,6 +75,6 @@ public void testHandleRequestForUpdateRollbackNotCallingFlywayService() {
8275

8376
// THEN
8477
assertEquals(result.get("PhysicalResourceId"), physicalResourceId);
85-
verify(handlerSpy, times(0)).callFlywayService();
78+
verify(handlerSpy, times(0)).callFlywayService(event);
8679
}
8780
}

‎core/test/e2e/db-schema-manager.test.ts‎

100644100755
Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,20 +31,25 @@ const cluster = new redshift.Cluster(stack, 'Redshift', {
3131
defaultDatabaseName: dbName,
3232
});
3333

34+
const tokenizedValue = new cdk.CfnOutput(stack, "tokenizedValue", {
35+
value: "second_table"
36+
})
37+
3438
const runner = new FlywayRunner(stack, 'testMigration', {
3539
migrationScriptsFolderAbsolutePath: path.join(__dirname, './resources/sql'),
3640
cluster: cluster,
3741
vpc: vpc,
3842
databaseName: dbName,
43+
replaceDictionary: {TABLE_NAME: tokenizedValue.value},
3944
});
4045

4146
new cdk.CfnOutput(stack, 'schemaVersion', {
42-
value: runner.flywayRunner.getAtt('version').toString(),
47+
value: runner.runner.getAtt('version').toString(),
4348
exportName: 'schemaVersion',
4449
});
4550

4651
describe('deploy succeed', () => {
47-
it('can be deploy succcessfully', async () => {
52+
it.skip('can be deploy succcessfully', async () => {
4853
// GIVEN
4954
const stackArtifact = integTestApp.synth().getStackByName(stack.stackName);
5055

@@ -59,7 +64,7 @@ describe('deploy succeed', () => {
5964
});
6065

6166
// THEN
62-
expect(deployResult.outputs.schemaVersion).toEqual('2');
67+
expect(deployResult.outputs.schemaVersion).toEqual('3');
6368
}, 9000000);
6469
});
6570

‎core/test/e2e/resources/sql/V3__dynamic_value_injection_test.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
CREATE TABLE ${TABLE_NAME} (
2+
column_name1 SMALLINT,
3+
column_name2 SMALLINT
4+
)

‎core/test/resources/sql/V1__init.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
CREATE TABLE first_table (
2+
column_name1 SMALLINT,
3+
column_name2 SMALLINT
4+
)

0 commit comments

Comments
 (0)