Merge pull request #6 from artekorzel/feature/aws-parameter-store-integration

Add integration with AWS Parameter Store

Author: jcustovic

README.md

@@ -191,13 +191,17 @@ app:
     index_pattern: "'data-exchange-client-' + T(java.time.LocalDate).now().format(T(java.time.format.DateTimeFormatter).ofPattern('YYYY-MM'))"
 ```
 
-#### Consul configuration section
+#### Remote configuration
 
-Currently this library comes with support of consul config. By default consul is disabled but by writing your own bootstrap.yml file in your project,
-you can make use of it. It is mainly for storing connection credentials (i.e. hostname, port, username, password) in one common place. 
+This library comes with a support for reading remote configuration from Consul or AWS Parameter Store. By default, this feature is disabled, 
+but by writing your own bootstrap.yml file in your project, you can make use of it. It is mainly for storing connection credentials (i.e. hostname, port, 
+username, password) in one common place. 
 
-Here is one example configuration of your bootstrap.yml file. Don't forget to specify -Dspring.profiles.active in order for the correct profile to be loaded.
-```
+##### Consul connection configuration
+
+Here is one example configuration of your bootstrap.yml file, which enables Consul support. 
+Don't forget to specify `-Dspring.profiles.active` in order for the correct profile to be loaded.
+```yaml
 spring:
   cloud:
     consul:
@@ -221,13 +225,48 @@ spring:
 
 ```
 
-In order to let the pollers know that they will take the configuration from Consul, you need to do the following changes. Following is an example of a download poller
+##### AWS Parameter Store connection configuration
+
+This library uses Spring Cloud AWS to integrate with AWS Parameter Store.
+It is configured using `aws.paramstore.*` properties described in Spring Cloud AWS documentation.
+Its functionality is extended by allowing you to configure `AWSStaticCredentialsProvider` or `STSAssumeRoleSessionCredentialsProvider` 
+using `aws.credentials.*` properties described below.
+
+Here is one example configuration of your bootstrap.yml file, which enables AWS Parameter Store support. 
+```yaml
+spring:
+  cloud:
+    consul:
+      enabled: false
+
+aws:
+  credentials:
+    use-static-provider: true
+    access-key: ...
+    secret-key: ...
+    region: ...
+    # specify below properties if you need to assume STS role  
+    sts-role-arn: ...
+    role-session-name: data-exchange-client
+  paramstore:
+    enabled: true
+    prefix: /config
+    name: data-exchange-client
+
+```
+
+Using the above configuration, a parameter stored in AWS Parameter Store under the key `/config/data-exchange-client/connections/test/password`
+will be available in Spring as `connections.test.password`. 
+
+##### Pollers configuration
+
+In order to let the pollers know that they will take the configuration from remote store, you need to do the following changes. Following is an example of a download poller
 but same works for upload poller:
 ```
 ftps:
     -
       name: localhostFTP # arbitrary name, should be unique
-      remoteConfigName: <config_name_from_consul's_yaml>
+      remoteConfigName: <config_name_from_remote_configuration_store>
       download-pollers:
         -
           name: testFtpDownloadPoller
@@ -240,7 +279,7 @@ ftps:
           poll-interval-milliseconds: 30000 # 30s, defaults to 10s
 ```
 
-So with consul the following have been replaced.
+So with Consul the following have been replaced.
 remote-config-name --> user, password, host, port
 file-type          --> remote-input-folder or remote-output-folder
 

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.dataexchange</groupId>
     <artifactId>data-exchange-client</artifactId>
-    <version>1.4.0</version>
+    <version>1.5.0</version>
     <packaging>jar</packaging>
 
     <name>data-exchange-client</name>
@@ -63,6 +63,10 @@
             <groupId>org.springframework.cloud</groupId>
             <artifactId>spring-cloud-starter-consul-config</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-aws-parameter-store-config</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.cloud</groupId>

src/main/java/com/dataexchange/client/config/ConsulConfiguration.java renamed to src/main/java/com/dataexchange/client/config/RemoteConfiguration.java

@@ -11,15 +11,15 @@
 @Component
 @RefreshScope
 @ConfigurationProperties
-public class ConsulConfiguration {
+public class RemoteConfiguration {
 
     private Map<String, RemoteConnectionConfiguration> connections;
 
     public RemoteConnectionConfiguration getConnectionConfigByName(String configName) {
         return connections.get(configName);
     }
 
-    public Map<String, RemoteConnectionConfiguration> getAllConnections() {
+    public Map<String, RemoteConnectionConfiguration> getConnections() {
         return connections;
     }
 

src/main/java/com/dataexchange/client/config/aws/AWSConfiguration.java

@@ -0,0 +1,59 @@
+package com.dataexchange.client.config.aws;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
+import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement;
+import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClient;
+import org.springframework.boot.autoconfigure.AutoConfigureBefore;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.cloud.aws.autoconfigure.paramstore.AwsParamStoreBootstrapConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import static com.amazonaws.util.StringUtils.isNullOrEmpty;
+
+
+@Configuration
+@EnableConfigurationProperties(AWSCredentials.class)
+@AutoConfigureBefore(value = AwsParamStoreBootstrapConfiguration.class)
+@ConditionalOnProperty(
+        prefix = "aws.credentials",
+        name = {"use-static-provider"}
+)
+public class AWSConfiguration {
+
+    @Bean
+    public AWSSimpleSystemsManagement ssmClient(AWSCredentials credentials) {
+        return AWSSimpleSystemsManagementClient.builder()
+                .withCredentials(buildCredentialsProvider(credentials))
+                .withRegion(credentials.getRegion())
+                .build();
+    }
+
+    private AWSCredentialsProvider buildCredentialsProvider(AWSCredentials credentials) {
+        if (isNullOrEmpty(credentials.getStsRoleArn())) {
+            return buildDefaultCredentialsProvider(credentials);
+        } else {
+            return new STSAssumeRoleSessionCredentialsProvider.Builder(credentials.getStsRoleArn(), credentials.getRoleSessionName())
+                    .withStsClient(buildSecurityTokenService(credentials))
+                    .build();
+        }
+    }
+
+    private AWSSecurityTokenService buildSecurityTokenService(AWSCredentials credentials) {
+        return AWSSecurityTokenServiceClientBuilder.standard()
+                .withCredentials(buildDefaultCredentialsProvider(credentials))
+                .withRegion(credentials.getRegion())
+                .build();
+    }
+
+    private AWSCredentialsProvider buildDefaultCredentialsProvider(AWSCredentials credentials) {
+        return new AWSStaticCredentialsProvider(new BasicAWSCredentials(credentials.getAccessKey(), credentials.getSecretKey()));
+    }
+
+}

src/main/java/com/dataexchange/client/config/aws/AWSCredentials.java

@@ -0,0 +1,55 @@
+package com.dataexchange.client.config.aws;
+
+import com.amazonaws.regions.Regions;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+
+@ConfigurationProperties("aws.credentials")
+public class AWSCredentials {
+
+    private Regions region;
+    private String accessKey;
+    private String secretKey;
+    private String stsRoleArn;
+    private String roleSessionName;
+
+    public Regions getRegion() {
+        return region;
+    }
+
+    public void setRegion(Regions region) {
+        this.region = region;
+    }
+
+    public String getAccessKey() {
+        return accessKey;
+    }
+
+    public void setAccessKey(String accessKey) {
+        this.accessKey = accessKey;
+    }
+
+    public String getSecretKey() {
+        return secretKey;
+    }
+
+    public void setSecretKey(String secretKey) {
+        this.secretKey = secretKey;
+    }
+
+    public String getStsRoleArn() {
+        return stsRoleArn;
+    }
+
+    public void setStsRoleArn(String stsRoleArn) {
+        this.stsRoleArn = stsRoleArn;
+    }
+
+    public String getRoleSessionName() {
+        return roleSessionName;
+    }
+
+    public void setRoleSessionName(String roleSessionName) {
+        this.roleSessionName = roleSessionName;
+    }
+}

src/main/java/com/dataexchange/client/config/model/MainConfiguration.java

@@ -1,6 +1,6 @@
 package com.dataexchange.client.config.model;
 
-import com.dataexchange.client.config.ConsulConfiguration;
+import com.dataexchange.client.config.RemoteConfiguration;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.core.io.ByteArrayResource;
@@ -18,7 +18,7 @@
 public class MainConfiguration {
 
     @Autowired
-    private ConsulConfiguration consulConfiguration;
+    private RemoteConfiguration remoteConfiguration;
 
     @Valid
     private List<SftpPollerConfiguration> sftps = new ArrayList<>();
@@ -31,7 +31,7 @@ public void setUpFromRemote() {
                 .filter(SftpPollerConfiguration::hasRemoteConfig)
                 .forEach(sftp -> {
                     RemoteConnectionConfiguration remoteConfig =
-                            consulConfiguration.getConnectionConfigByName(sftp.getRemoteConfigName());
+                            remoteConfiguration.getConnectionConfigByName(sftp.getRemoteConfigName());
                     setConnectionConfiguration(sftp, remoteConfig);
                     setRemoteFolders(sftp, remoteConfig);
                     sftp.setPrivateKey(new ByteArrayResource(remoteConfig.getPrivateKey().getBytes()));
@@ -41,7 +41,7 @@ public void setUpFromRemote() {
                 .filter(FtpPollerConfiguration::hasRemoteConfig)
                 .forEach(ftp -> {
                     RemoteConnectionConfiguration remoteConfig =
-                            consulConfiguration.getConnectionConfigByName(ftp.getRemoteConfigName());
+                            remoteConfiguration.getConnectionConfigByName(ftp.getRemoteConfigName());
                     setConnectionConfiguration(ftp, remoteConfig);
                     setRemoteFolders(ftp, remoteConfig);
                 });

src/main/resources/META-INF/spring.factories

@@ -0,0 +1,2 @@
+org.springframework.cloud.bootstrap.BootstrapConfiguration=\
+com.dataexchange.client.config.aws.AWSConfiguration

src/main/resources/bootstrap.yml

@@ -3,4 +3,10 @@ spring:
     consul:
       enabled: false
       config:
-        enabled: false
+        enabled: false
+
+aws:
+  credentials:
+    use-static-provider: false
+  paramstore:
+    enabled: false