Merge pull request #17864 from iterate-ch/feature/GH-11664

Add support for AWS credential_process configuration, allowing external processes to provide temporary AWS credentials.

Author: dkocher

pom.xml

@@ -396,7 +396,7 @@
             <dependency>
                 <groupId>com.amazonaws</groupId>
                 <artifactId>aws-java-sdk-bom</artifactId>
-                <version>1.12.778</version>
+                <version>1.12.797</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>

s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java

@@ -17,6 +17,7 @@
 
 import ch.cyberduck.core.Credentials;
 import ch.cyberduck.core.CredentialsConfigurator;
+import ch.cyberduck.core.Factory;
 import ch.cyberduck.core.Host;
 import ch.cyberduck.core.Local;
 import ch.cyberduck.core.LocalFactory;
@@ -33,8 +34,11 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
+import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
 
@@ -94,6 +98,48 @@ else if(StringUtils.equals(entry.getValue().getAwsAccessIdKey(), credentials.get
             return false;
         }).map(Map.Entry::getValue).findFirst().orElse(StringUtils.isBlank(host.getCredentials().getUsername()) ? profiles.get("default") : null);
         if(null != profile) {
+            if(profile.isProcessBasedProfile()) {
+                // Uses external process to retrieve temporary credentials
+                final String command = profile.getCredentialProcess();
+                final ObjectMapper mapper = JsonMapper.builder()
+                        .serializationInclusion(Include.NON_NULL)
+                        .enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY)
+                        .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+                        .visibility(PropertyAccessor.FIELD, Visibility.ANY).build();
+                List<String> cmd = new ArrayList<>();
+                switch(Factory.Platform.getDefault()) {
+                    case windows:
+                        cmd.add("cmd");
+                        cmd.add("/c");
+                        break;
+                    default:
+                        cmd.add("sh");
+                        cmd.add("-c");
+                        break;
+                }
+                cmd.add(command);
+                final ProcessBuilder builder = new ProcessBuilder(cmd);
+                try {
+                    final Process process = builder.start();
+                    try(InputStream reader = process.getInputStream()) {
+                        final CachedCredential cached = mapper.readValue(reader, CachedCredential.class);
+                        credentials.setTokens(new TemporaryAccessTokens(
+                                cached.accessKey, cached.secretKey, cached.sessionToken, cached.getExpiration()));
+                        process.waitFor();
+                        if(process.exitValue() != 0) {
+                            throw new IOException(String.format("Unexpected exit code %d for process %s", process.exitValue(), command));
+                        }
+                        return credentials;
+                    }
+                    finally {
+                        process.destroy();
+                    }
+                }
+                catch(IOException | InterruptedException e) {
+                    log.warn("Failure \"{}\" parsing credentials from output of command {}", e.getMessage(), command);
+                    return credentials;
+                }
+            }
             if(profile.isRoleBasedProfile()) {
                 log.debug("Configure credentials from role based profile {}", profile.getProfileName());
                 if(StringUtils.isBlank(profile.getRoleSourceProfile())) {
@@ -115,7 +161,7 @@ else if(!profiles.containsKey(profile.getRoleSourceProfile())) {
                         }
                         // No further token exchange required
                         return credentials.setTokens(new TemporaryAccessTokens(
-                                cached.accessKey, cached.secretKey, cached.sessionToken, Instant.parse(cached.expiration).toEpochMilli()));
+                                cached.accessKey, cached.secretKey, cached.sessionToken, cached.getExpiration()));
                     }
                     else {
                         // If a profile defines the role_arn property then the profile is treated as an assume role profile
@@ -137,7 +183,7 @@ else if(!profiles.containsKey(profile.getRoleSourceProfile())) {
                         return credentials;
                     }
                     return credentials.setTokens(new TemporaryAccessTokens(
-                            cached.accessKey, cached.secretKey, cached.sessionToken, Instant.parse(cached.expiration).toEpochMilli()));
+                            cached.accessKey, cached.secretKey, cached.sessionToken, cached.getExpiration()));
                 }
                 log.debug("Set credentials from profile {}", profile.getProfileName());
                 return credentials
@@ -241,7 +287,7 @@ private CachedCredential fetchSsoCredentials(final Map<String, String> propertie
                     log.warn("Failure parsing SSO credentials.");
                     return null;
                 }
-                final Instant expiration = Instant.parse(cached.credentials.expiration);
+                final Instant expiration = Instant.ofEpochMilli(cached.credentials.getExpiration());
                 if(expiration.isBefore(Instant.now())) {
                     log.warn("Expired AWS SSO credentials.");
                     return null;
@@ -276,6 +322,15 @@ private static class CachedCredential {
         private String sessionToken;
         @JsonProperty("Expiration")
         private String expiration;
+
+        public Long getExpiration() {
+            try {
+                return Instant.parse(expiration).toEpochMilli();
+            }
+            catch(DateTimeParseException e) {
+                return -1L;
+            }
+        }
     }
 
     /**

s3/src/test/java/ch/cyberduck/core/s3/S3CredentialsConfiguratorTest.java

@@ -16,6 +16,7 @@
  */
 
 import ch.cyberduck.core.Credentials;
+import ch.cyberduck.core.Factory;
 import ch.cyberduck.core.Host;
 import ch.cyberduck.core.LocalFactory;
 import ch.cyberduck.core.TestProtocol;
@@ -26,6 +27,7 @@
 import java.io.File;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeFalse;
 
 public class S3CredentialsConfiguratorTest {
 
@@ -38,16 +40,14 @@ public void testConfigure() throws Exception {
     @Test
     public void readFailureForInvalidAWSCredentialsProfileEntry() throws Exception {
         final Credentials credentials = new Credentials("test_s3_profile");
-        final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/invalid/.aws").getAbsolutePath())
-        )
+        final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/invalid/.aws").getAbsolutePath()))
                 .reload().configure(new Host(new TestProtocol(), StringUtils.EMPTY, credentials));
         assertEquals(credentials, verify);
     }
 
     @Test
     public void readSuccessForValidAWSCredentialsProfileEntry() throws Exception {
-        final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath())
-        )
+        final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath()))
                 .reload().configure(new Host(new TestProtocol(), StringUtils.EMPTY, new Credentials("test_s3_profile")));
         assertEquals("test_s3_profile", verify.getUsername());
         assertEquals("", verify.getPassword());
@@ -58,12 +58,38 @@ public void readSuccessForValidAWSCredentialsProfileEntry() throws Exception {
 
     @Test
     public void readSSOCachedTemporaryTokens() throws Exception {
-        final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath())
-        )
+        final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath()))
                 .reload().configure(new Host(new TestProtocol(), StringUtils.EMPTY, new Credentials("ReadOnlyAccess-189584543480")));
         assertEquals("TESTACCESSKEY", verify.getTokens().getAccessKeyId());
         assertEquals("TESTSECRETKEY", verify.getTokens().getSecretAccessKey());
         assertEquals("TESTSESSIONTOKEN", verify.getTokens().getSessionToken());
         assertEquals(3497005724000L, verify.getTokens().getExpiryInMilliseconds(), 0L);
     }
+
+    @Test
+    public void readCredentialProcessTokens() throws Exception {
+        assumeFalse(Factory.Platform.getDefault().equals(Factory.Platform.Name.windows));
+        final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath()))
+                .reload().configure(new Host(new TestProtocol(), StringUtils.EMPTY, new Credentials("credential_process_profile")));
+        assertEquals("PROCESSACCESSKEY", verify.getTokens().getAccessKeyId());
+        assertEquals("PROCESSSECRETKEY", verify.getTokens().getSecretAccessKey());
+        assertEquals("PROCESSSESSIONTOKEN", verify.getTokens().getSessionToken());
+        assertEquals(3497005724000L, verify.getTokens().getExpiryInMilliseconds(), 0L);
+    }
+
+    @Test
+    public void readCredentialProcessTokensExitCode() throws Exception {
+        assumeFalse(Factory.Platform.getDefault().equals(Factory.Platform.Name.windows));
+        final Credentials credentials = new Credentials("credential_process_profile_error_exit");
+        assertEquals(credentials, new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath()))
+                .reload().configure(new Host(new TestProtocol(), StringUtils.EMPTY, credentials)));
+    }
+
+    @Test
+    public void readCredentialProcessTokensParseError() throws Exception {
+        assumeFalse(Factory.Platform.getDefault().equals(Factory.Platform.Name.windows));
+        final Credentials credentials = new Credentials("credential_process_profile_parser_error");
+        assertEquals(credentials, new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath()))
+                .reload().configure(new Host(new TestProtocol(), StringUtils.EMPTY, credentials)));
+    }
 }

s3/src/test/resources/valid/.aws/config

@@ -4,3 +4,12 @@ sso_region = us-east-1
 sso_account_id = 189584543480
 sso_role_name = ReadOnlyAccess
 region = eu-west-1
+
+[profile credential_process_profile]
+credential_process = printf '{"Version": 1,"AccessKeyId":"PROCESSACCESSKEY","SecretAccessKey":"PROCESSSECRETKEY","SessionToken":"PROCESSSESSIONTOKEN","Expiration":"2080-10-24T14:28:44Z"}'
+
+[profile credential_process_profile_error_exit]
+credential_process = exit 1
+
+[profile credential_process_profile_parser_error]
+credential_process = printf '{"Version": 1'