diff --git a/agent/pom.xml b/agent/pom.xml index 1ae72c37b8..6fd9e2b018 100644 --- a/agent/pom.xml +++ b/agent/pom.xml @@ -29,6 +29,10 @@ com.walmartlabs.concord concord-common + + com.walmartlabs.concord + concord-github-app-installation + com.walmartlabs.concord concord-client2 @@ -143,6 +147,11 @@ mockito-core test + + org.mockito + mockito-junit-jupiter + test + diff --git a/agent/src/main/java/com/walmartlabs/concord/agent/AgentAuthTokenProvider.java b/agent/src/main/java/com/walmartlabs/concord/agent/AgentAuthTokenProvider.java new file mode 100644 index 0000000000..26ed86af44 --- /dev/null +++ b/agent/src/main/java/com/walmartlabs/concord/agent/AgentAuthTokenProvider.java @@ -0,0 +1,85 @@ +package com.walmartlabs.concord.agent; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.typesafe.config.Config; +import com.walmartlabs.concord.agent.remote.ApiClientFactory; +import com.walmartlabs.concord.common.AuthTokenProvider; +import com.walmartlabs.concord.common.ExternalAuthToken; +import com.walmartlabs.concord.github.appinstallation.GitHubAppInstallation; +import com.walmartlabs.concord.sdk.Secret; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.net.URI; +import java.util.List; +import java.util.Optional; + +public class AgentAuthTokenProvider implements AuthTokenProvider { + private final List authTokenProviders; + + @Inject + public AgentAuthTokenProvider(GitHubAppInstallation githubProvider, + OauthTokenProvider oauthTokenProvider) { + + this.authTokenProviders = List.of(githubProvider, oauthTokenProvider); + } + + @Override + public boolean supports(URI repo, @Nullable Secret secret) { + return authTokenProviders.stream() + .anyMatch(p -> p.supports(repo, secret)); + } + + public Optional getToken(URI repo, @Nullable Secret secret) { + for (var tokenProvider : authTokenProviders) { + if (tokenProvider.supports(repo, secret)) { + return tokenProvider.getToken(repo, secret); + } + } + + return Optional.empty(); + } + + public static class ConcordServerTokenProvider implements AuthTokenProvider { + private final ApiClientFactory apiClientFactory; + private final Config config; + + @Inject + public ConcordServerTokenProvider(ApiClientFactory apiClientFactory, Config config) { + this.apiClientFactory = apiClientFactory; + this.config = config; + } + + + @Override + public boolean supports(URI repo, @Nullable Secret secret) { + // TODO implement + return false; + } + + @Override + public Optional getToken(URI repo, @Nullable Secret secret) { + // TODO implement + return Optional.empty(); + } + } +} diff --git a/agent/src/main/java/com/walmartlabs/concord/agent/AgentModule.java b/agent/src/main/java/com/walmartlabs/concord/agent/AgentModule.java index 81ad08b741..cc6e06f1e1 100644 --- a/agent/src/main/java/com/walmartlabs/concord/agent/AgentModule.java +++ b/agent/src/main/java/com/walmartlabs/concord/agent/AgentModule.java @@ -30,7 +30,9 @@ import com.walmartlabs.concord.agent.remote.ApiClientFactory; import com.walmartlabs.concord.agent.remote.QueueClientProvider; import com.walmartlabs.concord.common.ObjectMapperProvider; +import com.walmartlabs.concord.common.cfg.OauthTokenConfig; import com.walmartlabs.concord.config.ConfigModule; +import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig; import com.walmartlabs.concord.server.queueclient.QueueClient; import javax.inject.Named; @@ -59,6 +61,10 @@ public void configure(Binder binder) { binder.bind(DockerConfiguration.class).in(SINGLETON); binder.bind(RuntimeConfiguration.class).asEagerSingleton(); binder.bind(GitConfiguration.class).in(SINGLETON); + binder.bind(OauthTokenConfig.class).to(GitConfiguration.class).in(SINGLETON); + binder.bind(GitHubConfiguration.class).in(SINGLETON); + binder.bind(GitHubAppInstallationConfig.class).to(GitHubConfiguration.class).in(SINGLETON); + binder.bind(AgentAuthTokenProvider.ConcordServerTokenProvider.class).in(SINGLETON); binder.bind(ImportConfiguration.class).in(SINGLETON); binder.bind(PreForkConfiguration.class).in(SINGLETON); binder.bind(RepositoryCacheConfiguration.class).in(SINGLETON); diff --git a/agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java b/agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java index e743d4203d..25ea100e0f 100644 --- a/agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java +++ b/agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java @@ -34,7 +34,6 @@ import javax.inject.Inject; import java.io.IOException; import java.nio.file.Path; -import java.util.Arrays; import java.util.List; public class RepositoryManager { @@ -51,13 +50,13 @@ public RepositoryManager(SecretClient secretClient, GitConfiguration gitCfg, RepositoryCacheConfiguration cacheCfg, ObjectMapper objectMapper, - DependencyManager dependencyManager) throws IOException { + DependencyManager dependencyManager, + AgentAuthTokenProvider agentAuthTokenProvider) throws IOException { this.secretClient = secretClient; this.gitCfg = gitCfg; GitClientConfiguration clientCfg = GitClientConfiguration.builder() - .oauthToken(gitCfg.getToken()) .defaultOperationTimeout(gitCfg.getDefaultOperationTimeout()) .fetchTimeout(gitCfg.getFetchTimeout()) .httpLowSpeedLimit(gitCfg.getHttpLowSpeedLimit()) @@ -66,9 +65,10 @@ public RepositoryManager(SecretClient secretClient, .sshTimeoutRetryCount(gitCfg.getSshTimeoutRetryCount()) .build(); - List providers = Arrays.asList(new MavenRepositoryProvider(dependencyManager), new GitCliRepositoryProvider(clientCfg)); - this.providers = new RepositoryProviders(providers); - + this.providers = new RepositoryProviders(List.of( + new MavenRepositoryProvider(dependencyManager), + new GitCliRepositoryProvider(clientCfg, agentAuthTokenProvider) + )); this.repositoryCache = new RepositoryCache(cacheCfg.getCacheDir(), cacheCfg.getInfoDir(), cacheCfg.getLockTimeout(), diff --git a/agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitConfiguration.java b/agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitConfiguration.java index 19d63aa9a7..85abb768e9 100644 --- a/agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitConfiguration.java +++ b/agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitConfiguration.java @@ -21,15 +21,21 @@ */ import com.typesafe.config.Config; +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.common.cfg.OauthTokenConfig; import javax.inject.Inject; import java.time.Duration; +import java.util.List; +import java.util.Optional; import static com.walmartlabs.concord.agent.cfg.Utils.getStringOrDefault; -public class GitConfiguration { +public class GitConfiguration implements OauthTokenConfig { private final String token; + private final String oauthUsername; + private final String oauthUrlPattern; private final boolean shallowClone; private final boolean checkAlreadyFetched; private final Duration defaultOperationTimeout; @@ -39,10 +45,13 @@ public class GitConfiguration { private final Duration sshTimeout; private final int sshTimeoutRetryCount; private final boolean skip; + private final List authConfigs; @Inject public GitConfiguration(Config cfg) { this.token = getStringOrDefault(cfg, "git.oauth", () -> null); + this.oauthUsername = getStringOrDefault(cfg, "git.oauthUsername", () -> null); + this.oauthUrlPattern = getStringOrDefault(cfg, "git.oauthUrlPattern", () -> null); this.shallowClone = cfg.getBoolean("git.shallowClone"); this.checkAlreadyFetched = cfg.getBoolean("git.checkAlreadyFetched"); this.defaultOperationTimeout = cfg.getDuration("git.defaultOperationTimeout"); @@ -52,10 +61,22 @@ public GitConfiguration(Config cfg) { this.sshTimeout = cfg.getDuration("git.sshTimeout"); this.sshTimeoutRetryCount = cfg.getInt("git.sshTimeoutRetryCount"); this.skip = cfg.getBoolean("git.skip"); + this.authConfigs = cfg.getConfigList("git.systemAuth"); } - public String getToken() { - return token; + @Override + public Optional getOauthToken() { + return Optional.ofNullable(token); + } + + @Override + public Optional getOauthUsername() { + return Optional.ofNullable(oauthUsername); + } + + @Override + public Optional getOauthUrlPattern() { + return Optional.ofNullable(oauthUrlPattern); } public boolean isShallowClone() { @@ -93,4 +114,46 @@ public int getSshTimeoutRetryCount() { public boolean isSkip() { return skip; } + + public List getSystemAuth() { + return authConfigs.stream() + .map(o -> { + AuthSource type = AuthSource.valueOf(o.getString("type").toUpperCase()); + + return (AuthConfig) switch (type) { + case OAUTH_TOKEN -> OauthConfig.from(o); + }; + }) + .map(AuthConfig::toGitAuth) + .toList(); + + } + + enum AuthSource { + OAUTH_TOKEN + } + + public interface AuthConfig { + MappingAuthConfig toGitAuth(); + } + + public record OauthConfig(String id, String urlPattern, String token) implements AuthConfig { + + static OauthConfig from(Config cfg) { + return new OauthConfig( + getStringOrDefault(cfg, "id", () -> "system-oauth-token"), + cfg.getString("urlPattern"), + cfg.getString("token") + ); + } + + @Override + public MappingAuthConfig.OauthAuthConfig toGitAuth() { + return MappingAuthConfig.OauthAuthConfig.builder() + .id(this.id()) + .urlPattern(MappingAuthConfig.assertBaseUrlPattern(this.urlPattern())) + .token(this.token()) + .build(); + } + } } diff --git a/agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitHubConfiguration.java b/agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitHubConfiguration.java new file mode 100644 index 0000000000..9913ed4be4 --- /dev/null +++ b/agent/src/main/java/com/walmartlabs/concord/agent/cfg/GitHubConfiguration.java @@ -0,0 +1,63 @@ +package com.walmartlabs.concord.agent.cfg; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig; + +import javax.inject.Inject; +import java.time.Duration; +import java.util.List; + +public class GitHubConfiguration implements GitHubAppInstallationConfig { + + private static final String CFG_APP_INSTALLATION = "github.appInstallation"; + + private final GitHubAppInstallationConfig appInstallation; + + @Inject + public GitHubConfiguration(com.typesafe.config.Config config) { + if (config.hasPath(CFG_APP_INSTALLATION)) { + var raw = config.getConfig(CFG_APP_INSTALLATION); + this.appInstallation = GitHubAppInstallationConfig.fromConfig(raw); + } else { + this.appInstallation = GitHubAppInstallationConfig.builder() + .authConfigs(List.of()) + .build(); + } + } + + @Override + public List getAuthConfigs() { + return appInstallation.getAuthConfigs(); + } + + @Override + public Duration getSystemAuthCacheDuration() { + return appInstallation.getSystemAuthCacheDuration(); + } + + @Override + public long getSystemAuthCacheMaxWeight() { + return appInstallation.getSystemAuthCacheMaxWeight(); + } + +} diff --git a/agent/src/main/resources/concord-agent.conf b/agent/src/main/resources/concord-agent.conf index 47846066f7..ebdab716c2 100644 --- a/agent/src/main/resources/concord-agent.conf +++ b/agent/src/main/resources/concord-agent.conf @@ -188,9 +188,26 @@ concord-agent { # if true, skip Git fetch, use workspace state only skip = false - # GitHub auth token to use when cloning repositories without explicitly configured authentication + # GitHub auth token to use when cloning repositories without explicitly + # configured authentication. Deprecated in favor of systemAuth list of + # tokens or service-specific app config (e.g. github) # oauth = "..." + # specific username to use for auth + # oauthUsername = "" + + # regex to match against git server's hostname + port + path so oauth + # token isn't used for and unexpected host + # oauthUrlPattern = "" + + # List of system-provided auth token configs + # { + # "token" = "...", + # "username" = "...", # optional, username to send with auth token + # "urlPattern" = "..." # required, regex to match against target git host + port + path + # } + systemAuth = [] + # use GIT's shallow clone shallowClone = true @@ -212,6 +229,32 @@ concord-agent { sshTimeout = "10 minutes" } + # github app settings. While this works on the agent, it's preferable to + # get auth token from concord-server via externalTokenProvider + github { + # App installation settings. Multiple auth (private key) definitions are supported, + # as each is matched to a particular url pattern. + appInstallation { + # { + # type = "GITHUB_APP_INSTALLATION", + # urlPattern = "github.com", # regex + # username = "...", # optional, defaults to "x-access-token" + # apiUrl = "https://api.github.com", # github api url, usually *not* the same as the repo url host/path + # clientId = "...", + # privateKey = "/path/to/pk.pem" + # } + # or static oauth config. Not exactly a "GitHub App", but can do some + # API interactions and cloning. Less preferred to actual app. + # { + # type = "OAUTH_TOKEN", + # token = "...", + # username = "...", # optional, usually not necessary + # urlPattern = "..." # regex to match against git server's hostname + port + path + # } + auth = [] + } + } + imports { # base git url for imports src = "" diff --git a/agent/src/test/java/com/walmartlabs/concord/agent/AgentAuthTokenProviderTest.java b/agent/src/test/java/com/walmartlabs/concord/agent/AgentAuthTokenProviderTest.java new file mode 100644 index 0000000000..02d74c64ee --- /dev/null +++ b/agent/src/test/java/com/walmartlabs/concord/agent/AgentAuthTokenProviderTest.java @@ -0,0 +1,113 @@ +package com.walmartlabs.concord.agent; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.AuthTokenProvider; +import com.walmartlabs.concord.common.ExternalAuthToken; +import com.walmartlabs.concord.github.appinstallation.GitHubAppInstallation; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.net.URI; +import java.time.OffsetDateTime; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class AgentAuthTokenProviderTest { + + @Mock + GitHubAppInstallation ghApp; + + @Mock + AuthTokenProvider.OauthTokenProvider oauthTokenProvider; + + @Test + void testGitHubApp() { + when(ghApp.getToken(any(), any())). + thenReturn(Optional.of(ExternalAuthToken.SimpleToken.builder() + .token("gh-installation-token") + .expiresAt(OffsetDateTime.now().plusMinutes(60)) + .build())); + when(ghApp.supports(any(), any())).thenReturn(true); + + var provider = new AgentAuthTokenProvider(ghApp, oauthTokenProvider); + + // -- + + assertTrue(provider.supports(URI.create("https://github.local/owner/repo.git"), null)); + var o = provider.getToken(URI.create("https://github.local/owner/repo.git"), null); + + // -- + + assertTrue(o.isPresent()); + var result = assertInstanceOf(ExternalAuthToken.class, o.get()); + assertEquals("gh-installation-token", result.token()); + } + + @Test + void testOauth() { + when(oauthTokenProvider.supports(any(), any())).thenReturn(true); + when(oauthTokenProvider.getToken(any(), any())) + .thenReturn(Optional.of(ExternalAuthToken.StaticToken.builder() + .token("oauth-token") + .build())); + + var provider = new AgentAuthTokenProvider(ghApp, oauthTokenProvider); + + // -- + + assertTrue(provider.supports(URI.create("https://github.local/owner/repo.git"), null)); + var o = provider.getToken(URI.create("https://github.local/owner/repo.git"), null); + + // -- + + assertTrue(o.isPresent()); + var result = assertInstanceOf(ExternalAuthToken.class, o.get()); + assertEquals("oauth-token", result.token()); + } + + @Test + void testNoAuth() { + when(ghApp.supports(any(), any())).thenReturn(false); + when(oauthTokenProvider.supports(any(), any())).thenReturn(false); + + var provider = new AgentAuthTokenProvider(ghApp, oauthTokenProvider); + + // -- + + assertFalse(provider.supports(URI.create("https://github.local/owner/repo.git"), null)); + var o = provider.getToken(URI.create("https://github.local/owner/repo.git"), null); + + // -- + + assertFalse(o.isPresent()); + } + +} diff --git a/cli/src/main/java/com/walmartlabs/concord/cli/runner/CliRepositoryExporter.java b/cli/src/main/java/com/walmartlabs/concord/cli/runner/CliRepositoryExporter.java index 7237f1e904..5b851a9049 100644 --- a/cli/src/main/java/com/walmartlabs/concord/cli/runner/CliRepositoryExporter.java +++ b/cli/src/main/java/com/walmartlabs/concord/cli/runner/CliRepositoryExporter.java @@ -20,17 +20,22 @@ * ===== */ +import com.walmartlabs.concord.common.AuthTokenProvider; +import com.walmartlabs.concord.common.ExternalAuthToken; import com.walmartlabs.concord.imports.Import; import com.walmartlabs.concord.imports.RepositoryExporter; import com.walmartlabs.concord.repository.*; import com.walmartlabs.concord.sdk.Secret; +import javax.annotation.Nullable; import java.io.UnsupportedEncodingException; +import java.net.URI; import java.net.URLEncoder; import java.nio.file.Path; import java.time.Duration; -import java.util.Collections; +import java.util.List; import java.util.Objects; +import java.util.Optional; public class CliRepositoryExporter implements RepositoryExporter { @@ -50,7 +55,6 @@ public CliRepositoryExporter(Path repoCacheDir) { this.repoCacheDir = repoCacheDir; GitClientConfiguration clientCfg = GitClientConfiguration.builder() - .oauthToken(null) .defaultOperationTimeout(DEFAULT_OPERATION_TIMEOUT) .fetchTimeout(FETCH_TIMEOUT) .httpLowSpeedLimit(HTTP_LOW_SPEED_LIMIT) @@ -59,7 +63,19 @@ public CliRepositoryExporter(Path repoCacheDir) { .sshTimeoutRetryCount(SSH_TIMEOUT_RETRY_COUNT) .build(); - this.providers = new RepositoryProviders(Collections.singletonList(new GitCliRepositoryProvider(clientCfg))); + AuthTokenProvider authProvider = new AuthTokenProvider() { + @Override + public boolean supports(URI repo, @Nullable Secret secret) { + return false; + } + + @Override + public Optional getToken(URI repo, @Nullable Secret secret) throws RepositoryException { + throw new UnsupportedOperationException("Not supported"); + } + }; + + this.providers = new RepositoryProviders(List.of(new GitCliRepositoryProvider(clientCfg, authProvider))); } @Override diff --git a/common/pom.xml b/common/pom.xml index 46e3352516..0fb9d05ee0 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -49,6 +49,11 @@ commons-validator provided + + org.immutables + value + provided + com.fasterxml.jackson.core jackson-databind @@ -92,6 +97,16 @@ junit-jupiter-api test + + org.mockito + mockito-core + test + + + org.mockito + mockito-junit-jupiter + test + diff --git a/common/src/main/java/com/walmartlabs/concord/common/AuthTokenProvider.java b/common/src/main/java/com/walmartlabs/concord/common/AuthTokenProvider.java new file mode 100644 index 0000000000..34f0874788 --- /dev/null +++ b/common/src/main/java/com/walmartlabs/concord/common/AuthTokenProvider.java @@ -0,0 +1,142 @@ +package com.walmartlabs.concord.common; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.common.cfg.OauthTokenConfig; +import com.walmartlabs.concord.common.secret.BinaryDataSecret; +import com.walmartlabs.concord.sdk.Secret; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; + +public interface AuthTokenProvider { + + /** + * @return {@code true} if this the given repo URI and secret are compatible + * with this provider's {@link #getToken(URI, Secret)} method, + * {@code false} otherwise. + */ + boolean supports(URI repo, @Nullable Secret secret); + + Optional getToken(URI repo, @Nullable Secret secret); + + default URI addUserInfoToUri(URI repo, @Nullable Secret secret) { + if (!supports(repo, secret)) { + // not compatible with auth provider(s) + return repo; + } + + return getToken(repo, secret) + .map(expiringToken -> { + var token = expiringToken.token(); + var userInfo = expiringToken.username() != null + ? expiringToken.username() + ":" + token + : token; + + try { + return new URI(repo.getScheme(), userInfo, repo.getHost(), + repo.getPort(), repo.getPath(), repo.getQuery(), repo.getFragment()); + } catch (URISyntaxException e) { + return null; + } + }) + .orElse(repo); + } + + @SuppressWarnings("ClassCanBeRecord") + class OauthTokenProvider implements AuthTokenProvider { + // >0 length, printable ascii (no newlines, etc) + private static final Pattern BASIC_STRING_PTN = Pattern.compile("[ -~]+"); + private final List authConfigs; + + @Inject + public OauthTokenProvider(OauthTokenConfig config) { + this.authConfigs = toConfigList(config); + } + + @Override + public boolean supports(URI repo, @Nullable Secret secret) { + return validateSecret(secret) || systemSupports(repo); + } + + @Override + public Optional getToken(URI repo, @Nullable Secret secret) { + if (secret != null) { + if (secret instanceof BinaryDataSecret bds) { + return Optional.of(ExternalAuthToken.StaticToken.builder() + .token(new String(bds.getData())) + .build()); + } else { + return Optional.empty(); + } + } + + return authConfigs.stream() + .filter(auth -> auth.canHandle(repo)) + .filter(MappingAuthConfig.OauthAuthConfig.class::isInstance) + .map(MappingAuthConfig.OauthAuthConfig.class::cast) + .findFirst() + .map(auth -> ExternalAuthToken.StaticToken.builder() + .token(auth.token()) + .username(auth.username()) + .build()); + } + + private boolean validateSecret(Secret secret) { + if (secret == null) { + return false; + } + + if (!(secret instanceof BinaryDataSecret bds)) { + // this class is not the place for handling key pairs or username/password + return false; + } else { + var data = new String(bds.getData()); + return BASIC_STRING_PTN.matcher(data).matches(); + } + } + + private boolean systemSupports(URI repoUri) { + return authConfigs.stream().anyMatch(auth -> auth.canHandle(repoUri)); + } + + private static List toConfigList(OauthTokenConfig config) { + var token = config.getOauthToken().orElse(null); + + if (token == null || token.isBlank() && config.getSystemAuth().isEmpty()) { + return config.getSystemAuth(); + } + + return List.of(MappingAuthConfig.OauthAuthConfig.builder() + .id("static-token") + .token(token) + .username(config.getOauthUsername().orElse(null)) + .urlPattern(MappingAuthConfig.assertBaseUrlPattern(config.getOauthUrlPattern().orElse(".*")))// for backwards compat with git.oauth + .build()); + } + } +} diff --git a/common/src/main/java/com/walmartlabs/concord/common/ExternalAuthToken.java b/common/src/main/java/com/walmartlabs/concord/common/ExternalAuthToken.java new file mode 100644 index 0000000000..952f02f64b --- /dev/null +++ b/common/src/main/java/com/walmartlabs/concord/common/ExternalAuthToken.java @@ -0,0 +1,94 @@ +package com.walmartlabs.concord.common; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.immutables.value.Value; + +import javax.annotation.Nullable; +import java.time.Duration; +import java.time.OffsetDateTime; + +@JsonDeserialize(as = ImmutableSimpleToken.class) +public interface ExternalAuthToken { + + @Nullable + @JsonProperty("auth_id") + String authId(); + + @JsonProperty("token") + String token(); + + @Nullable + @JsonProperty("username") + String username(); + + @Nullable + @JsonProperty("expires_at") + // GitHub gives time in seconds, but most parsers (e.g. jackson) expect milliseconds + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss[.SSS]X") + OffsetDateTime expiresAt(); + + @Value.Default + @JsonIgnore + default long secondsUntilExpiration() { + if (expiresAt() == null) { + return Long.MAX_VALUE; + } + + var d = Duration.between(OffsetDateTime.now(), expiresAt()); + return d.getSeconds(); + } + + /** + * Basic implementation of an expiring token. + */ + @Value.Immutable + @Value.Style(jdkOnly = true) + interface SimpleToken extends ExternalAuthToken { + static ImmutableSimpleToken.Builder builder() { + return ImmutableSimpleToken.builder(); + } + } + + /** + * A token that effectively never expires. + */ + @Value.Immutable + @Value.Style(jdkOnly = true) + interface StaticToken extends ExternalAuthToken { + + @Value.Default + @Nullable + @Override + default OffsetDateTime expiresAt() { + return null; + } + + static ImmutableStaticToken.Builder builder() { + return ImmutableStaticToken.builder(); + } + } + +} diff --git a/common/src/main/java/com/walmartlabs/concord/common/cfg/MappingAuthConfig.java b/common/src/main/java/com/walmartlabs/concord/common/cfg/MappingAuthConfig.java new file mode 100644 index 0000000000..a390d12011 --- /dev/null +++ b/common/src/main/java/com/walmartlabs/concord/common/cfg/MappingAuthConfig.java @@ -0,0 +1,91 @@ +package com.walmartlabs.concord.common.cfg; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import org.immutables.value.Value; + +import javax.annotation.Nullable; +import java.net.URI; +import java.util.regex.Pattern; + +/** + * Configuration for mapping Git repository URLs to an authentication method. + * Mapping is based on regex matching (see {@link #urlPattern()}) against the + * repository URL. + */ +public interface MappingAuthConfig { + + /** + * Identification for the auth config. Should be unique within the + * application config. May be used for identifying source configs in metrics */ + String id(); + + /** Regex matching the host, optional port and path of a Git repository URL. */ + Pattern urlPattern(); + + /** + * Username to use for authentication with a provided token. Some services + * (e.g. GitHub API for app installation) require a specific username. Others + * (e.g. GitHub API for personal access tokens) accept just the token and no username + */ + @Nullable + String username(); + + /** + * For compatibility with a {@link MappingAuthConfig} instance, a URI must match the + * {@link #urlPattern()} regex. The regex may match against the path to support + * either a Git host behind a reverse proxy or restricting the auth to specific + * org/repo patterns. + * @return {@code true} if this provider can handle the given repo URI, {@code false} otherwise. + */ + default boolean canHandle(URI repo) { + var port = (repo.getPort() == -1 ? "" : (":" + repo.getPort())); + var path = (repo.getPath() == null ? "" : repo.getPath()); + var repoHostPortAndPath = repo.getHost() + port + path; + + return repoHostPortAndPath.matches(urlPattern() + ".*"); + } + + static Pattern assertBaseUrlPattern(String pattern) { + return pattern.endsWith(".*") + ? Pattern.compile(pattern) + : Pattern.compile(pattern + ".*"); + } + + @Value.Immutable + @Value.Style(jdkOnly = true) + interface OauthAuthConfig extends MappingAuthConfig { + String token(); + + static ImmutableOauthAuthConfig.Builder builder() { + return ImmutableOauthAuthConfig.builder(); + } + } + + @Value.Immutable + @Value.Style(jdkOnly = true) + interface ConcordServerAuthConfig extends MappingAuthConfig { + static ImmutableConcordServerAuthConfig.Builder builder() { + return ImmutableConcordServerAuthConfig.builder(); + } + } + +} diff --git a/common/src/main/java/com/walmartlabs/concord/common/cfg/OauthTokenConfig.java b/common/src/main/java/com/walmartlabs/concord/common/cfg/OauthTokenConfig.java new file mode 100644 index 0000000000..5fb708b272 --- /dev/null +++ b/common/src/main/java/com/walmartlabs/concord/common/cfg/OauthTokenConfig.java @@ -0,0 +1,36 @@ +package com.walmartlabs.concord.common.cfg; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import java.util.List; +import java.util.Optional; + +public interface OauthTokenConfig { + + Optional getOauthToken(); + + Optional getOauthUsername(); + + Optional getOauthUrlPattern(); + + List getSystemAuth(); + +} diff --git a/common/src/test/java/com/walmartlabs/concord/common/AuthTokenProviderTest.java b/common/src/test/java/com/walmartlabs/concord/common/AuthTokenProviderTest.java new file mode 100644 index 0000000000..b1a07b0f2b --- /dev/null +++ b/common/src/test/java/com/walmartlabs/concord/common/AuthTokenProviderTest.java @@ -0,0 +1,155 @@ +package com.walmartlabs.concord.common; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.common.cfg.OauthTokenConfig; +import com.walmartlabs.concord.common.secret.BinaryDataSecret; +import com.walmartlabs.concord.common.secret.UsernamePassword; +import org.immutables.value.Value; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class AuthTokenProviderTest { + + private static final byte[] SECRET_BYTES = "abc123".getBytes(StandardCharsets.UTF_8); + private static final String MOCK_TOKEN = "mock-token"; + private static final String MOCK_USERNAME = "mock-username"; + private static final String VALID_REPO = "https://github.local/owner/repo.git"; + + @Mock + BinaryDataSecret binaryDataSecret; + + @Mock + UsernamePassword usernamePassword; + + @Mock + MappingAuthConfig.OauthAuthConfig oauth; + + @Mock + TestOauthTokenConfig oauthTokenConfig; + + @Test + void testSingleOauth() { + // the "old" config approach + when(oauthTokenConfig.getOauthToken()).thenReturn(Optional.of(MOCK_TOKEN)); + when(oauthTokenConfig.getOauthUrlPattern()).thenReturn(Optional.of("github\\.local")); + when(oauthTokenConfig.getOauthUsername()).thenReturn(Optional.of(MOCK_USERNAME)); + + executeWithoutSecret(oauthTokenConfig); + + verify(oauthTokenConfig, times(1)).getOauthUrlPattern(); // retrieved once and stored + } + + @Test + void testSystemAuth() { + when(oauth.canHandle(any())).thenCallRealMethod(); + when(oauth.urlPattern()).thenReturn(Pattern.compile("github\\.local")); + when(oauth.token()).thenReturn(MOCK_TOKEN); + when(oauth.username()).thenReturn(MOCK_USERNAME); + + var cfg = TestOauthTokenConfig.builder() + .addSystemAuth(oauth) + .build(); + + executeWithoutSecret(cfg); + + verify(oauth, times(12)).canHandle(any()); + } + + void executeWithoutSecret(OauthTokenConfig cfg) { + var provider = new AuthTokenProvider.OauthTokenProvider(cfg); + + assertTrue(provider.supports(URI.create("https://github.local/owner/repo.git"), null)); + assertTrue(provider.supports(URI.create("https://github.local/owner/repo"), null)); + assertTrue(provider.supports(URI.create("https://github.local/owner/repo/"), null)); + assertFalse(provider.supports(URI.create("https://elsewhere.local/owner/repo.git"), null)); + assertFalse(provider.supports(URI.create("https://elsewhere.local/owner/repo"), null)); + + assertEquals(MOCK_TOKEN, provider.getToken(URI.create("https://github.local/owner/repo.git"), null).map(ExternalAuthToken::token).orElse(null)); + assertEquals(MOCK_TOKEN, provider.getToken(URI.create("https://github.local/owner/repo"), null).map(ExternalAuthToken::token).orElse(null)); + assertEquals(MOCK_TOKEN, provider.getToken(URI.create("https://github.local/owner/repo/"), null).map(ExternalAuthToken::token).orElse(null)); + assertFalse(provider.getToken(URI.create("https://elsewhere.local/owner/repo.git"), null).isPresent()); + assertFalse(provider.getToken(URI.create("https://elsewhere.local/owner/repo"), null).isPresent()); + + var enriched = provider.addUserInfoToUri(URI.create("https://github.local/owner/repo.git"), null); + assertEquals(MOCK_USERNAME + ":" + MOCK_TOKEN, enriched.getUserInfo()); + assertEquals("https://" + MOCK_USERNAME + ":" + MOCK_TOKEN + "@github.local/owner/repo.git", enriched.toString()); + } + + @Test + void testUsernamePassword() { + var cfg = TestOauthTokenConfig.builder().build(); + var provider = new AuthTokenProvider.OauthTokenProvider(cfg); + + assertFalse(provider.supports(URI.create(VALID_REPO), usernamePassword)); + } + + @Test + void testWithSecret() { + var cfg = TestOauthTokenConfig.builder() + .addSystemAuth(oauth) // won't be used + .build(); + + executeWithSecret(cfg); + } + + @Test + void testWithSecretNoDefault() { + var cfg = TestOauthTokenConfig.builder().build(); + + executeWithSecret(cfg); + } + + private void executeWithSecret(TestOauthTokenConfig cfg) { + var provider = new AuthTokenProvider.OauthTokenProvider(cfg); + + when(binaryDataSecret.getData()).thenReturn(SECRET_BYTES); + assertTrue(provider.supports(URI.create("https://github.local/owner/repo.git"), binaryDataSecret)); + + verify(oauth, never()).token(); // prove it wasn't used + verify(binaryDataSecret, times(1)).getData(); + } + + @Value.Immutable + interface TestOauthTokenConfig extends OauthTokenConfig { + static ImmutableTestOauthTokenConfig.Builder builder() { + return ImmutableTestOauthTokenConfig.builder(); + } + } +} diff --git a/common/src/test/java/com/walmartlabs/concord/common/ExternalAuthTokenTest.java b/common/src/test/java/com/walmartlabs/concord/common/ExternalAuthTokenTest.java new file mode 100644 index 0000000000..c365f2f125 --- /dev/null +++ b/common/src/test/java/com/walmartlabs/concord/common/ExternalAuthTokenTest.java @@ -0,0 +1,123 @@ +package com.walmartlabs.concord.common; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.time.OffsetDateTime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ExternalAuthTokenTest { + + static final String MOCK_TOKEN = "mock-token"; + static final ObjectMapper MAPPER = new ObjectMapperProvider().get(); + + @Test + void testExpiration() { + var externalToken = ExternalAuthToken.SimpleToken.builder() + .token(MOCK_TOKEN) + .expiresAt(OffsetDateTime.now().minusSeconds((100))) + .build(); + + assertTrue(externalToken.secondsUntilExpiration() < 0); + } + + @Test + void testStaticExpiration() { + var externalToken = ExternalAuthToken.StaticToken.builder() + .token(MOCK_TOKEN) + .build(); + + assertEquals(MOCK_TOKEN, externalToken.token()); + assertEquals(Long.MAX_VALUE, externalToken.secondsUntilExpiration()); + } + + @Test + void testMinimalDeserialization() throws JsonProcessingException { + var minimalFromJson = MAPPER.readValue(""" + { + "token": "mock-token" + } + """, ExternalAuthToken.class); + + assertEquals(MOCK_TOKEN, minimalFromJson.token()); + assertEquals(Long.MAX_VALUE, minimalFromJson.secondsUntilExpiration()); + } + + @Test + void testFullDeserialization() throws JsonProcessingException { + var fullFromJson = MAPPER.readValue(""" + { + "token": "mock-token", + "expires_at": "2099-12-31T23:59:59Z", + "username": "mock-username" + } + """, ExternalAuthToken.class); + + assertEquals(MOCK_TOKEN, fullFromJson.token()); + assertEquals("mock-username", fullFromJson.username()); + var dt = fullFromJson.expiresAt(); + assertNotNull(dt); + assertEquals(2099, dt.getYear()); + } + + @Test + void testFullDeserializationMillis() throws JsonProcessingException { + var fullFromJson = MAPPER.readValue(""" + { + "token": "mock-token", + "expires_at": "2099-12-31T23:59:59.123Z", + "username": "mock-username" + } + """, ExternalAuthToken.class); + + assertEquals(MOCK_TOKEN, fullFromJson.token()); + var dt = fullFromJson.expiresAt(); + assertNotNull(dt); + assertEquals(2099, dt.getYear()); + assertEquals(123, dt.getNano() / 1_000_000); + } + + @Test + void testDateSerializationSecondsToMillis() throws JsonProcessingException { + var json = MAPPER.writeValueAsString(ExternalAuthToken.SimpleToken.builder() + .token(MOCK_TOKEN) + .expiresAt(OffsetDateTime.parse("2099-12-31T23:59:59Z")) + .build()); + + assertTrue(json.contains("23:59:59.000Z")); + } + + @Test + void testDateSerializationMillis() throws JsonProcessingException { + var json = MAPPER.writeValueAsString(ExternalAuthToken.SimpleToken.builder() + .token(MOCK_TOKEN) + .expiresAt(OffsetDateTime.parse("2099-12-31T23:59:59.123Z")) + .build()); + + assertTrue(json.contains("23:59:59.123Z")); + } +} diff --git a/github-app-installation/pom.xml b/github-app-installation/pom.xml new file mode 100644 index 0000000000..d8d5e5c2fa --- /dev/null +++ b/github-app-installation/pom.xml @@ -0,0 +1,156 @@ + + + + 4.0.0 + + + com.walmartlabs.concord + parent + 2.34.1-SNAPSHOT + ../pom.xml + + + concord-github-app-installation + jar + + ${project.groupId}:${project.artifactId} + + + + com.walmartlabs.concord + concord-sdk + + + com.walmartlabs.concord + concord-common + + + io.takari.bpm + bpm-engine-impl + + + javax.validation + validation-api + provided + + + org.slf4j + slf4j-api + provided + + + com.typesafe + config + + + com.nimbusds + nimbus-jose-jwt + + + org.bouncycastle + bcprov-ext-jdk15on + 1.70 + + + org.bouncycastle + bcpkix-jdk15on + 1.70 + + + org.bouncycastle + bcprov-jdk15on + 1.70 + + + org.immutables + value + provided + + + org.immutables + builder + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + provided + + + javax.inject + javax.inject + provided + + + com.fasterxml.jackson.datatype + jackson-datatype-guava + provided + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + provided + + + + + javax.xml.bind + jaxb-api + provided + + + com.sun.xml.bind + jaxb-impl + provided + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.mockito + mockito-core + test + + + org.mockito + mockito-junit-jupiter + test + + + ch.qos.logback + logback-classic + test + + + + + + + org.eclipse.sisu + sisu-maven-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${java.io.tmpdir} + + + + + + diff --git a/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/AccessTokenProvider.java b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/AccessTokenProvider.java new file mode 100644 index 0000000000..3eac8516fd --- /dev/null +++ b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/AccessTokenProvider.java @@ -0,0 +1,200 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.nimbusds.jose.JOSEException; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.JWSHeader; +import com.nimbusds.jose.crypto.RSASSASigner; +import com.nimbusds.jose.jwk.JWK; +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.jwt.SignedJWT; +import com.walmartlabs.concord.common.ExternalAuthToken; +import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig; +import com.walmartlabs.concord.github.appinstallation.exception.GitHubAppException; +import org.immutables.value.Value; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.time.Duration; +import java.util.Date; +import java.util.function.BiFunction; + +public class AccessTokenProvider { + + private static final Logger log = LoggerFactory.getLogger(AccessTokenProvider.class); + + private final HttpClient httpClient; + private final Duration httpTimeout; + private final ObjectMapper objectMapper; + + public AccessTokenProvider(GitHubAppInstallationConfig cfg, ObjectMapper objectMapper) { + this.httpTimeout = cfg.getHttpClientTimeout(); + this.objectMapper = objectMapper; + this.httpClient = HttpClient.newBuilder() + .connectTimeout(httpTimeout) + .build(); + } + + public AccessTokenProvider(GitHubAppInstallationConfig cfg, + ObjectMapper objectMapper, + HttpClient httpClient) { + this.httpTimeout = cfg.getHttpClientTimeout(); + this.objectMapper = objectMapper; + this.httpClient = httpClient; + } + + ExternalAuthToken getRepoInstallationToken(GitHubAppAuthConfig app, String orgRepo) throws GitHubAppException { + try { + var jwt = generateJWT(app); + var accessTokenUrl = getAccessTokenUrl(app.apiUrl(), orgRepo, jwt); + return ExternalAuthToken.StaticToken.builder() + .from(createAccessToken(accessTokenUrl, jwt)) + .authId(app.id()) + .username(app.username()) + .build(); + } catch (JOSEException e) { + throw new GitHubAppException("Error generating JWT for app: " + app.clientId()); + } + } + + private String getAccessTokenUrl(String apiBaseUrl, String installationRepo, String jwt) throws GitHubAppException { + var req = HttpRequest.newBuilder().GET() + .uri(URI.create(apiBaseUrl + "/repos/" + installationRepo + "/installation")) + .header("Authorization", "Bearer " + jwt) + .header("Accept", "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28") + .timeout(httpTimeout) + .build(); + + var appInstallation = sendRequest(req, 200, AccessTokenProvider.GitHubAppInstallationResp.class, (code, body) -> { + if (code == 404) { + // not possible to discern between repo not found and app not installed for existing (private) repo + log.warn("getAccessTokenUrl ['{}'] -> not found", installationRepo); + return new GitHubAppException.NotFoundException("Repo not found or App installation not found for repo"); + } + + log.warn("getAccessTokenUrl ['{}'] -> error: {} : {}", installationRepo, code, body); + return new GitHubAppException("Unexpected error locating repo installation: " + code); + }); + + return appInstallation.accessTokensUrl(); + } + + private ExternalAuthToken createAccessToken(String accessTokenUrl, String jwt) { + var req = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.noBody()) + .uri(URI.create(accessTokenUrl)) + .header("Authorization", "Bearer " + jwt) + .header("Accept", "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28") + .timeout(httpTimeout) + .build(); + + return sendRequest(req, 201, ExternalAuthToken.class, (code, body) -> { + log.warn("createAccessToken ['{}'] -> error: {} : {}", accessTokenUrl, code, body); + + if (code == 404) { + // this would be pretty odd to hit, this means the url returned from the installation lookup is invalid + return new GitHubAppException.NotFoundException("App access token url not found"); + } + + return new GitHubAppException("Unexpected error creating app access token: " + code); + }); + } + + private static String generateJWT(GitHubAppAuthConfig auth) throws JOSEException { + var pk = auth.privateKey(); + var rsaJWK = JWK.parseFromPEMEncodedObjects(pk).toRSAKey(); + + // Create RSA-signer with the private key + var signer = new RSASSASigner(rsaJWK); + + // Prepare JWT with claims set + var claimsSet = new JWTClaimsSet.Builder() + .issueTime(new Date()) + .issuer(auth.clientId()) + // JWT expiration. GH requires less than 10 minutes + .expirationTime(new Date(new Date().getTime() + Duration.ofMinutes(10).toMillis())) + .build(); + + var signedJWT = new SignedJWT( + new JWSHeader.Builder(JWSAlgorithm.RS256) + .keyID(rsaJWK.getKeyID()) + .build(), + claimsSet); + + // Compute the RSA signature + signedJWT.sign(signer); + + // To serialize to compact form, produces something like + return signedJWT.serialize(); + } + + private T sendRequest(HttpRequest httpRequest, int expectedCode, Class clazz, BiFunction exFun) throws GitHubAppException { + try { + var resp = httpClient.send(httpRequest, BodyHandlers.ofInputStream()); + if (resp.statusCode() != expectedCode) { + throw exFun.apply(resp.statusCode(), readBody(resp)); + } + return objectMapper.readValue(resp.body(), clazz); + } catch (IOException e) { + throw new GitHubAppException("Error sending request", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + throw new IllegalStateException("Unexpected error sending HTTP request"); + } + + private static String readBody(HttpResponse resp) throws IOException { + try (var is = resp.body()) { + return new String(is.readAllBytes()); + } + } + + @Value.Immutable + @Value.Style(jdkOnly = true) + @JsonDeserialize(as = ImmutableGitHubAppInstallationResp.class) + @JsonIgnoreProperties(ignoreUnknown = true) + public interface GitHubAppInstallationResp { + + /* + This is the only attribute we **need**, even though there's other + attributes. Some may differ between GitHub "cloud" and GitHub Enterprise/private. + Be care if/when adding more. + */ + @JsonProperty("access_tokens_url") + String accessTokensUrl(); + + } +} diff --git a/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubAppAuthCacheKey.java b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubAppAuthCacheKey.java new file mode 100644 index 0000000000..d27bd9256e --- /dev/null +++ b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubAppAuthCacheKey.java @@ -0,0 +1,66 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import java.net.URI; +import java.util.Arrays; + +public record GitHubAppAuthCacheKey(URI repoUri, byte[] secretData, int weight) { + + static GitHubAppAuthCacheKey from(URI repoUri) { + return from(repoUri, null); + } + + static GitHubAppAuthCacheKey from(URI repoUri, byte[] secretData) { + var weight = 1; + + if (secretData != null) { + weight += 1; + weight += secretData.length / 1024; + } + + return new GitHubAppAuthCacheKey(repoUri, secretData, weight); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + + var key = (GitHubAppAuthCacheKey) o; + return weight() == key.weight() && repoUri().equals(key.repoUri()) && Arrays.equals(secretData(), key.secretData()); + } + + @Override + public int hashCode() { + int result = repoUri().hashCode(); + result = 31 * result + Arrays.hashCode(secretData()); + result = 31 * result + weight(); + return result; + } + + @Override + public String toString() { + return "GitHubAppAuthCacheKey{" + + "repoUri=" + repoUri + + ", weight=" + weight + + '}'; + } +} diff --git a/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubAppAuthConfig.java b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubAppAuthConfig.java new file mode 100644 index 0000000000..0e20a0f468 --- /dev/null +++ b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubAppAuthConfig.java @@ -0,0 +1,55 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; + +import java.util.regex.Pattern; + +public record GitHubAppAuthConfig(String id, + String apiUrl, + String clientId, + String privateKey, + String username, + Pattern urlPattern) implements MappingAuthConfig { + + public GitHubAppAuthConfig(String id, String apiUrl, String clientId, String privateKey, String username, Pattern urlPattern) { + if (clientId == null || clientId.isBlank()) { + throw new IllegalArgumentException("clientId must be provided"); + } + + if (privateKey == null || privateKey.isBlank()) { + throw new IllegalArgumentException("privateKey must be provided"); + } + + // sanity check url pattern before this object gets too far out there + if (!urlPattern.toString().contains("?")) { + throw new IllegalArgumentException("The url pattern must contain the ? named group"); + } + + this.id = id; + this.apiUrl = (apiUrl == null) ? "https://api.github.com" : apiUrl; + this.clientId = clientId; + this.privateKey = privateKey; + this.username = username; + this.urlPattern = urlPattern; + } +} diff --git a/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubAppInstallation.java b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubAppInstallation.java new file mode 100644 index 0000000000..fc5dc8e777 --- /dev/null +++ b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubAppInstallation.java @@ -0,0 +1,205 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.Weigher; +import com.google.common.util.concurrent.UncheckedExecutionException; +import com.walmartlabs.concord.common.AuthTokenProvider; +import com.walmartlabs.concord.common.ExternalAuthToken; +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.common.secret.BinaryDataSecret; +import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig; +import com.walmartlabs.concord.github.appinstallation.exception.GitHubAppException; +import com.walmartlabs.concord.github.appinstallation.exception.RepoExtractionException; +import com.walmartlabs.concord.sdk.Secret; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.net.URI; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +public class GitHubAppInstallation implements AuthTokenProvider { + + private static final Logger log = LoggerFactory.getLogger(com.walmartlabs.concord.github.appinstallation.GitHubAppInstallation.class); + + private final GitHubAppInstallationConfig cfg; + private final AccessTokenProvider tokenProvider; + private final ObjectMapper objectMapper; + + private final LoadingCache> cache; + + @Inject + public GitHubAppInstallation(GitHubAppInstallationConfig cfg, ObjectMapper objectMapper) { + this.cfg = cfg; + this.objectMapper = objectMapper; + this.tokenProvider = new AccessTokenProvider(cfg, objectMapper); + + this.cache = CacheBuilder.newBuilder() + .expireAfterWrite(cfg.getSystemAuthCacheDuration()) + .maximumWeight(cfg.getSystemAuthCacheMaxWeight()) + .weigher((Weigher>) (key, value) -> key.weight()) + .build(new CacheLoader<>() { + @Override + public @Nonnull Optional load(@Nonnull GitHubAppAuthCacheKey key) { + return fetchToken(key.repoUri(), key.secretData()); + } + }); + } + + @Override + public boolean supports(URI repo, @Nullable Secret secret) { + return Utils.validateSecret(secret, objectMapper) || systemSupports(repo); + } + + private GitHubAppAuthCacheKey createKey(URI repoUri, @Nullable Secret secret) { + if (secret == null) { + return GitHubAppAuthCacheKey.from(repoUri); + } + + if (secret instanceof BinaryDataSecret bds) { + return GitHubAppAuthCacheKey.from(repoUri, bds.getData()); + } + + return null; + } + + @Override + public Optional getToken(URI repo, @Nullable Secret secret) { + var cacheKey = createKey(repo, secret); + + if (cacheKey == null) { + return Optional.empty(); + } + + try { + var activeToken = cache.get(cacheKey); + + return activeToken.map(t -> refreshBeforeExpire(t, cacheKey)); + } catch (ExecutionException e) { + throw new GitHubAppException("Error retrieving access token for repo: " + repo, e); + } catch (UncheckedExecutionException e) { + // unwrap from guava + if (e.getCause() instanceof GitHubAppException repoEx) { + throw repoEx; + } + + log.warn("getAccessToken ['{}'] -> error: {}", repo, e.getMessage()); + + throw new GitHubAppException("Unexpected error retrieving access token for repo: " + repo); + } + } + + public long cacheSize() { + return cache.size(); + } + + private boolean systemSupports(URI repoUri) { + return cfg.getAuthConfigs().stream().anyMatch(auth -> auth.canHandle(repoUri)); + } + + private Optional fetchToken(URI repo, @Nullable byte[] secret) { + if (secret != null) { + return Optional.ofNullable(fromBinaryData(repo, secret)); + } + + // no secret, see if system config has something for this repo + return cfg.getAuthConfigs().stream() + .filter(auth -> auth.canHandle(repo)) + .findFirst() + .map(auth -> { + if (auth instanceof MappingAuthConfig.OauthAuthConfig tokenAuth) { + return ExternalAuthToken.StaticToken.builder() + .authId(tokenAuth.id()) + .token(tokenAuth.token()) + .username(tokenAuth.username()) + .build(); + } + + if (auth instanceof GitHubAppAuthConfig app) { + return getTokenFromAppInstall(app, repo); + } + + throw new IllegalArgumentException("Unsupported GitAuth type for repo: " + repo); + }); + } + + /** + * Cache may return a token that's close to expiring. If it's too close, + * invalidate and get a new one. If it's just a little close, refresh the + * cache in the background and return the still-active token. + */ + private ExternalAuthToken refreshBeforeExpire(@Nonnull ExternalAuthToken token, GitHubAppAuthCacheKey cacheKey) { + if (token.secondsUntilExpiration() < 10) { + // not enough time to be useful. get a new token right now + cache.invalidate(cacheKey); + try { + return cache.get(cacheKey).orElse(null); + } catch (ExecutionException e) { + throw new GitHubAppException("Error retrieving access token for repo: " + cacheKey.repoUri(), e); + } + } + + // refresh cache if the token is expiring soon, doesn't affect current token + if (token.secondsUntilExpiration() < 300) { + cache.refresh(cacheKey); + } + + return token; + } + + private ExternalAuthToken fromBinaryData(URI repo, byte[] data) { + var appInfo = Utils.parseAppInstallation(data, objectMapper); + if (appInfo.isPresent()) { + // great, it's apparently a valid app installation config + return getTokenFromAppInstall(appInfo.get(), repo); + } + + // hopefully it's just a token a plaintext token + return ExternalAuthToken.StaticToken.builder() + .token(new String(data).trim()) + .build(); + } + + private ExternalAuthToken getTokenFromAppInstall(GitHubAppAuthConfig app, URI repo) { + try { + var ownerAndRepo = Utils.extractOwnerAndRepo(app, repo); + return accessTokenProvider().getRepoInstallationToken(app, ownerAndRepo); + } catch (RepoExtractionException | GitHubAppException e) { + var msg = e.getMessage(); + log.warn("getTokenFromAppInstall ['{}', '{}'] Error retrieving GitHub access token: {}", app.apiUrl(), repo, msg); + } + + return null; + } + + AccessTokenProvider accessTokenProvider() { + return tokenProvider; + } + +} diff --git a/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubInstallationToken.java b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubInstallationToken.java new file mode 100644 index 0000000000..0833bd7fc5 --- /dev/null +++ b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/GitHubInstallationToken.java @@ -0,0 +1,33 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.ExternalAuthToken; +import org.immutables.value.Value; + +@Value.Immutable +@Value.Style(jdkOnly = true) +public interface GitHubInstallationToken extends ExternalAuthToken { + + static ImmutableGitHubInstallationToken.Builder builder() { + return ImmutableGitHubInstallationToken.builder(); + } +} diff --git a/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/Utils.java b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/Utils.java new file mode 100644 index 0000000000..014efd1fec --- /dev/null +++ b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/Utils.java @@ -0,0 +1,167 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.typesafe.config.Config; +import com.walmartlabs.concord.common.secret.BinaryDataSecret; +import com.walmartlabs.concord.github.appinstallation.exception.GitHubAppException; +import com.walmartlabs.concord.github.appinstallation.exception.RepoExtractionException; +import com.walmartlabs.concord.sdk.Secret; + +import java.net.URI; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +public class Utils { + + private static final String KEY_GITHUB_APP_INSTALLATION = "githubAppInstallation"; + + /** + * Validates given secret is usable enough to attempt a remote lookup. Not + * guaranteed to actually work, just a sanity check useful to avoid attempting + * API calls with something that will definitely not work. + */ + static boolean validateSecret(Secret secret, ObjectMapper mapper) { + // secret must be one of: + // * JSON-formatted GitHub app installation details: clientId, privateKey, urlPattern + // * single-line, plaintext access token + + if (secret == null) { + return false; + } + + if (!(secret instanceof BinaryDataSecret bds)) { + // this class is not the place for handling key pairs or username/password + return false; + } + + var base = parseRawAppInstallation(bds.getData(), mapper); + if (base == null) { + // It's not JSON, may be an oauth token + return isPrintableAscii(bds.getData()); + } else if (base.isEmpty()) { + // Doesn't match something we can parse + return false; + } + + // App installation config format is either valid or not + return parseAppInstallation(bds.getData(), mapper).isPresent(); + } + + private static boolean isPrintableAscii(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return false; + } + + for (byte b : bytes) { + // Cast byte to int to avoid issues with negative byte values + int asciiValue = b & 0xFF; // Use bitwise AND to get unsigned value + + if (asciiValue < 32 || asciiValue > 126) { + return false; + } + } + return true; + } + + static Map parseRawAppInstallation(byte[] bds, ObjectMapper mapper) { + var t = mapper.getTypeFactory().constructMapType(HashMap.class, String.class, Object.class); + + try { // find out if it's at least valid JSON. + var base = mapper.>readValue(bds, t); + if (base.containsKey(KEY_GITHUB_APP_INSTALLATION)) { + return base; + } else { + // it's JSON, but not in our format + return Map.of(); + } + } catch (Exception e) { + // invalid JSON, may be a plaintext token + return null; + } + } + + static Optional parseAppInstallation(byte[] bds, ObjectMapper mapper) { + var base = parseRawAppInstallation(bds, mapper); + + if (base == null || !base.containsKey(KEY_GITHUB_APP_INSTALLATION)) { + // it's either not JSON or not in our format + return Optional.empty(); + } + + try { // great, now convert it to the expected structure + return Optional.of(mapper.convertValue(base.get(KEY_GITHUB_APP_INSTALLATION), GitHubAppAuthConfig.class)); + } catch (IllegalArgumentException e) { + // doesn't match the expected structure + throw new GitHubAppException("Invalid app installation definition.", e); + } + } + + static String extractOwnerAndRepo(GitHubAppAuthConfig auth, URI repo) throws RepoExtractionException { + var baseUrl = getBaseUrl(auth, repo); + var relevantPath = repo.toString().replaceAll("^.*" + baseUrl + "/?", "") + .replaceAll(repo.getQuery() != null ? "\\?" + repo.getQuery() : "", "") + .replaceFirst("\\.git$", ""); + + // parse out the owner/repo from the path + var pathParts = Arrays.stream(relevantPath.split("/")) + .filter(e -> !e.isBlank()) + .limit(2) + .toList(); + + if (pathParts.size() != 2) { + throw new RepoExtractionException("Failed to parse owner and repository from path: " + repo.getPath()); + } + + return pathParts.get(0) + "/" + pathParts.get(1); + } + + private static String getBaseUrl(GitHubAppAuthConfig auth, URI repo) { + var port = (repo.getPort() == -1 ? "" : (":" + repo.getPort())); + var path = (repo.getPath() == null ? "" : repo.getPath()); + var repoHostPortAndPath = repo.getHost() + port + path; + + var match = auth.urlPattern().matcher(repoHostPortAndPath); + + if (!match.matches()) { + // at this point, this should only fail if the urlPattern is not + // constructed correctly. We wouldn't get there if the pattern didn't + // match the repo in the first place. + throw new RepoExtractionException("Failed to parse owner and repository from path: " + repo.getPath()); + } + + return match.group("baseUrl"); + } + + public static String getStringOrDefault(Config cfg, String key, Supplier defaultValueSupplier) { + if (cfg.hasPath(key)) { + return cfg.getString(key); + } + return defaultValueSupplier.get(); + } + + private Utils() {} + +} diff --git a/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/cfg/GitHubAppInstallationConfig.java b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/cfg/GitHubAppInstallationConfig.java new file mode 100644 index 0000000000..8357fbca57 --- /dev/null +++ b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/cfg/GitHubAppInstallationConfig.java @@ -0,0 +1,178 @@ +package com.walmartlabs.concord.github.appinstallation.cfg; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.typesafe.config.Config; +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.github.appinstallation.GitHubAppAuthConfig; +import com.walmartlabs.concord.github.appinstallation.exception.GitHubAppException; +import org.immutables.value.Value; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +import static com.walmartlabs.concord.github.appinstallation.Utils.getStringOrDefault; + +@Value.Immutable +@Value.Style(jdkOnly = true) +public interface GitHubAppInstallationConfig { + + List getAuthConfigs(); + + @Value.Default + default Duration getSystemAuthCacheDuration() { + return Duration.ofMinutes(50); + } + + @Value.Default + default Duration getHttpClientTimeout() { + return Duration.ofSeconds(30); + } + + /** + * Weight is roughly calculated in kilobytes. Any cached item will have a + * minimum weight of 1. While further weight calculations are based on size + * of the given secret, if any. + *

+ * The default of 10,240 (~10MB) can hold approximately: + *

    + *
  • 10,000 tokens with no secret
    • + *
  • 5,000 tokens with string or credentials secret
    • + *
  • 3,500 tokens with app private key secret
    • + *
+ */ + @Value.Default + default long getSystemAuthCacheMaxWeight() { + return 1024 * 10L; + } + + static ImmutableGitHubAppInstallationConfig.Builder builder() { + return ImmutableGitHubAppInstallationConfig.builder(); + } + + static GitHubAppInstallationConfig fromConfig(Config config) { + var auths = config.getConfigList("auth").stream() + .map(GitHubAppInstallationConfig::toGitAuth) + .toList(); + + var builder = builder(); + + if (config.hasPath("httpClientTimeout")) { + builder.httpClientTimeout(config.getDuration("httpClientTimeout")); + } + + if (config.hasPath("systemAuthCacheDuration")) { + builder.systemAuthCacheDuration(config.getDuration("systemAuthCacheDuration")); + } + + if (config.hasPath("systemAuthCacheMaxWeight")) { + builder.systemAuthCacheMaxWeight(config.getInt("systemAuthCacheMaxWeight")); + } + + return builder + .authConfigs(auths) + .build(); + } + + enum AuthSource { + OAUTH_TOKEN, + GITHUB_APP_INSTALLATION, + } + + interface AuthConfig { + MappingAuthConfig toGitAuth(); + } + + static MappingAuthConfig toGitAuth(com.typesafe.config.Config auth) { + var a = switch (AuthSource.valueOf(auth.getString("type").toUpperCase())) { + case OAUTH_TOKEN -> OauthConfig.from(auth); + case GITHUB_APP_INSTALLATION -> AppInstallationConfig.from(auth); + }; + return a.toGitAuth(); + } + + record OauthConfig(String id, String urlPattern, String username, String token) implements AuthConfig { + + static OauthConfig from(com.typesafe.config.Config cfg) { + return new OauthConfig( + getStringOrDefault(cfg, "id", () -> "github-oauth-token"), + cfg.getString("urlPattern"), + getStringOrDefault(cfg, "username", () -> null), + cfg.getString("token") + ); + } + + @Override + public MappingAuthConfig toGitAuth() { + return MappingAuthConfig.OauthAuthConfig.builder() + .id(this.id()) + .token(this.token()) + .username(this.username()) + .urlPattern(MappingAuthConfig.assertBaseUrlPattern(this.urlPattern())) + .build(); + } + } + + record AppInstallationConfig(String id, String urlPattern, String username, String apiUrl, String clientId, String privateKey) implements AuthConfig { + + static AppInstallationConfig from(com.typesafe.config.Config cfg) { + + var username = Optional.ofNullable(getStringOrDefault(cfg, "username", () -> null)) + .filter(s -> !s.isBlank()) + .orElse("x-access-token"); + + var apiUrl = Optional.ofNullable(getStringOrDefault(cfg, "apiUrl", () -> null)) + .filter(s -> !s.isBlank()) + .orElse("https://api.github.com"); + + return new AppInstallationConfig( + getStringOrDefault(cfg, "id", () -> "github-app-installation"), + cfg.getString("urlPattern"), + username, + apiUrl, + cfg.getString("clientId"), + cfg.getString("privateKey") + ); + } + + @Override + public MappingAuthConfig toGitAuth() { + try { + var pkData = Files.readString(Paths.get(this.privateKey())); + + return new GitHubAppAuthConfig( + this.id(), + this.apiUrl(), + this.clientId(), + pkData, + this.username(), + MappingAuthConfig.assertBaseUrlPattern(this.urlPattern()) + ); + } catch (IOException e) { + throw new GitHubAppException("Error initializing Git App Installation auth", e); + } + } + } +} diff --git a/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/exception/GitHubAppException.java b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/exception/GitHubAppException.java new file mode 100644 index 0000000000..f883cfd06e --- /dev/null +++ b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/exception/GitHubAppException.java @@ -0,0 +1,41 @@ +package com.walmartlabs.concord.github.appinstallation.exception; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +public class GitHubAppException extends RuntimeException { + public GitHubAppException(String message) { + super(message); + } + + public GitHubAppException(String message, Throwable cause) { + super(message, cause); + } + + public static class NotFoundException extends GitHubAppException { + public NotFoundException(String message) { + super(message); + } + + public NotFoundException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/exception/RepoExtractionException.java b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/exception/RepoExtractionException.java new file mode 100644 index 0000000000..b004a6af94 --- /dev/null +++ b/github-app-installation/src/main/java/com/walmartlabs/concord/github/appinstallation/exception/RepoExtractionException.java @@ -0,0 +1,28 @@ +package com.walmartlabs.concord.github.appinstallation.exception; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +public class RepoExtractionException extends IllegalArgumentException { + + public RepoExtractionException(String s) { + super(s); + } +} diff --git a/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/AccessTokenProviderTest.java b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/AccessTokenProviderTest.java new file mode 100644 index 0000000000..d53458debc --- /dev/null +++ b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/AccessTokenProviderTest.java @@ -0,0 +1,145 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig; +import com.walmartlabs.concord.github.appinstallation.exception.GitHubAppException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.List; + +import static com.walmartlabs.concord.github.appinstallation.TestConstants.PRIVATE_KEY_TEXT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class AccessTokenProviderTest { + + @Mock + HttpClient httpClient; + + @Mock + HttpResponse tokenUrlResponse; + + @Mock + HttpResponse accessTokenResponse; + + private static final GitHubAppAuthConfig auth = new GitHubAppAuthConfig("test-auth", null, "123", PRIVATE_KEY_TEXT, null, MappingAuthConfig.assertBaseUrlPattern("(?github.local)/")); + + private static final GitHubAppInstallationConfig CFG = GitHubAppInstallationConfig.builder() + .authConfigs(List.of(auth)) + .build(); + + @Test + void test() throws Exception { + when(httpClient.send(any(), any(HttpResponse.BodyHandler.class))) + .thenReturn(tokenUrlResponse, accessTokenResponse); + + when(tokenUrlResponse.statusCode()).thenReturn(200); + when(tokenUrlResponse.body()).thenReturn(asInputStream(ACCESS_TOKEN_URL_RESPONSE)); + + when(accessTokenResponse.statusCode()).thenReturn(201); + when(accessTokenResponse.body()).thenReturn(asInputStream(ACCESS_TOKEN_RESPONSE)); + + var provider = new AccessTokenProvider(CFG, TestConstants.MAPPPER, httpClient); + + // -- + + var result = provider.getRepoInstallationToken(auth, "owner/repo"); + + // -- + + assertNotNull(result); + assertEquals("mock-token", result.token()); + assertTrue(result.secondsUntilExpiration() > 300); + } + + @Test + void testAppNotInstalled() throws Exception { + when(httpClient.send(any(), any(HttpResponse.BodyHandler.class))) + .thenReturn(tokenUrlResponse); + + when(tokenUrlResponse.statusCode()).thenReturn(404); + when(tokenUrlResponse.body()).thenReturn(asInputStream("App is not installed on repo")); + + var provider = new AccessTokenProvider(CFG, TestConstants.MAPPPER, httpClient); + + // -- + + var ex = assertThrows(GitHubAppException.NotFoundException.class, + () -> provider.getRepoInstallationToken(auth, "owenr/repo")); + + // -- + + assertTrue(ex.getMessage().contains("Repo not found or App installation not found for repo")); + } + + @Test + void testErrorCreatingToken() throws Exception { + when(httpClient.send(any(), any(HttpResponse.BodyHandler.class))) + .thenReturn(tokenUrlResponse, accessTokenResponse); + + when(tokenUrlResponse.statusCode()).thenReturn(200); + when(tokenUrlResponse.body()).thenReturn(asInputStream(ACCESS_TOKEN_URL_RESPONSE)); + + when(accessTokenResponse.statusCode()).thenReturn(500); + when(accessTokenResponse.body()).thenReturn(asInputStream("server error")); + + var provider = new AccessTokenProvider(CFG, TestConstants.MAPPPER, httpClient); + + // -- + + var ex = assertThrows(GitHubAppException.class, + () -> provider.getRepoInstallationToken(auth, "owenr/repo")); + + // -- + + assertTrue(ex.getMessage().contains("Unexpected error creating app access token: 500")); + } + + private static final String ACCESS_TOKEN_URL_RESPONSE = """ + { + "access_tokens_url": "https://github.local/access_tokens" + }"""; + + private static final String ACCESS_TOKEN_RESPONSE = """ + { + "token": "mock-token", + "expires_at": "2099-12-31T23:59:59Z" + }"""; + + private static InputStream asInputStream(String s) { + return new ByteArrayInputStream(s.getBytes()); + } + +} diff --git a/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/GitHubAppAuthConfigTest.java b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/GitHubAppAuthConfigTest.java new file mode 100644 index 0000000000..f647cab864 --- /dev/null +++ b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/GitHubAppAuthConfigTest.java @@ -0,0 +1,44 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class GitHubAppAuthConfigTest { + + @Test + void testUrlPatternMissingNamedGroup() { + var ex = assertThrows(IllegalArgumentException.class, () -> new GitHubAppAuthConfig( + "test-auth", + "https://api.github.com", + "mock-client-id", + "/not/used/in/test", + null, + MappingAuthConfig.assertBaseUrlPattern(".*") + )); + + assertTrue(ex.getMessage().contains("The url pattern must contain the ? named group")); + } +} diff --git a/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/GitHubAppInstallationTest.java b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/GitHubAppInstallationTest.java new file mode 100644 index 0000000000..1a549b923a --- /dev/null +++ b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/GitHubAppInstallationTest.java @@ -0,0 +1,238 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.walmartlabs.concord.common.ExternalAuthToken; +import com.walmartlabs.concord.common.ObjectMapperProvider; +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.common.secret.BinaryDataSecret; +import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig; +import com.walmartlabs.concord.sdk.Secret; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.List; + +import static com.walmartlabs.concord.github.appinstallation.TestConstants.APP_INSTALL_CONTENT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class GitHubAppInstallationTest { + + private static final ObjectMapper MAPPER = new ObjectMapperProvider().get(); + + @Mock + AccessTokenProvider accessTokenProvider; + + private static final URI VALID_APP_AUTH_URI_01 = URI.create("https://github.local/owner/repo-1.git"); + private static final URI VALID_APP_AUTH_URI_02 = URI.create("https://github.local/owner/repo-2.git"); + private static final URI VALID_STATIC_AUTH_URI_01 = URI.create("https://staticgithub.local/owner/repo-1.git"); + + private static final GitHubAppAuthConfig auth = new GitHubAppAuthConfig( + "test-app-auth", + null, + "123", + "/does/not/exist", + null, + MappingAuthConfig.assertBaseUrlPattern("(?github.local)/")); + private static final MappingAuthConfig.OauthAuthConfig staticKeyAuth = MappingAuthConfig.OauthAuthConfig.builder() + .id("test-static-auth") + .urlPattern(MappingAuthConfig.assertBaseUrlPattern("(?staticgithub.local)/")) + .token("static-token") + .build(); + private final GitHubAppInstallationConfig CFG = GitHubAppInstallationConfig.builder() + .authConfigs(List.of(staticKeyAuth, auth)) + .build(); + private static final ExternalAuthToken TOKEN_1HR = ExternalAuthToken.SimpleToken.builder() + .token("mock-installation-token") + .expiresAt(OffsetDateTime.now().plusHours(1)) + .build(); + private static final ExternalAuthToken TOKEN_200S = ExternalAuthToken.SimpleToken.builder() + .token("mock-200-seconds-expiration-token") + .expiresAt(OffsetDateTime.now().plusSeconds(200)) + .build(); + private static final ExternalAuthToken TOKEN_9S = ExternalAuthToken.SimpleToken.builder() + .token("mock-9-seconds-expiration-token") + .expiresAt(OffsetDateTime.now().plusSeconds(9)) + .build(); + private static final Secret MOCK_STATIC_SECRET = new BinaryDataSecret(staticKeyAuth.token().getBytes(StandardCharsets.UTF_8)); + + private static final Secret MOCK_APP_INSTALL_SECRET = new BinaryDataSecret(APP_INSTALL_CONTENT.getBytes(StandardCharsets.UTF_8)); + + @Test + void testCache() { + // -- prepare 0 + var app = new TestApp(CFG, MAPPER, accessTokenProvider); + + // -- validate 0 + // empty when nothing was ever looked up + assertEquals(0, app.cacheSize()); + + // -- prepare 1 + when(accessTokenProvider.getRepoInstallationToken(any(), any())) + .thenReturn(TOKEN_1HR); + + // -- execute 1 + var tokenResp = app.getToken(VALID_APP_AUTH_URI_01, null); + + // -- verify 1 + assertTrue(tokenResp.isPresent()); + assertEquals(TOKEN_1HR, tokenResp.get()); + assertEquals(1, app.cacheSize()); + verify(accessTokenProvider, times(1)) + .getRepoInstallationToken(any(), any()); + + // -- execute 2 + // should hit cache + app.getToken(VALID_APP_AUTH_URI_01, null); + app.getToken(VALID_APP_AUTH_URI_01, null); + + // -- verify 2 + verify(accessTokenProvider, times(1)) + .getRepoInstallationToken(any(), any()); + + // -- execute 3 + // Different repo, will retrieve first and hit cache second + app.getToken(VALID_APP_AUTH_URI_02, null); + app.getToken(VALID_APP_AUTH_URI_02, null); + + // -- verify 3 + // cache now has 2 entries + verify(accessTokenProvider, times(2)) + .getRepoInstallationToken(any(), any()); + } + + @Test + void testRefreshBeforeExpire() { + // -- prepare 0 + when(accessTokenProvider.getRepoInstallationToken(any(), any())) + .thenReturn(TOKEN_200S, TOKEN_1HR); + var app = new TestApp(CFG, MAPPER, accessTokenProvider); + + // -- execute 1 + var tokenResp = app.getToken(VALID_APP_AUTH_URI_01, null); + + // -- verify 1 + // first is usable so we receive that token, new token is refreshed in background + assertTrue(tokenResp.isPresent()); + assertEquals(TOKEN_200S, tokenResp.get()); + assertEquals(1, app.cacheSize()); + // + verify(accessTokenProvider, times(2)) + .getRepoInstallationToken(any(), any()); + } + + @Test + void testForceRefreshBeforeExpire() { + // -- prepare 0 + when(accessTokenProvider.getRepoInstallationToken(any(), any())) + .thenReturn(TOKEN_9S, TOKEN_1HR); + var app = new TestApp(CFG, MAPPER, accessTokenProvider); + + // -- execute 1 + var tokenResp = app.getToken(VALID_APP_AUTH_URI_01, null); + + // -- verify 1 + assertTrue(tokenResp.isPresent()); + assertEquals(TOKEN_1HR, tokenResp.get()); + assertEquals(1, app.cacheSize()); + // first is usable refresh is forced we get the new token + verify(accessTokenProvider, times(2)) + .getRepoInstallationToken(any(), any()); + } + + @Test + void testFromStaticTokenTokenConfig() { + // -- prepare 0 + var app = new TestApp(CFG, MAPPER, accessTokenProvider); + + // -- execute 1 + var tokenResp = app.getToken(VALID_STATIC_AUTH_URI_01, null); + + // -- verify 1 + assertTrue(tokenResp.isPresent()); + assertEquals(staticKeyAuth.token(), tokenResp.get().token()); + assertEquals(1, app.cacheSize()); + // not an app installation, no lookup expected + verify(accessTokenProvider, times(0)) + .getRepoInstallationToken(any(), any()); + } + + @Test + void testFromStaticTokenSecret() { + // -- prepare 0 + var app = new TestApp(CFG, MAPPER, accessTokenProvider); + + // -- execute 1 + var tokenResp = app.getToken(VALID_APP_AUTH_URI_01, MOCK_STATIC_SECRET); + + // -- verify 1 + assertTrue(tokenResp.isPresent()); + assertEquals(staticKeyAuth.token(), tokenResp.get().token()); + assertEquals(1, app.cacheSize()); + // not an app installation, no lookup expected + verify(accessTokenProvider, times(0)) + .getRepoInstallationToken(any(), any()); + } + + @Test + void testFromAppInstallSecret() { + // -- prepare 0 + when(accessTokenProvider.getRepoInstallationToken(any(), any())) + .thenReturn(TOKEN_1HR); + var app = new TestApp(CFG, MAPPER, accessTokenProvider); + + // -- execute 1 + var tokenResp = app.getToken(VALID_APP_AUTH_URI_01, MOCK_APP_INSTALL_SECRET); + + // -- verify 1 + assertTrue(tokenResp.isPresent()); + assertEquals(TOKEN_1HR, tokenResp.get()); + assertEquals(1, app.cacheSize()); + verify(accessTokenProvider, times(1)) + .getRepoInstallationToken(any(), any()); + } + + private static class TestApp extends GitHubAppInstallation { + private final AccessTokenProvider tokenProvider; + + public TestApp(GitHubAppInstallationConfig cfg, ObjectMapper objectMapper, AccessTokenProvider tokenProvider) { + super(cfg, objectMapper); + this.tokenProvider = tokenProvider; + } + + @Override + AccessTokenProvider accessTokenProvider() { + return tokenProvider; + } + } +} diff --git a/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/RepoExtractionTest.java b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/RepoExtractionTest.java new file mode 100644 index 0000000000..2f3a2511d1 --- /dev/null +++ b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/RepoExtractionTest.java @@ -0,0 +1,120 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.net.URI; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class RepoExtractionTest { + + static Stream validPublicUris() { + return Stream.of( + "https://github.com/owner/repo.git", // typicalRepo + "https://github.com/owner/repo", // no trailing '.git', should still work + "https://github.com/owner/repo/", // ...same with trailing slash + // with query params. not very typical, but does work in this pattern matching + "https://github.com/owner/repo.git?hello=world", + "https://github.com/owner/repo?hello=world", + "https://github.com/owner/repo/?hello=world" + ).map(URI::create); + } + + static Stream invalidPublicUris() { + return Stream.of( + "https://github.com/owner", // no repo + "https://github.com/owner/" // no repo with slash + ).map(URI::create); + } + + static Stream validProxiedUris() { + return Stream.of( + "https://git.company.local/proxypath/owner/repo.git", // typicalRepo + "https://git.company.local/proxypath/owner/repo", // no trailing '.git', should still work + "https://git.company.local/proxypath/owner/repo/", // ...same with trailing slash + // with query params. not very typical, but does work in this pattern matching + "https://git.company.local/proxypath/owner/repo.git?hello=world", + "https://git.company.local/proxypath/owner/repo?hello=world", + "https://git.company.local/proxypath/owner/repo/?hello=world" + ).map(URI::create); + } + + static Stream invalidProxiedUris() { + return Stream.of( + "https://git.company.local/proxypath/owner", // no repo + "https://git.company.local/proxypath/owner/" // no repo with slash + ).map(URI::create); + } + + @ParameterizedTest + @MethodSource("validPublicUris") + void testValidPublicUris(URI repo) { + var ownerAndRepo = assertDoesNotThrow(() -> runExtract("(?github.com).*", repo)); + assertEquals("owner/repo", ownerAndRepo); + } + + @ParameterizedTest + @MethodSource("invalidPublicUris") + void testInvalidPublicUris(URI repo) { + var ex = assertThrows(IllegalArgumentException.class, + () -> runExtract("(?github.com).*", repo)); + assertTrue(ex.getMessage().contains("Failed to parse owner and repository from path")); + } + + @ParameterizedTest + @MethodSource("validProxiedUris") + void testValidProxiedUris(URI repo) { + var ownerAndRepo = assertDoesNotThrow(() -> runExtract("(?git.company.local/proxypath).*", repo)); + assertEquals("owner/repo", ownerAndRepo); + } + + @ParameterizedTest + @MethodSource("invalidProxiedUris") + void testInvalidProxiedUris(URI repo) { + var ex = assertThrows(IllegalArgumentException.class, + () -> runExtract("(?git.company.local/proxypath).*", repo)); + assertTrue(ex.getMessage().contains("Failed to parse owner and repository from path")); + } + + private static String runExtract(String pattern, URI repo) { + var auth = getAuth(pattern); + return Utils.extractOwnerAndRepo(auth, repo); + } + + private static GitHubAppAuthConfig getAuth(String urlPattern) { + return new GitHubAppAuthConfig( + "test-auth", + "https://api.github.com", + "1234", + "/not/used", + null, + MappingAuthConfig.assertBaseUrlPattern(urlPattern) + ); + } +} diff --git a/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/TestConstants.java b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/TestConstants.java new file mode 100644 index 0000000000..9d54dea56f --- /dev/null +++ b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/TestConstants.java @@ -0,0 +1,66 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.walmartlabs.concord.common.ObjectMapperProvider; +import com.walmartlabs.concord.common.secret.BinaryDataSecret; +import com.walmartlabs.concord.sdk.Secret; + +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; + +public class TestConstants { + static final ObjectMapper MAPPPER = new ObjectMapperProvider().get(); + static final String APP_INSTALL_CONTENT = """ + { + "githubAppInstallation": { + "urlPattern": "(?github.local)/.*", + "clientId": "123", + "privateKey": "mock-key-data" + } + }"""; + static final Secret MOCK_APP_INSTALL_SECRET = new BinaryDataSecret(APP_INSTALL_CONTENT.getBytes(StandardCharsets.UTF_8)); + static final Secret MOCK_STATIC_TOKEN_SECRET = new BinaryDataSecret("mock-static-token".getBytes(StandardCharsets.UTF_8)); + + public static final String PRIVATE_KEY_TEXT = generatePrivateKey(); + + private static String generatePrivateKey() { + KeyPairGenerator kpg; + try { + kpg = KeyPairGenerator.getInstance("RSA"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Algorithm not found", e); + } + + kpg.initialize(2048); + KeyPair kp = kpg.generateKeyPair(); + + Base64.Encoder encoder = Base64.getEncoder(); + + return "-----BEGIN PRIVATE KEY-----\n" + + encoder.encodeToString(kp.getPrivate().getEncoded()) + + "\n-----END PRIVATE KEY-----\n"; + } +} diff --git a/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/UtilsTest.java b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/UtilsTest.java new file mode 100644 index 0000000000..62355c8c0c --- /dev/null +++ b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/UtilsTest.java @@ -0,0 +1,105 @@ +package com.walmartlabs.concord.github.appinstallation; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.common.secret.BinaryDataSecret; +import com.walmartlabs.concord.common.secret.UsernamePassword; +import com.walmartlabs.concord.github.appinstallation.exception.GitHubAppException; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import static com.walmartlabs.concord.github.appinstallation.TestConstants.APP_INSTALL_CONTENT; +import static com.walmartlabs.concord.github.appinstallation.TestConstants.MAPPPER; +import static com.walmartlabs.concord.github.appinstallation.TestConstants.MOCK_APP_INSTALL_SECRET; +import static com.walmartlabs.concord.github.appinstallation.TestConstants.MOCK_STATIC_TOKEN_SECRET; +import static com.walmartlabs.concord.github.appinstallation.Utils.parseAppInstallation; +import static com.walmartlabs.concord.github.appinstallation.Utils.parseRawAppInstallation; +import static com.walmartlabs.concord.github.appinstallation.Utils.validateSecret; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class UtilsTest { + + @Test + void testValidateSecret() { + assertTrue(validateSecret(MOCK_APP_INSTALL_SECRET, MAPPPER)); + assertTrue(validateSecret(MOCK_STATIC_TOKEN_SECRET, MAPPPER)); + assertFalse(validateSecret(null, MAPPPER)); + assertFalse(validateSecret(Mockito.mock(UsernamePassword.class), MAPPPER)); + assertFalse(validateSecret(new BinaryDataSecret(null), MAPPPER)); + assertFalse(validateSecret(new BinaryDataSecret(new byte[]{}), MAPPPER)); + assertFalse(validateSecret(new BinaryDataSecret("\nmytoken".getBytes()), MAPPPER)); + assertFalse(validateSecret(new BinaryDataSecret("{\"hello\":\"world\"}".getBytes()), MAPPPER)); + } + + @Test + void parseAppInstallation_ValidJson() { + var o = parseAppInstallation(APP_INSTALL_CONTENT.getBytes(), MAPPPER); + + assertTrue(o.isPresent()); + var result = o.get(); + assertEquals("123", result.clientId()); + } + + @Test + void parseAppInstallation_MissingElement() { + var missingClientId = """ + { + "githubAppInstallation": { + "urlPattern": "(?github.local)/.*", + "privateKey": "mock-key-data" + } + }"""; + var data = missingClientId.getBytes(); + var ex = assertThrows(GitHubAppException.class, () -> parseAppInstallation(data, MAPPPER)); + + assertTrue(ex.getMessage().contains("Invalid app installation definition")); + } + + @Test + void parseAppInstallation_OtherJson() { + var unexpectedJson = "{ \"valid\": \"but not usable here\"}"; + var result = parseAppInstallation(unexpectedJson.getBytes(), MAPPPER); + + assertFalse(result.isPresent()); + } + + @Test + void parseRawAppInstallation_NotJson() { + var unexpectedJson = "justText"; + var result = parseRawAppInstallation(unexpectedJson.getBytes(), MAPPPER); + + assertNull(result); + } + + @Test + void parseRawAppInstallation_OtherJson() { + var unexpectedJson = "{ \"valid\": \"but not usable here\"}"; + var result = parseRawAppInstallation(unexpectedJson.getBytes(), MAPPPER); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } +} diff --git a/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/cfg/ConfigTest.java b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/cfg/ConfigTest.java new file mode 100644 index 0000000000..1992b0742c --- /dev/null +++ b/github-app-installation/src/test/java/com/walmartlabs/concord/github/appinstallation/cfg/ConfigTest.java @@ -0,0 +1,102 @@ +package com.walmartlabs.concord.github.appinstallation.cfg; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.typesafe.config.ConfigFactory; +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.github.appinstallation.GitHubAppAuthConfig; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; + +import static com.walmartlabs.concord.github.appinstallation.TestConstants.PRIVATE_KEY_TEXT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +class ConfigTest { + + @TempDir + private static Path workDir; + + @Test + void simpleConfig() throws Exception { + var pk = Files.writeString(workDir.resolve("pk.pem"), PRIVATE_KEY_TEXT); + var typesafeConfig = ConfigFactory.parseString(""" + { + "auth" = [ + { id = "app-auth", type = "GITHUB_APP_INSTALLATION", urlPattern = "(?github.local)", clientId = "123", privateKey = "{{PK_PATH}}" }, + { id = "static-auth", type = "OAUTH_TOKEN", urlPattern = "(?github.local)", token = "mock-token" } + ] + }""".replace("{{PK_PATH}}", pk.toString())); + var cfg = GitHubAppInstallationConfig.fromConfig(typesafeConfig); + assertNotNull(cfg); + assertEquals(10240, cfg.getSystemAuthCacheMaxWeight()); + assertEquals(Duration.ofSeconds(30), cfg.getHttpClientTimeout()); + assertEquals(Duration.ofMinutes(50), cfg.getSystemAuthCacheDuration()); + assertEquals(2, cfg.getAuthConfigs().size()); + + var appInstall = assertInstanceOf(GitHubAppAuthConfig.class, cfg.getAuthConfigs().get(0)); + assertEquals("app-auth", appInstall.id()); + assertEquals("x-access-token", appInstall.username()); + assertEquals("https://api.github.com", appInstall.apiUrl()); + + var oauth = assertInstanceOf(MappingAuthConfig.OauthAuthConfig.class, cfg.getAuthConfigs().get(1)); + assertEquals("static-auth", oauth.id()); + assertNull(oauth.username()); + assertEquals("mock-token", oauth.token()); + } + + @Test + void overrideConfig() throws Exception { + var pk = Files.writeString(workDir.resolve("pk.pem"), PRIVATE_KEY_TEXT); + var typesafeConfig = ConfigFactory.parseString(""" + { + httpClientTimeout = "1 minute", + systemAuthCacheDuration = "1 minute", + systemAuthCacheMaxWeight = "10" + "auth" = [ + { id = "app-auth", type = "GITHUB_APP_INSTALLATION", urlPattern = "(?github.local)", username = "custom", apiUrl = "https://api.github.local", clientId = "123", privateKey = "{{PK_PATH}}" }, + { id = "static-auth", type = "OAUTH_TOKEN", urlPattern = "(?github.local)", token = "mock-token", username = "custom" } + ] + }""".replace("{{PK_PATH}}", pk.toString())); + var cfg = GitHubAppInstallationConfig.fromConfig(typesafeConfig); + assertNotNull(cfg); + assertEquals(10, cfg.getSystemAuthCacheMaxWeight()); + assertEquals(Duration.ofMinutes(1), cfg.getHttpClientTimeout()); + assertEquals(Duration.ofMinutes(1), cfg.getSystemAuthCacheDuration()); + assertEquals(2, cfg.getAuthConfigs().size()); + + var appInstall = assertInstanceOf(GitHubAppAuthConfig.class, cfg.getAuthConfigs().get(0)); + assertEquals("app-auth", appInstall.id()); + assertEquals("custom", appInstall.username()); + assertEquals("https://api.github.local", appInstall.apiUrl()); + + var oauth = assertInstanceOf(MappingAuthConfig.OauthAuthConfig.class, cfg.getAuthConfigs().get(1)); + assertEquals("static-auth", oauth.id()); + assertEquals("custom", oauth.username()); + assertEquals("mock-token", oauth.token()); + } +} diff --git a/github-app-installation/src/test/resources/logback-test.xml b/github-app-installation/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..00003ef25f --- /dev/null +++ b/github-app-installation/src/test/resources/logback-test.xml @@ -0,0 +1,13 @@ + + + + %d{HH:mm:ss.SSS} [%thread] [%-5level] %logger{36} - %msg%n + + + + + + + + + diff --git a/pom.xml b/pom.xml index 3abb373f2a..9aaf161bcc 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,7 @@ common policy-engine dependency-manager + github-app-installation repository imports forms diff --git a/repository/pom.xml b/repository/pom.xml index 812f380ed1..5bc9e982b4 100644 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -85,5 +85,15 @@ org.eclipse.jgit test + + org.mockito + mockito-core + test + + + org.mockito + mockito-junit-jupiter + test + diff --git a/repository/src/main/java/com/walmartlabs/concord/repository/FetchRequest.java b/repository/src/main/java/com/walmartlabs/concord/repository/FetchRequest.java index 7ab3812867..9e2534a4ea 100644 --- a/repository/src/main/java/com/walmartlabs/concord/repository/FetchRequest.java +++ b/repository/src/main/java/com/walmartlabs/concord/repository/FetchRequest.java @@ -41,8 +41,10 @@ public interface FetchRequest { Path destination(); /** - * Concord secret for authentication. - * If not provided {@link GitClientConfiguration#oauthToken()} is used for {@code https://} URLs. + * Concord secret for authentication. If not provided, an available + * {@link com.walmartlabs.concord.common.AuthTokenProvider} matching + * {@link #url()} will be used for {@code https://} URLs. Otherwise, + * anonymous auth is attempted. */ @Nullable Secret secret(); diff --git a/repository/src/main/java/com/walmartlabs/concord/repository/GitCliRepositoryProvider.java b/repository/src/main/java/com/walmartlabs/concord/repository/GitCliRepositoryProvider.java index 2ab61cd447..99b9de7947 100644 --- a/repository/src/main/java/com/walmartlabs/concord/repository/GitCliRepositoryProvider.java +++ b/repository/src/main/java/com/walmartlabs/concord/repository/GitCliRepositoryProvider.java @@ -20,6 +20,7 @@ * ===== */ +import com.walmartlabs.concord.common.AuthTokenProvider; import com.walmartlabs.concord.common.PathUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,8 +38,8 @@ public class GitCliRepositoryProvider implements RepositoryProvider { private final GitClient client; - public GitCliRepositoryProvider(GitClientConfiguration cfg) { - this.client = new GitClient(cfg); + public GitCliRepositoryProvider(GitClientConfiguration cfg, AuthTokenProvider authProvider) { + this.client = new GitClient(cfg, authProvider); } @Override diff --git a/repository/src/main/java/com/walmartlabs/concord/repository/GitClient.java b/repository/src/main/java/com/walmartlabs/concord/repository/GitClient.java index f9642b9c3d..cf0bcc030e 100644 --- a/repository/src/main/java/com/walmartlabs/concord/repository/GitClient.java +++ b/repository/src/main/java/com/walmartlabs/concord/repository/GitClient.java @@ -20,7 +20,7 @@ * ===== */ -import com.google.common.collect.ImmutableSet; +import com.walmartlabs.concord.common.AuthTokenProvider; import com.walmartlabs.concord.common.PathUtils; import com.walmartlabs.concord.common.secret.BinaryDataSecret; import com.walmartlabs.concord.common.secret.KeyPair; @@ -54,14 +54,22 @@ public class GitClient { private static final int SUCCESS_EXIT_CODE = 0; private final GitClientConfiguration cfg; - - private final List sensitiveData; + private final AuthTokenProvider authProvider; + private final Set sensitiveData; private final ExecutorService executor; - public GitClient(GitClientConfiguration cfg) { + public GitClient(GitClientConfiguration cfg, AuthTokenProvider authProvider) { this.cfg = cfg; - this.sensitiveData = cfg.oauthToken() != null ? Collections.singletonList(cfg.oauthToken()) : Collections.emptyList(); + this.authProvider = authProvider; this.executor = Executors.newCachedThreadPool(); + this.sensitiveData = new LinkedHashSet<>(); + + // urls with user info. + sensitiveData.add(new Obfuscation("https://([^@]*)@", "https://***@")); + + cfg.oauthToken().ifPresent(oauth -> + // definitely don't print a hard-code oauth token + sensitiveData.add(new Obfuscation(oauth, "***"))); } public FetchResult fetch(FetchRequest req) { @@ -310,23 +318,22 @@ String updateUrl(String url, Secret secret) { URI uri = assertUriAllowed(url); - List allowedDefaultHosts = cfg.authorizedGitHosts(); - - if(secret == null && allowedDefaultHosts!= null && !allowedDefaultHosts.contains(uri.getHost())) { - // in this case the user has not provided authentication AND the host is not in the whitelist of hosts - // which may use the default git credentials. return the url un-modified to attempt anonymous auth; - // if it fails + if (url.contains("@") || !url.startsWith("https://")) { + // provided url already has credentials OR it's a non-https url return url; } - if (secret != null || cfg.oauthToken() == null || url.contains("@") || !url.startsWith("https://")) { - // provided url already has credentials OR there are no default credentials to use. - // anonymous auth is the only viable option. - return url; + if (secret instanceof UsernamePassword up) { + // Secret contains static auth (token or username). No lookup needed. + return "https://" + + (up.getUsername() == null ? "" : up.getUsername() + ":") + + String.valueOf(up.getPassword()) + + "@" + + url.substring("https://".length()); } - // using default credentials - return "https://" + cfg.oauthToken() + "@" + url.substring("https://".length()); + // This will either add auth from a matching provider, or none for anonymous access + return authProvider.addUserInfoToUri(uri, secret).toString(); } private URI assertUriAllowed(String rawUri) { @@ -554,9 +561,7 @@ private String execWithCredentials(Command cmd, Secret secret) { env.put("GIT_TERMINAL_PROMPT", "0"); try { - if (secret instanceof KeyPair) { - KeyPair keyPair = (KeyPair) secret; - + if (secret instanceof KeyPair keyPair) { key = createSshKeyFile(keyPair); ssh = createUnixGitSSH(key); @@ -569,24 +574,15 @@ private String execWithCredentials(Command cmd, Secret secret) { } log.info("using GIT_SSH to set credentials"); - } else if (secret instanceof UsernamePassword) { - UsernamePassword userPass = (UsernamePassword) secret; - + } else if (secret instanceof UsernamePassword userPass) { askpass = createUnixStandardAskpass(userPass); env.put("GIT_ASKPASS", askpass.toAbsolutePath().toString()); env.put("SSH_ASKPASS", askpass.toAbsolutePath().toString()); - log.info("using GIT_ASKPASS to set credentials "); - } else if (secret instanceof BinaryDataSecret) { - BinaryDataSecret token = (BinaryDataSecret) secret; - - askpass = createUnixStandardAskpass(new UsernamePassword(new String(token.getData()), "".toCharArray())); - - env.put("GIT_ASKPASS", askpass.toAbsolutePath().toString()); - log.info("using GIT_ASKPASS to set credentials "); } + // if secret is single-value, it was already applied to the URL in updateUrl() env.put("GIT_HTTP_LOW_SPEED_LIMIT", String.valueOf(cfg.httpLowSpeedLimit())); env.put("GIT_HTTP_LOW_SPEED_TIME", String.valueOf(cfg.httpLowSpeedTime().getSeconds())); @@ -608,8 +604,8 @@ private String hideSensitiveData(String s) { return null; } - for (String p : sensitiveData) { - s = s.replaceAll(p, "***"); + for (Obfuscation o : sensitiveData) { + s = s.replaceAll(o.pattern(), o.replacement()); } return s; } @@ -628,7 +624,7 @@ private Path createUnixGitSSH(Path key) throws IOException { " -o ServerAliveInterval=" + cfg.sshTimeout().getSeconds() + " -o StrictHostKeyChecking=no \"$@\""); } - Files.setPosixFilePermissions(ssh, ImmutableSet.of(OWNER_READ, OWNER_EXECUTE)); + Files.setPosixFilePermissions(ssh, Set.of(OWNER_READ, OWNER_EXECUTE)); return ssh; } @@ -647,7 +643,7 @@ private static Path createUnixStandardAskpass(UsernamePassword creds) throws IOE w.println("Password*) echo '" + quoteUnixCredentials(new String(creds.getPassword())) + "' ;;"); w.println("esac"); } - Files.setPosixFilePermissions(askpass, ImmutableSet.of(OWNER_READ, OWNER_EXECUTE)); + Files.setPosixFilePermissions(askpass, Set.of(OWNER_READ, OWNER_EXECUTE)); return askpass; } @@ -783,4 +779,8 @@ static ImmutableCommitInfo.Builder builder() { return ImmutableCommitInfo.builder(); } } + + private record Obfuscation(String pattern, + String replacement) { + } } diff --git a/repository/src/main/java/com/walmartlabs/concord/repository/GitClientConfiguration.java b/repository/src/main/java/com/walmartlabs/concord/repository/GitClientConfiguration.java index 122dd3467e..d99362713a 100644 --- a/repository/src/main/java/com/walmartlabs/concord/repository/GitClientConfiguration.java +++ b/repository/src/main/java/com/walmartlabs/concord/repository/GitClientConfiguration.java @@ -25,17 +25,18 @@ import javax.annotation.Nullable; import java.time.Duration; import java.util.List; +import java.util.Optional; import java.util.Set; @Value.Immutable @Value.Style(jdkOnly = true) public interface GitClientConfiguration { - @Nullable - String oauthToken(); - - @Nullable - List authorizedGitHosts(); + Optional oauthToken(); + + Optional oauthUsername(); + + Optional oauthUrlPattern(); @Value.Default default Set allowedSchemes() { diff --git a/repository/src/main/java/com/walmartlabs/concord/repository/RepositoryException.java b/repository/src/main/java/com/walmartlabs/concord/repository/RepositoryException.java index fecc45e3e0..9b277a4631 100644 --- a/repository/src/main/java/com/walmartlabs/concord/repository/RepositoryException.java +++ b/repository/src/main/java/com/walmartlabs/concord/repository/RepositoryException.java @@ -21,8 +21,11 @@ */ +import java.io.Serial; + public class RepositoryException extends RuntimeException { + @Serial private static final long serialVersionUID = 1L; public RepositoryException(String message) { diff --git a/repository/src/test/java/com/walmartlabs/concord/repository/GitClientFetch2Test.java b/repository/src/test/java/com/walmartlabs/concord/repository/GitClientFetch2Test.java index fbd22cd0f2..36eda84716 100644 --- a/repository/src/test/java/com/walmartlabs/concord/repository/GitClientFetch2Test.java +++ b/repository/src/test/java/com/walmartlabs/concord/repository/GitClientFetch2Test.java @@ -20,11 +20,15 @@ * ===== */ +import com.walmartlabs.concord.common.AuthTokenProvider; import com.walmartlabs.concord.common.PathUtils; import com.walmartlabs.concord.common.TemporaryPath; import org.eclipse.jgit.revwalk.RevCommit; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; import java.nio.file.Files; @@ -38,10 +42,14 @@ /** * test for checkout prev commitId on master branch + checkAlreadyFetched=true */ +@ExtendWith(MockitoExtension.class) public class GitClientFetch2Test { private GitClient client; + @Mock + AuthTokenProvider authProvider; + @BeforeEach public void init() { client = new GitClient(GitClientConfiguration.builder() @@ -49,7 +57,7 @@ public void init() { .sshTimeoutRetryCount(1) .httpLowSpeedLimit(1) .httpLowSpeedTime(Duration.ofMinutes(10)) - .build()); + .build(), authProvider); } @Test diff --git a/repository/src/test/java/com/walmartlabs/concord/repository/GitClientFetchTest.java b/repository/src/test/java/com/walmartlabs/concord/repository/GitClientFetchTest.java index 7c608fed05..f7ef97e666 100644 --- a/repository/src/test/java/com/walmartlabs/concord/repository/GitClientFetchTest.java +++ b/repository/src/test/java/com/walmartlabs/concord/repository/GitClientFetchTest.java @@ -20,6 +20,7 @@ * ===== */ +import com.walmartlabs.concord.common.AuthTokenProvider; import com.walmartlabs.concord.common.PathUtils; import com.walmartlabs.concord.common.TemporaryPath; import com.walmartlabs.concord.sdk.Secret; @@ -28,6 +29,9 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; import java.nio.file.Files; @@ -40,10 +44,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +@ExtendWith(MockitoExtension.class) public class GitClientFetchTest { private GitClient client; + @Mock + AuthTokenProvider authProvider; + @BeforeEach public void init() { client = new GitClient(GitClientConfiguration.builder() @@ -52,7 +60,7 @@ public void init() { .sshTimeoutRetryCount(1) .httpLowSpeedLimit(1) .httpLowSpeedTime(Duration.ofMinutes(10)) - .build()); + .build(), authProvider); } @Test diff --git a/repository/src/test/java/com/walmartlabs/concord/repository/GitClientRealTest.java b/repository/src/test/java/com/walmartlabs/concord/repository/GitClientRealTest.java index 5f717fc010..72c440a669 100644 --- a/repository/src/test/java/com/walmartlabs/concord/repository/GitClientRealTest.java +++ b/repository/src/test/java/com/walmartlabs/concord/repository/GitClientRealTest.java @@ -20,6 +20,7 @@ * ===== */ +import com.walmartlabs.concord.common.AuthTokenProvider; import com.walmartlabs.concord.common.PathUtils; import com.walmartlabs.concord.common.TemporaryPath; import com.walmartlabs.concord.common.secret.KeyPair; @@ -28,6 +29,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; import java.nio.file.Files; @@ -38,6 +42,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; @Disabled +@ExtendWith(MockitoExtension.class) public class GitClientRealTest { private static final String HTTPS_REPO_URL = System.getenv("HTTPS_REPO_URL"); @@ -61,6 +66,9 @@ private static Secret createKeypair() { private GitClient client; + @Mock + AuthTokenProvider authProvider; + @BeforeEach public void init() { client = new GitClient(GitClientConfiguration.builder() @@ -69,7 +77,7 @@ public void init() { .sshTimeoutRetryCount(1) .httpLowSpeedLimit(1) .httpLowSpeedTime(Duration.ofMinutes(10)) - .build()); + .build(), authProvider); } @Test diff --git a/repository/src/test/java/com/walmartlabs/concord/repository/GitClientSpeedTest.java b/repository/src/test/java/com/walmartlabs/concord/repository/GitClientSpeedTest.java index dc906b98ec..7b100bd575 100644 --- a/repository/src/test/java/com/walmartlabs/concord/repository/GitClientSpeedTest.java +++ b/repository/src/test/java/com/walmartlabs/concord/repository/GitClientSpeedTest.java @@ -20,6 +20,7 @@ * ===== */ +import com.walmartlabs.concord.common.AuthTokenProvider; import com.walmartlabs.concord.common.PathUtils; import com.walmartlabs.concord.common.TemporaryPath; import com.walmartlabs.concord.sdk.Secret; @@ -27,6 +28,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; import java.nio.file.Files; @@ -54,10 +58,14 @@ * Require internet connection */ @Disabled +@ExtendWith(MockitoExtension.class) public class GitClientSpeedTest { private GitClient client; + @Mock + AuthTokenProvider authProvider; + @BeforeEach public void init() { client = new GitClient(GitClientConfiguration.builder() @@ -65,7 +73,7 @@ public void init() { .sshTimeoutRetryCount(1) .httpLowSpeedLimit(1) .httpLowSpeedTime(Duration.ofMinutes(10)) - .build()); + .build(), authProvider); } @Test diff --git a/repository/src/test/java/com/walmartlabs/concord/repository/GitUriTest.java b/repository/src/test/java/com/walmartlabs/concord/repository/GitUriTest.java index 933560689a..86847144e4 100644 --- a/repository/src/test/java/com/walmartlabs/concord/repository/GitUriTest.java +++ b/repository/src/test/java/com/walmartlabs/concord/repository/GitUriTest.java @@ -20,19 +20,24 @@ * ===== */ +import com.walmartlabs.concord.common.AuthTokenProvider; +import com.walmartlabs.concord.common.cfg.OauthTokenConfig; import com.walmartlabs.concord.common.secret.BinaryDataSecret; import com.walmartlabs.concord.sdk.Secret; +import org.immutables.value.Value; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; class GitUriTest { + private static final AuthTokenProvider AUTH_PROVIDER = authProvider(null); + private static final AuthTokenProvider RESTRICTED_AUTH_PROVIDER = authProvider("gitserver.local"); private static final GitClientConfiguration cfg = GitClientConfiguration.builder() .oauthToken("mock-token") .build(); - private static final GitClient client = new GitClient(cfg); - private static Secret secret = new BinaryDataSecret(new byte[0]); + private static final GitClient client = new GitClient(cfg, AUTH_PROVIDER); + private static Secret secret = new BinaryDataSecret("secret-mock-token".getBytes()); @Test void testSsh() { @@ -53,7 +58,7 @@ void testHttps() { @Test void testHttpWithSecret() { var httpsSecret = client.updateUrl("https://gitserver.local/my-org/my-repo.git", secret); - assertEquals("https://gitserver.local/my-org/my-repo.git", httpsSecret); + assertEquals("https://secret-mock-token@gitserver.local/my-org/my-repo.git", httpsSecret); } @Test @@ -72,8 +77,7 @@ void testUnrestrictedHost() { void testGitHostRestriction() { var restrictedClient = new GitClient(GitClientConfiguration.builder() .from(cfg) - .addAuthorizedGitHosts("gitserver.local") - .build()); + .build(), RESTRICTED_AUTH_PROVIDER); var anonAuth = restrictedClient.updateUrl("https://elsewhere.local/my-org/my-repo.git", null); // unchanged @@ -84,4 +88,22 @@ void testGitHostRestriction() { assertEquals("https://mock-token@gitserver.local/my-org/my-repo.git", url2); } + private static AuthTokenProvider authProvider(String urlPattern) { + var builder = TestOauthTokenConfig.builder() + .oauthToken("mock-token"); + + if (urlPattern != null) { + builder.oauthUrlPattern(urlPattern); + } + + return new AuthTokenProvider.OauthTokenProvider(builder.build()); + } + + @Value.Immutable + @Value.Style(jdkOnly = true) + interface TestOauthTokenConfig extends OauthTokenConfig { + static ImmutableTestOauthTokenConfig.Builder builder() { + return ImmutableTestOauthTokenConfig.builder(); + } + } } diff --git a/server/dist/src/main/resources/concord-server.conf b/server/dist/src/main/resources/concord-server.conf index f4fba570db..7eb36ae610 100644 --- a/server/dist/src/main/resources/concord-server.conf +++ b/server/dist/src/main/resources/concord-server.conf @@ -463,9 +463,29 @@ concord-server { # git clone config git { - # GitHub auth token to use when cloning repositories without explicitly configured authentication + allowedSchemes = [ "https", "ssh", "classpath" ] + + # GitHub auth token to use when cloning repositories without explicitly + # configured authentication. Deprecated in favor of systemAuth list of + # tokens or service-specific app config (e.g. github) # oauth = "..." + # specific username to use for auth + # oauthUsername = "" + + # regex to match against git server's hostname + port + path so oauth + # token isn't used for and unexpected host + # oauthUrlPattern = "" + + # List of system-provided auth token configs + # { + # type = "OAUTH_TOKEN", + # token = "...", + # username = "...", # optional, username to send with auth token + # urlPattern = "..." # required, regex to match against target git host + port + path + # } + systemAuth = [] + # use GIT's shallow clone shallowClone = true @@ -487,7 +507,7 @@ concord-server { sshTimeout = "10 minutes" } - # GitHub webhook integration + # GitHub app and webhook integration github { # default value, for testing only secret = "12345" @@ -500,6 +520,45 @@ concord-server { # disable concord repos on push event with deleted ref (branch, tag) disableReposOnDeletedRef = false + + # App installation settings. Multiple auth (private key) definitions are supported, + # as each is matched to a particular url pattern. + appInstallation { + # Importantly, urlPattern must include a regex capture group named 'baseUrl'. + # This is necessary to detect where the owner/repo values begin in the path. + # { + # id = "my-gh-app", # identifier for the config, e.g. for metrics + # type = "GITHUB_APP_INSTALLATION", + # urlPattern = "(?github.com)", # regex + # username = "...", # optional, defaults to "x-access-token" + # apiUrl = "https://api.github.com", # github API url, usually *not* the same as the repo url host/path + # clientId = "...", + # privateKey = "/path/to/pk.pem" + # } + # or static oauth config. Not exactly a "GitHub App", but can do some + # API interactions and cloning. Actual app installation is preferred. + # { + # id = "my-static-token", # identifier for the config, e.g. for metrics + # type = "OAUTH_TOKEN", + # token = "...", + # username = "...", # optional, usually not necessary + # urlPattern = "..." # regex to match against git server's hostname + port + path + # apiUrl = "https://api.github.com", # github API url, usually *not* the same as the repo url host/path + # } + auth = [] + + # Timeout duration for http calls from the app + httpClientTimeout = "30 seconds" + + # Duration of time to keep tokens cached after creation. May be purged + # earlier depending on overall cache weight and usage. GitHub installation + # tokens expire after 1 hour + systemAuthCacheDuration = "50 minutes" + + # Max "weight" of the token cache. Defaults to ~10MB which should hold + # between 3,500 and 10,000 tokens depending on associated secret size. + systemAuthCacheMaxWeight = 10240 + } } # Ansible event processor configuration diff --git a/server/impl/pom.xml b/server/impl/pom.xml index 0576fb3afb..b0ca0fc98d 100644 --- a/server/impl/pom.xml +++ b/server/impl/pom.xml @@ -102,6 +102,10 @@ com.walmartlabs.concord concord-common + + com.walmartlabs.concord + concord-github-app-installation + io.takari.bpm bpm-engine-api diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/ConcordServerModule.java b/server/impl/src/main/java/com/walmartlabs/concord/server/ConcordServerModule.java index b542e17542..8179ae6346 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/ConcordServerModule.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/ConcordServerModule.java @@ -24,6 +24,7 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.typesafe.config.Config; +import com.walmartlabs.concord.common.AuthTokenProvider; import com.walmartlabs.concord.common.ObjectMapperProvider; import com.walmartlabs.concord.config.ConfigModule; import com.walmartlabs.concord.db.DatabaseModule; @@ -42,6 +43,7 @@ import com.walmartlabs.concord.server.policy.PolicyModule; import com.walmartlabs.concord.server.process.ProcessModule; import com.walmartlabs.concord.server.repository.RepositoryModule; +import com.walmartlabs.concord.server.repository.ServerAuthTokenProvider; import com.walmartlabs.concord.server.role.RoleModule; import com.walmartlabs.concord.server.security.SecurityModule; import com.walmartlabs.concord.server.security.apikey.ApiKeyModule; @@ -89,6 +91,8 @@ public void configure(Binder binder) { binder.bind(Listeners.class).in(SINGLETON); binder.bind(SecureRandom.class).toProvider(SecureRandomProvider.class); + binder.bind(AuthTokenProvider.class).to(ServerAuthTokenProvider.class).in(SINGLETON); + binder.bind(MessageChannelManager.class).in(SINGLETON); binder.bind(DependencyManagerConfiguration.class).toProvider(DependencyManagerConfigurationProvider.class); diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/ConfigurationModule.java b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/ConfigurationModule.java index 14894dd55a..a4e9fa58f5 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/ConfigurationModule.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/ConfigurationModule.java @@ -23,7 +23,9 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.typesafe.config.Config; +import com.walmartlabs.concord.common.cfg.OauthTokenConfig; import com.walmartlabs.concord.config.ConfigModule; +import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig; import static com.google.inject.Scopes.SINGLETON; @@ -52,7 +54,9 @@ public void configure(Binder binder) { binder.bind(EnqueueWorkersConfiguration.class).in(SINGLETON); binder.bind(ExternalEventsConfiguration.class).in(SINGLETON); binder.bind(GitConfiguration.class).in(SINGLETON); + binder.bind(OauthTokenConfig.class).to(GitConfiguration.class).in(SINGLETON); binder.bind(GithubConfiguration.class).in(SINGLETON); + binder.bind(GitHubAppInstallationConfig.class).to(GithubConfiguration.class).in(SINGLETON); binder.bind(ImportConfiguration.class).in(SINGLETON); binder.bind(LdapConfiguration.class).in(SINGLETON); binder.bind(LdapGroupSyncConfiguration.class).in(SINGLETON); diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/GitConfiguration.java b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/GitConfiguration.java index 07bfd08971..1558b256c5 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/GitConfiguration.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/GitConfiguration.java @@ -20,6 +20,8 @@ * ===== */ +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; +import com.walmartlabs.concord.common.cfg.OauthTokenConfig; import com.walmartlabs.concord.config.Config; import org.eclipse.sisu.Nullable; @@ -27,8 +29,11 @@ import java.io.Serializable; import java.time.Duration; import java.util.List; +import java.util.Optional; -public class GitConfiguration implements Serializable { +import static com.walmartlabs.concord.server.cfg.Utils.getStringOrDefault; + +public class GitConfiguration implements OauthTokenConfig, Serializable { private static final long serialVersionUID = 1L; @@ -38,9 +43,14 @@ public class GitConfiguration implements Serializable { private String oauthToken; @Inject - @Config("git.authorizedGitHosts") + @Config("git.oauthUsername") + @Nullable + private String oauthUsername; + + @Inject + @Config("git.oauthUrlPattern") @Nullable - private List authorizedGitHosts; + private String oauthUrlPattern; @Inject @Config("git.shallowClone") @@ -74,6 +84,10 @@ public class GitConfiguration implements Serializable { @Config("git.sshTimeoutRetryCount") private int sshTimeoutRetryCount; + @Inject + @Config("git.systemAuth") + List authConfigs; + public boolean isShallowClone() { return shallowClone; } @@ -90,12 +104,19 @@ public Duration getFetchTimeout() { return fetchTimeout; } - public String getOauthToken() { - return oauthToken; + @Override + public Optional getOauthToken() { + return Optional.ofNullable(oauthToken); + } + + @Override + public Optional getOauthUsername() { + return Optional.ofNullable(oauthUsername); } - public List getAuthorizedGitHosts() { - return authorizedGitHosts; + @Override + public Optional getOauthUrlPattern() { + return Optional.ofNullable(oauthUrlPattern); } public int getHttpLowSpeedLimit() { @@ -113,4 +134,45 @@ public Duration getSshTimeout() { public int getSshTimeoutRetryCount() { return sshTimeoutRetryCount; } + + public List getSystemAuth() { + return authConfigs.stream() + .map(o -> { + AuthSource type = AuthSource.valueOf(o.getString("type").toUpperCase()); + + return (AuthConfig) switch (type) { + case OAUTH_TOKEN -> OauthConfig.from(o); + }; + }) + .map(AuthConfig::toGitAuth) + .toList(); + } + + enum AuthSource { + OAUTH_TOKEN + } + + public interface AuthConfig { + MappingAuthConfig toGitAuth(); + } + + public record OauthConfig(String id, String urlPattern, String token) implements AuthConfig { + + static OauthConfig from(com.typesafe.config.Config cfg) { + return new OauthConfig( + getStringOrDefault(cfg, "id", () -> "system-oauth-token"), + cfg.getString("urlPattern"), + cfg.getString("token") + ); + } + + @Override + public MappingAuthConfig.OauthAuthConfig toGitAuth() { + return MappingAuthConfig.OauthAuthConfig.builder() + .id(this.id()) + .urlPattern(MappingAuthConfig.assertBaseUrlPattern(this.urlPattern())) + .token(this.token()) + .build(); + } + } } diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/GithubConfiguration.java b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/GithubConfiguration.java index d606e3699b..3b948e2c76 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/GithubConfiguration.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/GithubConfiguration.java @@ -20,12 +20,16 @@ * ===== */ +import com.walmartlabs.concord.common.cfg.MappingAuthConfig; import com.walmartlabs.concord.config.Config; +import com.walmartlabs.concord.github.appinstallation.cfg.GitHubAppInstallationConfig; import org.eclipse.sisu.Nullable; import javax.inject.Inject; +import java.time.Duration; +import java.util.List; -public class GithubConfiguration { +public class GithubConfiguration implements GitHubAppInstallationConfig { @Inject @Config("github.secret") @@ -44,6 +48,18 @@ public class GithubConfiguration { @Config("github.disableReposOnDeletedRef") private boolean disableReposOnDeletedRef; + private final GitHubAppInstallationConfig appInstallation; + + @Inject + public GithubConfiguration(com.typesafe.config.Config config) { + if (config.hasPath("github.appInstallation")) { + var raw = config.getConfig("github.appInstallation"); + this.appInstallation = GitHubAppInstallationConfig.fromConfig(raw); + } else { + this.appInstallation = GitHubAppInstallationConfig.builder().authConfigs(List.of()).build(); + } + } + public String getSecret() { return secret; } @@ -59,4 +75,25 @@ public boolean isLogEvents() { public boolean isDisableReposOnDeletedRef() { return disableReposOnDeletedRef; } + + @Override + public List getAuthConfigs() { + return appInstallation.getAuthConfigs(); + } + + @Override + public Duration getHttpClientTimeout() { + return appInstallation.getHttpClientTimeout(); + } + + @Override + public Duration getSystemAuthCacheDuration() { + return appInstallation.getSystemAuthCacheDuration(); + } + + @Override + public long getSystemAuthCacheMaxWeight() { + return appInstallation.getSystemAuthCacheMaxWeight(); + } + } diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/Utils.java b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/Utils.java index e4c38d243d..25fa287aa3 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/Utils.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/Utils.java @@ -20,11 +20,13 @@ * ===== */ +import com.typesafe.config.Config; import com.walmartlabs.concord.common.PathUtils; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.function.Supplier; public final class Utils { @@ -35,6 +37,13 @@ public static Path getPath(String s, String defaultPrefix) throws IOException { return Paths.get(s); } + public static String getStringOrDefault(Config cfg, String key, Supplier defaultValueSupplier) { + if (cfg.hasPath(key)) { + return cfg.getString(key); + } + return defaultValueSupplier.get(); + } + private Utils() { } } diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/ProcessModule.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/ProcessModule.java index b3247d744d..62615395d9 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/ProcessModule.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/ProcessModule.java @@ -23,7 +23,6 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.walmartlabs.concord.imports.ImportManager; -import com.walmartlabs.concord.process.loader.DelegatingProjectLoader; import com.walmartlabs.concord.process.loader.ProjectLoader; import com.walmartlabs.concord.runtime.v1.ProjectLoaderV1; import com.walmartlabs.concord.runtime.v2.ProjectLoaderV2; diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/repository/RepositoryManager.java b/server/impl/src/main/java/com/walmartlabs/concord/server/repository/RepositoryManager.java index c50f8c5145..8fa80f4303 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/repository/RepositoryManager.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/repository/RepositoryManager.java @@ -21,6 +21,7 @@ */ import com.fasterxml.jackson.databind.ObjectMapper; +import com.walmartlabs.concord.common.AuthTokenProvider; import com.walmartlabs.concord.common.PathUtils; import com.walmartlabs.concord.dependencymanager.DependencyManager; import com.walmartlabs.concord.repository.*; @@ -38,7 +39,6 @@ import javax.inject.Inject; import java.io.IOException; import java.nio.file.Path; -import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.UUID; @@ -63,11 +63,10 @@ public RepositoryManager(ObjectMapper objectMapper, RepositoryConfiguration repoCfg, ProjectDao projectDao, SecretManager secretManager, - DependencyManager dependencyManager) throws IOException { + DependencyManager dependencyManager, + AuthTokenProvider authProvider) throws IOException { GitClientConfiguration gitCliCfg = GitClientConfiguration.builder() - .oauthToken(gitCfg.getOauthToken()) - .authorizedGitHosts(gitCfg.getAuthorizedGitHosts()) .defaultOperationTimeout(gitCfg.getDefaultOperationTimeout()) .fetchTimeout(gitCfg.getFetchTimeout()) .httpLowSpeedLimit(gitCfg.getHttpLowSpeedLimit()) @@ -76,10 +75,13 @@ public RepositoryManager(ObjectMapper objectMapper, .sshTimeoutRetryCount(gitCfg.getSshTimeoutRetryCount()) .build(); - List providers = Arrays.asList(new ClasspathRepositoryProvider(), new MavenRepositoryProvider(dependencyManager), new GitCliRepositoryProvider(gitCliCfg)); - this.gitCfg = gitCfg; - this.providers = new RepositoryProviders(providers); + this.providers = + new RepositoryProviders(List.of( + new ClasspathRepositoryProvider(), + new MavenRepositoryProvider(dependencyManager), + new GitCliRepositoryProvider(gitCliCfg, authProvider) + )); this.secretManager = secretManager; this.projectDao = projectDao; this.repoCfg = repoCfg; diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/repository/ServerAuthTokenProvider.java b/server/impl/src/main/java/com/walmartlabs/concord/server/repository/ServerAuthTokenProvider.java new file mode 100644 index 0000000000..08e4df662f --- /dev/null +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/repository/ServerAuthTokenProvider.java @@ -0,0 +1,70 @@ +package com.walmartlabs.concord.server.repository; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.codahale.metrics.MetricRegistry; +import com.walmartlabs.concord.common.AuthTokenProvider; +import com.walmartlabs.concord.common.ExternalAuthToken; +import com.walmartlabs.concord.github.appinstallation.GitHubAppInstallation; +import com.walmartlabs.concord.sdk.Secret; +import com.walmartlabs.concord.server.sdk.metrics.WithTimer; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; +import java.net.URI; +import java.util.List; +import java.util.Optional; + +@Named +@Singleton +public class ServerAuthTokenProvider implements AuthTokenProvider { + + private final List authTokenProviders; + + @Inject + public ServerAuthTokenProvider(GitHubAppInstallation githubProvider, + OauthTokenProvider oauthTokenProvider, + MetricRegistry metricRegistry) { + this.authTokenProviders = List.of(githubProvider, oauthTokenProvider); + + metricRegistry.gauge("github-token-cache-size", () -> githubProvider::cacheSize); + } + + @Override + public boolean supports(URI repo, @Nullable Secret secret) { + return authTokenProviders.stream() + .anyMatch(p -> p.supports(repo, secret)); + } + + @WithTimer + public Optional getToken(URI repo, @Nullable Secret secret) { + for (var tokenProvider : authTokenProviders) { + if (tokenProvider.supports(repo, secret)) { + return tokenProvider.getToken(repo, secret); + } + } + + return Optional.empty(); + } + +} diff --git a/server/impl/src/test/java/com/walmartlabs/concord/server/repository/ServerAuthTokenProviderTest.java b/server/impl/src/test/java/com/walmartlabs/concord/server/repository/ServerAuthTokenProviderTest.java new file mode 100644 index 0000000000..e0f46ca2f1 --- /dev/null +++ b/server/impl/src/test/java/com/walmartlabs/concord/server/repository/ServerAuthTokenProviderTest.java @@ -0,0 +1,109 @@ +package com.walmartlabs.concord.server.repository; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.codahale.metrics.MetricRegistry; +import com.walmartlabs.concord.common.AuthTokenProvider; +import com.walmartlabs.concord.common.ExternalAuthToken; +import com.walmartlabs.concord.github.appinstallation.GitHubAppInstallation; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.net.URI; +import java.time.OffsetDateTime; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ServerAuthTokenProviderTest { + + @Mock + private MetricRegistry metricRegistry; + + @Mock + GitHubAppInstallation ghApp; + + @Mock + AuthTokenProvider.OauthTokenProvider oauthTokenProvider; + + @Test + void testGitHubApp() { + when(ghApp.getToken(any(), any())). + thenReturn(Optional.of(ExternalAuthToken.SimpleToken.builder() + .token("gh-installation-token") + .expiresAt(OffsetDateTime.now().plusMinutes(60)) + .build())); + when(ghApp.supports(any(), any())).thenReturn(true); + + var provider = new ServerAuthTokenProvider(ghApp, oauthTokenProvider, metricRegistry); + + // -- + + assertTrue(provider.supports(URI.create("https://github.local/owner/repo.git"), null)); + var o = provider.getToken(URI.create("https://github.local/owner/repo.git"), null); + + // -- + + assertTrue(o.isPresent()); + var result = assertInstanceOf(ExternalAuthToken.class, o.get()); + assertEquals("gh-installation-token", result.token()); + } + + @Test + void testOauth() { + when(oauthTokenProvider.supports(any(), any())).thenReturn(true); + when(oauthTokenProvider.getToken(any(), any())) + .thenReturn(Optional.of(ExternalAuthToken.StaticToken.builder() + .token("oauth-token") + .build())); + + var provider = new ServerAuthTokenProvider(ghApp, oauthTokenProvider, metricRegistry); + + assertTrue(provider.supports(URI.create("https://github.local/owner/repo.git"), null)); + var o = provider.getToken(URI.create("https://github.local/owner/repo.git"), null); + + assertTrue(o.isPresent()); + var result = assertInstanceOf(ExternalAuthToken.class, o.get()); + assertEquals("oauth-token", result.token()); + } + + @Test + void testNoAuth() { + when(ghApp.supports(any(), any())).thenReturn(false); + when(oauthTokenProvider.supports(any(), any())).thenReturn(false); + + var provider = new ServerAuthTokenProvider(ghApp, oauthTokenProvider, metricRegistry); + + assertFalse(provider.supports(URI.create("https://github.local/owner/repo.git"), null)); + var o = provider.getToken(URI.create("https://github.local/owner/repo.git"), null); + + assertFalse(o.isPresent()); + } + +} diff --git a/targetplatform/pom.xml b/targetplatform/pom.xml index db427168b0..9435ea9360 100644 --- a/targetplatform/pom.xml +++ b/targetplatform/pom.xml @@ -147,6 +147,11 @@ concord-common ${project.version} + + com.walmartlabs.concord + concord-github-app-installation + ${project.version} + com.walmartlabs.concord concord-server