Skip to content

feat: add ability to configure tes error cache expire time #1751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rauthsuj merged 10 commits into from
Dec 16, 2025
Merged

feat: add ability to configure tes error cache expire time #1751

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7a35e70
feat: add ability to configure tes error cache expire time
rauthsuj Nov 20, 2025
6cd26a4
refactor: move config topic to tes class and rename for better user u…
rauthsuj Nov 21, 2025
732599d
refactor: find or default instead of lookup and only one subscribe to…
rauthsuj Nov 25, 2025
4698da7
refactor: when cache error is less than 10 tes should be broken
rauthsuj Nov 26, 2025
6ff27d3
refactor: code refactoring and do not restart tes when error cache is…
rauthsuj Nov 29, 2025
e9ec86c
refactor: moved config subscribe to startup and minor code changes
rauthsuj Dec 4, 2025
d047db0
refactor: do not brake tes on invalid cache and moved subscribe to co…
rauthsuj Dec 10, 2025
d36ab58
refactor: change the name of time before cache expire to have default…
rauthsuj Dec 10, 2025
eabcc78
refactor: added volatile to guarantee an updated value is used after …
rauthsuj Dec 12, 2025
1f80692
Merge branch 'main' into tes-configure
rauthsuj Dec 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 38 additions & 12 deletions src/main/java/com/aws/greengrass/tes/CredentialRequestHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,14 @@ public class CredentialRequestHandler implements HttpHandler {
public static final String AUTH_HEADER = "Authorization";
public static final String IOT_CREDENTIALS_HTTP_VERB = "GET";
public static final String SUPPORTED_REQUEST_VERB = "GET";
public static final int TIME_BEFORE_CACHE_EXPIRE_IN_MIN = 5;
public static final int CLOUD_4XX_ERROR_CACHE_IN_MIN = 2;
public static final int CLOUD_5XX_ERROR_CACHE_IN_MIN = 1;
public static final int UNKNOWN_ERROR_CACHE_IN_MIN = 5;
public static final int TIME_BEFORE_CACHE_EXPIRE_IN_SEC = 300;
public static final int CLOUD_4XX_ERROR_CACHE_IN_SEC = 120;
Copy link
Member

@aws-kevinrickard aws-kevinrickard Dec 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: DEFAULT_CLOUD_4XX_ERROR_CACHE_IN_SEC , etc.

public static final int CLOUD_5XX_ERROR_CACHE_IN_SEC = 60;
public static final int UNKNOWN_ERROR_CACHE_IN_SEC = 300;

private int cloud4xxErrorCacheInSec = CLOUD_4XX_ERROR_CACHE_IN_SEC;
private int cloud5xxErrorCacheInSec = CLOUD_5XX_ERROR_CACHE_IN_SEC;
private int unknownErrorCacheInSec = UNKNOWN_ERROR_CACHE_IN_SEC;

private String iotCredentialsPath;

Expand Down Expand Up @@ -142,6 +146,28 @@ void setIotCredentialsPath(String iotRoleAlias) {
this.iotCredentialsPath = "/role-aliases/" + iotRoleAlias + "/credentials";
}

/**
* Configure error cache settings for error responses.
*
* @param cloud4xxErrorCache error cache duration in seconds for 4xx errors.
* @param cloud5xxErrorCache error cache duration in seconds for 5xx errors.
* @param unknownErrorCache error cache duration in seconds for unknown errors.
*/
public void configureCacheSettings(int cloud4xxErrorCache, int cloud5xxErrorCache, int unknownErrorCache) {
this.cloud4xxErrorCacheInSec = cloud4xxErrorCache;
this.cloud5xxErrorCacheInSec = cloud5xxErrorCache;
this.unknownErrorCacheInSec = unknownErrorCache;
}

/**
* Get current error cache configuration settings.
*
* @return Array containing error cache durations: [4xx, 5xx, unknown] in seconds.
*/
public int[] getErrorCacheConfigSettings() {
return new int[]{this.cloud4xxErrorCacheInSec, this.cloud5xxErrorCacheInSec, this.unknownErrorCacheInSec};
}

@Override
@SuppressWarnings("PMD.AvoidCatchingThrowable")
public void handle(final HttpExchange exchange) throws IOException {
Expand Down Expand Up @@ -281,14 +307,14 @@ private byte[] getCredentialsBypassCache() {
LOGGER.atError().kv(IOT_CRED_PATH_KEY, iotCredentialsPath)
.log("Unable to cache expired credentials which expired at {}", expiry);
} else {
newExpiry = expiry.minus(Duration.ofMinutes(TIME_BEFORE_CACHE_EXPIRE_IN_MIN));
newExpiry = expiry.minus(Duration.ofSeconds(TIME_BEFORE_CACHE_EXPIRE_IN_SEC));
tesCache.get(iotCredentialsPath).responseCode = HttpURLConnection.HTTP_OK;

if (newExpiry.isBefore(Instant.now(clock))) {
LOGGER.atWarn().kv(IOT_CRED_PATH_KEY, iotCredentialsPath)
.log("Can't cache credentials as new credentials {} will "
+ "expire in less than {} minutes", expiry,
TIME_BEFORE_CACHE_EXPIRE_IN_MIN);
+ "expire in less than {} seconds", expiry,
TIME_BEFORE_CACHE_EXPIRE_IN_SEC);
} else {
LOGGER.atInfo().kv(IOT_CRED_PATH_KEY, iotCredentialsPath)
.log("Received IAM credentials that will be cached until {}", newExpiry);
Expand Down Expand Up @@ -318,7 +344,7 @@ private byte[] getCredentialsBypassCache() {
String responseString = "Failed to get connection";
response = responseString.getBytes(StandardCharsets.UTF_8);
// Use unknown error cache policy for SSL/TLS connection errors to prevent excessive retries
newExpiry = Instant.now(clock).plus(Duration.ofMinutes(UNKNOWN_ERROR_CACHE_IN_MIN));
newExpiry = Instant.now(clock).plus(Duration.ofSeconds(unknownErrorCacheInSec));
tesCache.get(iotCredentialsPath).responseCode = HttpURLConnection.HTTP_INTERNAL_ERROR;
tesCache.get(iotCredentialsPath).expiry = newExpiry;
tesCache.get(iotCredentialsPath).credentials = response;
Expand Down Expand Up @@ -421,16 +447,16 @@ private String parseExpiryFromResponse(final String credentials) throws AWSIotEx
}

private Instant getExpiryPolicyForErr(int statusCode) {
int expiryTime = UNKNOWN_ERROR_CACHE_IN_MIN; // In case of unrecognized cloud errors, back off
int expiryTime = unknownErrorCacheInSec; // In case of unrecognized cloud errors, back off
// Add caching Time-To-Live (TTL) for TES cloud errors
if (statusCode >= 400 && statusCode < 500) {
// 4xx retries are only meaningful unless a user action has been adopted, TTL should be longer
expiryTime = CLOUD_4XX_ERROR_CACHE_IN_MIN;
expiryTime = cloud4xxErrorCacheInSec;
} else if (statusCode >= 500 && statusCode < 600) {
// 5xx could be a temporary cloud unavailability, TTL should be shorter
expiryTime = CLOUD_5XX_ERROR_CACHE_IN_MIN;
expiryTime = cloud5xxErrorCacheInSec;
}
return Instant.now(clock).plus(Duration.ofMinutes(expiryTime));
return Instant.now(clock).plus(Duration.ofSeconds(expiryTime));
}

/**
Expand Down
87 changes: 80 additions & 7 deletions src/main/java/com/aws/greengrass/tes/TokenExchangeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.aws.greengrass.authorization.exceptions.AuthorizationException;
import com.aws.greengrass.config.Topic;
import com.aws.greengrass.config.Topics;
import com.aws.greengrass.config.WhatHappened;
import com.aws.greengrass.dependency.ImplementsService;
import com.aws.greengrass.dependency.State;
import com.aws.greengrass.deployment.DeviceConfiguration;
Expand Down Expand Up @@ -41,6 +42,14 @@ public class TokenExchangeService extends GreengrassService implements AwsCreden
private String iotRoleAlias;
private HttpServerImpl server;

public static final String CLOUD_4XX_ERROR_CACHE_TOPIC = "error4xxCredentialRetryInSec";
public static final String CLOUD_5XX_ERROR_CACHE_TOPIC = "error5xxCredentialRetryInSec";
public static final String UNKNOWN_ERROR_CACHE_TOPIC = "errorUnknownCredentialRetryInSec";
private static final int MINIMUM_ERROR_CACHE_IN_SEC = 10;
private static final int CLOUD_4XX_ERROR_CACHE_INDEX = 0;
private static final int CLOUD_5XX_ERROR_CACHE_INDEX = 1;
private static final int UNKNOWN_ERROR_CACHE_INDEX = 2;

private final AuthorizationHandler authZHandler;
private final CredentialRequestHandler credentialRequestHandler;

Expand All @@ -57,24 +66,61 @@ public TokenExchangeService(Topics topics,
AuthorizationHandler authZHandler, DeviceConfiguration deviceConfiguration) {
super(topics);
port = Coerce.toInt(config.lookup(CONFIGURATION_CONFIG_KEY, PORT_TOPIC).dflt(DEFAULT_PORT));

deviceConfiguration.getIotRoleAlias().subscribe((why, newv) -> {
iotRoleAlias = Coerce.toString(newv);
});

this.authZHandler = authZHandler;
this.credentialRequestHandler = credentialRequestHandler;

int[] errorCacheConfig = getErrorCacheConfigValues();
credentialRequestHandler.configureCacheSettings(
errorCacheConfig[CLOUD_4XX_ERROR_CACHE_INDEX],
errorCacheConfig[CLOUD_5XX_ERROR_CACHE_INDEX],
errorCacheConfig[UNKNOWN_ERROR_CACHE_INDEX]);

// Subscribe to cache configuration changes
config.subscribe((why, node) -> {
if (node != null && node.childOf(PORT_TOPIC)) {
if (why.equals(WhatHappened.timestampUpdated)) {
return;
}
if (node != null && (node.childOf(CLOUD_4XX_ERROR_CACHE_TOPIC)
|| node.childOf(CLOUD_5XX_ERROR_CACHE_TOPIC)
|| node.childOf(UNKNOWN_ERROR_CACHE_TOPIC))) {
logger.atDebug("tes-config-change").kv("node", node).kv("why", why).log();

int[] newErrorCacheConfig = getErrorCacheConfigValues();
int[] oldErrorCacheConfig = credentialRequestHandler.getErrorCacheConfigSettings();

if (oldErrorCacheConfig[CLOUD_4XX_ERROR_CACHE_INDEX]
!= newErrorCacheConfig[CLOUD_4XX_ERROR_CACHE_INDEX]
|| oldErrorCacheConfig[CLOUD_5XX_ERROR_CACHE_INDEX]
!= newErrorCacheConfig[CLOUD_5XX_ERROR_CACHE_INDEX]
|| oldErrorCacheConfig[UNKNOWN_ERROR_CACHE_INDEX]
!= newErrorCacheConfig[UNKNOWN_ERROR_CACHE_INDEX]) {

credentialRequestHandler.configureCacheSettings(
newErrorCacheConfig[CLOUD_4XX_ERROR_CACHE_INDEX],
newErrorCacheConfig[CLOUD_5XX_ERROR_CACHE_INDEX],
newErrorCacheConfig[UNKNOWN_ERROR_CACHE_INDEX]);

logger.atInfo("tes-error-cache-config-change")
.kv("node", node).kv("why", why)
.log("TES error cache configuration updated");
}
}
if (node != null && node.childOf(PORT_TOPIC)) {
port = Coerce.toInt(node);
Topic activePortTopic = config.lookup(CONFIGURATION_CONFIG_KEY, ACTIVE_PORT_TOPIC);

if (port != Coerce.toInt(activePortTopic)) {
logger.atInfo("tes-config-change").kv(PORT_TOPIC, port).kv("node", node).kv("why", why)
.log("Restarting TES server due to port config change");
requestRestart();
}
}
});
deviceConfiguration.getIotRoleAlias().subscribe((why, newv) -> {
iotRoleAlias = Coerce.toString(newv);
});

this.authZHandler = authZHandler;
this.credentialRequestHandler = credentialRequestHandler;
}

@Override
Expand All @@ -95,6 +141,7 @@ protected void startup() {
.log("Attempting to start server at configured port {}", port);
try {
validateConfig();
validateAllErrorCacheConfigs();
server = new HttpServerImpl(port, credentialRequestHandler);
server.start();
logger.atInfo().log("Started server at port {}", server.getServerPort());
Expand Down Expand Up @@ -130,6 +177,32 @@ private void validateConfig() {
}
}

private void validateAllErrorCacheConfigs() {
// Check if all the error cache configurations are valid else throw IllegalArgumentException.
getErrorCacheConfigValues();
}

private int[] getErrorCacheConfigValues() {
int cloud4xxErrorCache = validateErrorCacheConfig(Coerce.toInt(config.findOrDefault(
CredentialRequestHandler.CLOUD_4XX_ERROR_CACHE_IN_SEC, CONFIGURATION_CONFIG_KEY,
CLOUD_4XX_ERROR_CACHE_TOPIC)), CLOUD_4XX_ERROR_CACHE_TOPIC);
int cloud5xxErrorCache = validateErrorCacheConfig(Coerce.toInt(config.findOrDefault(
CredentialRequestHandler.CLOUD_5XX_ERROR_CACHE_IN_SEC, CONFIGURATION_CONFIG_KEY,
CLOUD_5XX_ERROR_CACHE_TOPIC)), CLOUD_5XX_ERROR_CACHE_TOPIC);
int unknownErrorCache = validateErrorCacheConfig(Coerce.toInt(config.findOrDefault(
CredentialRequestHandler.UNKNOWN_ERROR_CACHE_IN_SEC, CONFIGURATION_CONFIG_KEY,
UNKNOWN_ERROR_CACHE_TOPIC)), UNKNOWN_ERROR_CACHE_TOPIC);
return new int[]{cloud4xxErrorCache, cloud5xxErrorCache, unknownErrorCache};
}

private int validateErrorCacheConfig(int newCacheValue, String topic) {
if (newCacheValue < MINIMUM_ERROR_CACHE_IN_SEC) {
throw new IllegalArgumentException(
"Error cache value for " + topic + " must be at least " + MINIMUM_ERROR_CACHE_IN_SEC + " seconds");
}
return newCacheValue;
}

@Override
public AwsCredentials resolveCredentials() {
return credentialRequestHandler.getAwsCredentials();
Expand Down
18 changes: 9 additions & 9 deletions src/test/java/com/aws/greengrass/tes/CredentialRequestHandlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import static com.aws.greengrass.tes.CredentialRequestHandler.CLOUD_4XX_ERROR_CACHE_IN_MIN;
import static com.aws.greengrass.tes.CredentialRequestHandler.CLOUD_5XX_ERROR_CACHE_IN_MIN;
import static com.aws.greengrass.tes.CredentialRequestHandler.TIME_BEFORE_CACHE_EXPIRE_IN_MIN;
import static com.aws.greengrass.tes.CredentialRequestHandler.UNKNOWN_ERROR_CACHE_IN_MIN;
import static com.aws.greengrass.tes.CredentialRequestHandler.CLOUD_4XX_ERROR_CACHE_IN_SEC;
import static com.aws.greengrass.tes.CredentialRequestHandler.CLOUD_5XX_ERROR_CACHE_IN_SEC;
import static com.aws.greengrass.tes.CredentialRequestHandler.TIME_BEFORE_CACHE_EXPIRE_IN_SEC;
import static com.aws.greengrass.tes.CredentialRequestHandler.UNKNOWN_ERROR_CACHE_IN_SEC;
import static com.aws.greengrass.testcommons.testutilities.ExceptionLogProtector.ignoreExceptionOfType;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsString;
Expand Down Expand Up @@ -320,15 +320,15 @@ void GIVEN_credential_handler_WHEN_called_handle_THEN_caches_creds() throws Exce
verify(mockStream, times(1)).write(expectedResponse);

// Expiry time in recent future won't give error but there wil be no caching
expirationTime = Instant.now().plus(Duration.ofMinutes(TIME_BEFORE_CACHE_EXPIRE_IN_MIN - 1));
expirationTime = Instant.now().plus(Duration.ofSeconds(TIME_BEFORE_CACHE_EXPIRE_IN_SEC - 60));
responseStr = String.format(RESPONSE_STR, expirationTime.toString());
mockResponse = new IotCloudResponse(responseStr.getBytes(StandardCharsets.UTF_8), 200);
when(mockCloudHelper.sendHttpRequest(any(), any(), any(), any(), any())).thenReturn(mockResponse);
handler.handle(mockExchange);
verify(mockCloudHelper, times(2)).sendHttpRequest(any(), any(), any(), any(), any());

// Expiry time in future will result in credentials being cached
expirationTime = Instant.now().plus(Duration.ofMinutes(TIME_BEFORE_CACHE_EXPIRE_IN_MIN + 1));
expirationTime = Instant.now().plus(Duration.ofSeconds(TIME_BEFORE_CACHE_EXPIRE_IN_SEC + 60));
responseStr = String.format(RESPONSE_STR, expirationTime.toString());
mockResponse = new IotCloudResponse(responseStr.getBytes(StandardCharsets.UTF_8), 200);
when(mockCloudHelper.sendHttpRequest(any(), any(), any(), any(), any())).thenReturn(mockResponse);
Expand Down Expand Up @@ -401,7 +401,7 @@ void GIVEN_4xx_response_code_WHEN_called_handle_THEN_expire_in_2_minutes() throw
String.format("TES responded with status code: %d. Caching response. ", expectedStatus).getBytes();
// expire in 2 minutes
handler.getAwsCredentials();
Instant expirationTime = Instant.now().plus(Duration.ofMinutes(CLOUD_4XX_ERROR_CACHE_IN_MIN));
Instant expirationTime = Instant.now().plus(Duration.ofSeconds(CLOUD_4XX_ERROR_CACHE_IN_SEC));
Clock mockClock = Clock.fixed(expirationTime, ZoneId.of("UTC"));
handler.setClock(mockClock);
handler.getAwsCredentials();
Expand All @@ -425,7 +425,7 @@ void GIVEN_5xx_response_code_WHEN_called_handle_THEN_expire_in_1_minute() throws
String.format("TES responded with status code: %d. Caching response. ", expectedStatus).getBytes();
// expire in 1 minute
handler.getAwsCredentials();
Instant expirationTime = Instant.now().plus(Duration.ofMinutes(CLOUD_5XX_ERROR_CACHE_IN_MIN));
Instant expirationTime = Instant.now().plus(Duration.ofSeconds(CLOUD_5XX_ERROR_CACHE_IN_SEC));
Clock mockClock = Clock.fixed(expirationTime, ZoneId.of("UTC"));
handler.setClock(mockClock);
handler.getAwsCredentials();
Expand All @@ -449,7 +449,7 @@ void GIVEN_unknown_error_response_code_WHEN_called_handle_THEN_expire_in_5_minut
String.format("TES responded with status code: %d. Caching response. ", expectedStatus).getBytes();
// expire in 5 minutes
handler.getAwsCredentials();
Instant expirationTime = Instant.now().plus(Duration.ofMinutes(UNKNOWN_ERROR_CACHE_IN_MIN));
Instant expirationTime = Instant.now().plus(Duration.ofSeconds(UNKNOWN_ERROR_CACHE_IN_SEC));
Clock mockClock = Clock.fixed(expirationTime, ZoneId.of("UTC"));
handler.setClock(mockClock);
handler.getAwsCredentials();
Expand Down
Loading
Loading