adding http cache

Signed-off-by: liran2000 <[email protected]>

Author: liran2000

providers/flagd/README.md

@@ -55,7 +55,10 @@ This can be used to enrich evaluations with such data.
 If the `in-process` mode is not used, and before the provider is ready, the `getSyncMetadata` returns an empty map.
 
 #### Http Connector
-HttpConnector is responsible for polling data from a specified URL at regular intervals.
+HttpConnector is responsible for polling data from a specified URL at regular intervals.  
+It is implementing Http cache mechanism with 'ETag' header, then when receiving 304 Not Modified response, reducing traffic and 
+changes updates. Can be enabled via useHttpCache option.  
+One of its benefits is to reduce infrastructure/devops work, without additional containers needed.  
 The implementation is using Java HttpClient.
 
 ##### What happens if the Http source is down when application is starting ?

providers/flagd/src/main/java/dev/openfeature/contrib/providers/flagd/resolver/process/storage/connector/sync/HttpCacheFetcher.java

@@ -0,0 +1,46 @@
+package dev.openfeature.contrib.providers.flagd.resolver.process.storage.connector.sync;
+
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Fetches content from a given HTTP endpoint using caching headers to optimize network usage.
+ * If cached ETag or Last-Modified values are available, they are included in the request headers
+ * to potentially receive a 304 Not Modified response, reducing data transfer.
+ * Updates the cached ETag and Last-Modified values upon receiving a 200 OK response.
+ * It does not store the cached response, assuming not needed after first successful fetching.
+ *
+ * @param httpClient the HTTP client used to send the request
+ * @param httpRequestBuilder the builder for constructing the HTTP request
+ * @return the HTTP response received from the server
+ */
+@Slf4j
+public class HttpCacheFetcher {
+    private static String cachedETag = null;
+    private static String cachedLastModified = null;
+
+    @SneakyThrows
+    public HttpResponse<String> fetchContent(HttpClient httpClient, HttpRequest.Builder httpRequestBuilder) {
+        if (cachedETag != null) {
+            httpRequestBuilder.header("If-None-Match", cachedETag);
+        }
+        if (cachedLastModified != null) {
+            httpRequestBuilder.header("If-Modified-Since", cachedLastModified);
+        }
+
+        HttpRequest request = httpRequestBuilder.build();
+        HttpResponse<String> httpResponse = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+        if (httpResponse.statusCode() == 200) {
+            cachedETag = httpResponse.headers().firstValue("ETag").orElse(null);
+            cachedLastModified = httpResponse.headers().firstValue("Last-Modified").orElse(null);
+            log.debug("fetched new content");
+        } else if (httpResponse.statusCode() == 304) {
+            log.debug("got 304 Not Modified");
+        }
+        return httpResponse;
+    }
+}

providers/flagd/src/main/java/dev/openfeature/contrib/providers/flagd/resolver/process/storage/connector/sync/HttpConnector.java

@@ -58,6 +58,7 @@ public class HttpConnector implements QueueSource {
     private Map<String, String> headers;
     private PayloadCacheWrapper payloadCacheWrapper;
     private PayloadCache payloadCache;
+    private HttpCacheFetcher httpCacheFetcher;
 
     @NonNull
     private String url;
@@ -66,7 +67,7 @@ public class HttpConnector implements QueueSource {
     public HttpConnector(Integer pollIntervalSeconds, Integer linkedBlockingQueueCapacity,
              Integer scheduledThreadPoolSize, Integer requestTimeoutSeconds, Integer connectTimeoutSeconds, String url,
              Map<String, String> headers, ExecutorService httpClientExecutor, String proxyHost, Integer proxyPort,
-            PayloadCacheOptions payloadCacheOptions, PayloadCache payloadCache) {
+            PayloadCacheOptions payloadCacheOptions, PayloadCache payloadCache, Boolean useHttpCache) {
         validate(url, pollIntervalSeconds, linkedBlockingQueueCapacity, scheduledThreadPoolSize, requestTimeoutSeconds,
             connectTimeoutSeconds, proxyHost, proxyPort, payloadCacheOptions, payloadCache);
         this.pollIntervalSeconds = pollIntervalSeconds == null ? DEFAULT_POLL_INTERVAL_SECONDS : pollIntervalSeconds;
@@ -100,6 +101,9 @@ public HttpConnector(Integer pollIntervalSeconds, Integer linkedBlockingQueueCap
                 .payloadCacheOptions(payloadCacheOptions)
                 .build();
         }
+        if (Boolean.TRUE.equals(useHttpCache)) {
+            httpCacheFetcher = new HttpCacheFetcher();
+        }
     }
 
     @SneakyThrows
@@ -180,13 +184,11 @@ private boolean fetchAndUpdate() {
             .timeout(Duration.ofSeconds(requestTimeoutSeconds))
             .GET();
         headers.forEach(requestBuilder::header);
-        HttpRequest request = requestBuilder
-            .build();
 
         HttpResponse<String> response;
         try {
             log.debug("fetching response");
-            response = execute(request);
+            response = execute(requestBuilder);
         } catch (IOException e) {
             log.info("could not fetch", e);
             return false;
@@ -196,14 +198,18 @@ private boolean fetchAndUpdate() {
         }
         log.debug("fetched response");
         String payload = response.body();
-        if (response.statusCode() != 200) {
+        if (!isSuccessful(response)) {
             log.info("received non-successful status code: {} {}", response.statusCode(), payload);
             return false;
+        } else if (response.statusCode() == 304) {
+            log.debug("got 304 Not Modified, skipping update");
+            return false;
         }
         if (payload == null) {
             log.debug("payload is null");
             return false;
         }
+        log.debug("adding payload to queue");
         if (!this.queue.offer(new QueuePayload(QueuePayloadType.DATA, payload))) {
             log.warn("Unable to offer file content to queue: queue is full");
             return false;
@@ -217,8 +223,15 @@ private boolean fetchAndUpdate() {
         return payload != null;
     }
 
-    protected HttpResponse<String> execute(HttpRequest request) throws IOException, InterruptedException {
-        return client.send(request, HttpResponse.BodyHandlers.ofString());
+    private static boolean isSuccessful(HttpResponse<String> response) {
+        return response.statusCode() == 200 || response.statusCode() == 304;
+    }
+
+    protected HttpResponse<String> execute(HttpRequest.Builder requestBuilder) throws IOException, InterruptedException {
+        if (httpCacheFetcher != null) {
+            return httpCacheFetcher.fetchContent(client, requestBuilder);
+        }
+        return client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString());
     }
 
     @Override

providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/resolver/process/storage/connector/sync/HttpConnectorIntegrationTest.java

@@ -0,0 +1,78 @@
+package dev.openfeature.contrib.providers.flagd.resolver.process.storage.connector.sync;
+
+import static dev.openfeature.contrib.providers.flagd.resolver.process.storage.connector.sync.HttpConnectorTest.delay;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import dev.openfeature.contrib.providers.flagd.resolver.process.storage.connector.QueuePayload;
+import dev.openfeature.contrib.providers.flagd.resolver.process.storage.connector.QueuePayloadType;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.MalformedURLException;
+import java.net.ProxySelector;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+/**
+ * Integration test for the HttpConnector class, specifically testing the ability to fetch
+ * raw content from a GitHub URL. This test assumes that integration tests are enabled
+ * and verifies that the HttpConnector can successfully enqueue data from the specified URL.
+ * The test initializes the HttpConnector with specific configurations, waits for data
+ * to be enqueued, and asserts the expected queue size. The connector is shut down
+ * gracefully after the test execution.
+ * As this integration test using external request, it is disabled by default, and not part of the CI build.
+ */
+@Slf4j
+class HttpConnectorIntegrationTest {
+
+    @SneakyThrows
+    @Test
+    void testGithubRawContent() {
+        assumeTrue(parseBoolean("integrationTestsEnabled"));
+        HttpConnector connector = null;
+        try {
+            String testUrl = "https://raw.githubusercontent.com/open-feature/java-sdk-contrib/58fe5da7d4e2f6f4ae2c1caf3411a01e84a1dc1a/providers/flagd/version.txt";
+            connector = HttpConnector.builder()
+                .url(testUrl)
+                .connectTimeoutSeconds(10)
+                .requestTimeoutSeconds(10)
+                .useHttpCache(true)
+                .pollIntervalSeconds(5)
+                .build();
+            BlockingQueue<QueuePayload> queue = connector.getStreamQueue();
+            delay(20000);
+            assertEquals(1, queue.size());
+        } finally {
+            if (connector != null) {
+                connector.shutdown();
+            }
+        }
+    }
+
+    public static boolean parseBoolean(String key) {
+        return Boolean.parseBoolean(System.getProperty(key, System.getenv(key)));
+    }
+
+}

providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/resolver/process/storage/connector/sync/HttpConnectorTest.java

@@ -579,7 +579,7 @@ void testQueuePayloadTypeSetToDataOnSuccess() {
     }
 
     @SneakyThrows
-    private static void delay(long ms) {
+    protected static void delay(long ms) {
         Thread.sleep(ms);
     }
 

providers/flagd/src/test/resources/simplelogger.properties

@@ -1,4 +1,4 @@
-org.org.slf4j.simpleLogger.defaultLogLevel=debug
+org.slf4j.simpleLogger.defaultLogLevel=debug
 org.slf4j.simpleLogger.showDateTime=
 
 io.grpc.level=trace