adapt to latest version running on production

Author: netomi

server/src/main/java/org/eclipse/openvsx/cache/CacheService.java

@@ -101,12 +101,11 @@ public void evictExtensionJsons(Extension extension) {
             return;
         }
 
-        var namespaceName = extension.getNamespace().getName();
-        var extensionName = extension.getName();
-
-        // Special optimization in case of a redis cache: evict all keys that match the <namespace>.<extension>* pattern
+        // Special optimization in case of a redis cache: evict all keys that match the <namespace>.<extension>* pattern.
+        // This uses the redis KEYS command that might take a while but considering the typical size of the EXTENSION_JSON
+        // cache its acceptable.
         if (cache instanceof RedisCacheWriter redisCache) {
-            redisCache.clean(CACHE_EXTENSION_JSON, extensionJsonCacheKey.generateWildcard(namespaceName, extensionName).getBytes());
+            redisCache.clean(CACHE_EXTENSION_JSON, extensionJsonCacheKey.generateWildcard(extension).getBytes());
             return;
         }
 
@@ -115,6 +114,8 @@ public void evictExtensionJsons(Extension extension) {
                 .map(ExtensionVersion::getVersion)
                 .forEach(versions::add);
 
+        var namespaceName = extension.getNamespace().getName();
+        var extensionName = extension.getName();
         var targetPlatforms = new ArrayList<>(TargetPlatform.TARGET_PLATFORM_NAMES);
         targetPlatforms.add("null");
         for (var version : versions) {
@@ -154,6 +155,14 @@ public void evictLatestExtensionVersion(Extension extension) {
             return;
         }
 
+        // Special optimization in case of a redis cache: evict all keys that match the <namespace>.<extension>* pattern.
+        // This uses the redis KEYS command that might take a while but considering the typical size of the EXTENSION_JSON
+        // cache its acceptable.
+        if (cache instanceof RedisCacheWriter redisCache) {
+            redisCache.clean(CACHE_LATEST_EXTENSION_VERSION, latestExtensionVersionCacheKey.generateWildcard(extension).getBytes());
+            return;
+        }
+
         var targetPlatforms = new ArrayList<>(TargetPlatform.TARGET_PLATFORM_NAMES);
         targetPlatforms.add(null);
         for (var targetPlatform : targetPlatforms) {

server/src/main/java/org/eclipse/openvsx/cache/ExtensionJsonCacheKeyGenerator.java

@@ -9,6 +9,7 @@
  * ****************************************************************************** */
 package org.eclipse.openvsx.cache;
 
+import org.eclipse.openvsx.entities.Extension;
 import org.eclipse.openvsx.util.NamingUtil;
 import org.eclipse.openvsx.util.VersionAlias;
 import org.springframework.cache.interceptor.KeyGenerator;
@@ -29,7 +30,9 @@ public String generate(String namespaceName, String extensionName, String target
         return NamingUtil.toFileFormat(namespaceName, extensionName, version, targetPlatform);
     }
 
-    public String generateWildcard(String namespaceName, String extensionName) {
+    public String generateWildcard(Extension extension) {
+        var extensionName = extension.getName();
+        var namespaceName = extension.getNamespace().getName();
         return NamingUtil.toExtensionId(namespaceName, extensionName) + "*";
     }
 }

server/src/main/java/org/eclipse/openvsx/cache/LatestExtensionVersionCacheKeyGenerator.java

@@ -56,4 +56,10 @@ public String generate(Extension extension, String targetPlatform, boolean preRe
         return NamingUtil.toFileFormat(namespaceName, extensionName, targetPlatform, VersionAlias.LATEST) +
                 ",pre-release=" + preRelease + ",only-active=" + onlyActive + ",type=" + type;
     }
+
+    public String generateWildcard(Extension extension) {
+        var extensionName = extension.getName();
+        var namespaceName = extension.getNamespace().getName();
+        return NamingUtil.toExtensionId(namespaceName, extensionName) + "*";
+    }
 }

server/src/main/java/org/eclipse/openvsx/mirror/aop/DownloadCountServiceAspect.java

@@ -19,7 +19,7 @@
 @ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "true")
 public class DownloadCountServiceAspect {
 
-    @Around("execution(* org.eclipse.openvsx.storage.log.DownloadCountService.isEnabled(..))")
+    @Around("execution(* org.eclipse.openvsx.storage.log.*DownloadCountService.isEnabled(..))")
     public Object isEnabled() throws Throwable {
         return false;
     }

server/src/main/java/org/eclipse/openvsx/repositories/AzureDownloadCountProcessedItemRepository.java

@@ -10,6 +10,7 @@
 package org.eclipse.openvsx.storage.log;
 
 import org.apache.commons.lang3.StringUtils;
+import org.eclipse.openvsx.entities.Extension;
 import org.eclipse.openvsx.entities.FileResource;
 import org.eclipse.openvsx.storage.AwsStorageService;
 import org.eclipse.openvsx.util.TempFile;
@@ -52,6 +53,7 @@
 public class AwsDownloadCountService {
     private final Logger logger = LoggerFactory.getLogger(AwsDownloadCountService.class);
 
+    private static final String LOG_LOCATION_PREFIX = "AWSLogs/";
     private static final int MAX_KEYS = 100;
 
     private final AwsStorageService  awsStorageService;
@@ -60,8 +62,8 @@ public class AwsDownloadCountService {
     @Value("${ovsx.logs.aws.bucket:}")
     String bucket;
 
-    @Value("${ovsx.logs.aws.prefix:AWSLogs/}")
-    String prefix;
+    @Value("${ovsx.logs.aws.log-location-prefix:" + LOG_LOCATION_PREFIX + "}")
+    String logLocationPrefix;
 
     public AwsDownloadCountService(AwsStorageService awsStorageService, DownloadCountProcessor processor) {
         this.awsStorageService = awsStorageService;
@@ -138,39 +140,46 @@ private boolean processResponse(
         }
         logFiles.removeAll(failedItems);
 
-        for (var name : logFiles) {
-            var processedOn = LocalDateTime.now();
+        var allUpdatedExtensions = new HashMap<Long, Extension>();
 
-            if (processedOn.isAfter(maxExecutionTime)) {
-                logger.info("Failed to process all download counts within timeslot, next job run is at {}", nextJobRunTime);
-                return false;
-            }
+        try {
+            for (var name : logFiles) {
+                var processedOn = LocalDateTime.now();
 
-            var success = false;
-            stopWatch.start();
-            try {
-                var counts = processLogFile(name);
-                if (!counts.isEmpty()) {
-                    var extensionDownloads = processor.processDownloadCounts(FileResource.STORAGE_AWS, counts);
-                    var updatedExtensions = processor.increaseDownloadCounts(extensionDownloads);
-                    processor.evictCaches(updatedExtensions);
-                    processor.updateSearchEntries(updatedExtensions);
+                if (processedOn.isAfter(maxExecutionTime)) {
+                    logger.info("Failed to process all download counts within timeslot, next job run is at {}", nextJobRunTime);
+                    return false;
                 }
 
-                success = true;
-            } catch (Exception e) {
-                logger.error("failed to process log file: {}", name, e);
-            }
+                var success = false;
+                stopWatch.start();
+                try {
+                    var counts = processLogFile(name);
+                    if (!counts.isEmpty()) {
+                        var extensionDownloads = processor.processDownloadCounts(FileResource.STORAGE_AWS, counts);
+                        var updatedExtensions = processor.increaseDownloadCounts(extensionDownloads);
+                        updatedExtensions.forEach(extension -> allUpdatedExtensions.put(extension.getId(), extension));
+                    }
+
+                    success = true;
+                } catch (Exception e) {
+                    logger.error("failed to process log file: {}", name, e);
+                }
 
-            stopWatch.stop();
-            var executionTime = (int) stopWatch.lastTaskInfo().getTimeMillis();
-            processor.persistProcessedItem(name, FileResource.STORAGE_AWS, processedOn, executionTime, success);
-            if (success) {
-                deleteFile(name);
+                stopWatch.stop();
+                var executionTime = (int) stopWatch.lastTaskInfo().getTimeMillis();
+                processor.persistProcessedItem(name, FileResource.STORAGE_AWS, processedOn, executionTime, success);
+                if (success) {
+                    deleteFile(name);
+                }
             }
-        }
 
-        return true;
+            return true;
+        } finally {
+            // evict caches and update search entries for all updated extensions
+            allUpdatedExtensions.values().forEach(processor::evictCaches);
+            processor.updateSearchEntries(allUpdatedExtensions.values().stream().toList());
+        }
     }
 
     private Map<String, Integer> processLogFile(String fileName) throws IOException {
@@ -188,6 +197,8 @@ private Map<String, Integer> processLogFile(String fileName) throws IOException
                     continue;
                 }
 
+                // Format:
+                // date	time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status	cs(Referer)	cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type	x-edge-request-id	x-host-header	cs-protocol	cs-bytes	time-taken	x-forwarded-for	ssl-protocol	ssl-cipher	x-edge-response-result-type	cs-protocol-version	fle-status	fle-encrypted-fields	c-port	time-to-first-byte	x-edge-detailed-result-type	sc-content-type	sc-content-len	sc-range-start	sc-range-end
                 var components = line.split("[ \t]+");
 
                 if (isGetOperation(components) && isStatusOk(components) && isExtensionPackageUri(components)) {
@@ -232,7 +243,7 @@ private void deleteFile(String objectKey) {
     }
 
     private ListObjectsV2Response listObjects(String continuationToken) {
-        var builder = ListObjectsV2Request.builder().bucket(bucket).maxKeys(MAX_KEYS).prefix(prefix);
+        var builder = ListObjectsV2Request.builder().bucket(bucket).maxKeys(MAX_KEYS).prefix(logLocationPrefix);
 
         if (continuationToken != null) {
             builder.continuationToken(continuationToken);

server/src/main/java/org/eclipse/openvsx/storage/log/AwsDownloadCountService.java

@@ -145,7 +145,7 @@ private boolean processResponse(PagedResponse<BlobItem> response, StopWatch stop
                 if (!files.isEmpty()) {
                     var extensionDownloads = processor.processDownloadCounts(FileResource.STORAGE_AZURE, files);
                     var updatedExtensions = processor.increaseDownloadCounts(extensionDownloads);
-                    processor.evictCaches(updatedExtensions);
+                    updatedExtensions.forEach(processor::evictCaches);
                     processor.updateSearchEntries(updatedExtensions);
                 }
 

server/src/main/java/org/eclipse/openvsx/storage/log/AzureDownloadCountService.java

@@ -88,13 +88,13 @@ public List<Extension> increaseDownloadCounts(Map<Long, Integer> extensionDownlo
         });
     }
 
-    @Transactional //needs transaction for lazy-loading versions
-    public void evictCaches(List<Extension> extensions) {
-        Observation.createNotStarted("DownloadCountProcessor#evictCaches", observations).observe(() -> extensions.forEach(extension -> {
-            extension = entityManager.merge(extension);
-            cache.evictExtensionJsons(extension);
-            cache.evictLatestExtensionVersion(extension);
-        }));
+    @Transactional // needs transaction for lazy-loading versions
+    public void evictCaches(Extension extension) {
+        Observation.createNotStarted("DownloadCountProcessor#evictCaches", observations).observe(() -> {
+            var mergedExtension = entityManager.merge(extension);
+            cache.evictExtensionJsons(mergedExtension);
+            cache.evictLatestExtensionVersion(mergedExtension);
+        });
     }
 
     public void updateSearchEntries(List<Extension> extensions) {