Skip to content

Commit e9908e2

Browse files
netominetomi
authored
Add AwsDownloadCountService (#1451)
* Add AwsDownloadCountService Remove unused Shedlock classes Rename AzureDownloadCountProcessedItem entity to DownloadCountProcessedItem and include a storageType field * fix unit test and remove jobrunr migration statement * use correct storage type for downloads, drop table shedlock only if it exists * cleanup imports * add index for name, make migration script idempotent * add javadoc, finalize AwsDownloadCountService * skip processing of previously failed log files, correctly determine the next job run time * fix unit test * add optimization for redis cache when evicting cache extension_json for all versions of an extension * adapt to latest version running on production * fix unit test
1 parent 6e9e253 commit e9908e2

32 files changed

+868
-743
lines changed

‎server/src/main/java/org/eclipse/openvsx/cache/CacheService.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.eclipse.openvsx.util.TargetPlatform;
1616
import org.eclipse.openvsx.util.VersionAlias;
1717
import org.springframework.cache.CacheManager;
18+
import org.springframework.data.redis.cache.RedisCacheWriter;
1819
import org.springframework.stereotype.Component;
1920

2021
import java.util.ArrayList;
@@ -100,6 +101,14 @@ public void evictExtensionJsons(Extension extension) {
100101
return;
101102
}
102103

104+
// Special optimization in case of a redis cache: evict all keys that match the <namespace>.<extension>* pattern.
105+
// This uses the redis KEYS command that might take a while but considering the typical size of the EXTENSION_JSON
106+
// cache its acceptable.
107+
if (cache instanceof RedisCacheWriter redisCache) {
108+
redisCache.clean(CACHE_EXTENSION_JSON, extensionJsonCacheKey.generateWildcard(extension).getBytes());
109+
return;
110+
}
111+
103112
var versions = new ArrayList<>(VersionAlias.ALIAS_NAMES);
104113
extension.getVersions().stream()
105114
.map(ExtensionVersion::getVersion)
@@ -146,6 +155,14 @@ public void evictLatestExtensionVersion(Extension extension) {
146155
return;
147156
}
148157

158+
// Special optimization in case of a redis cache: evict all keys that match the <namespace>.<extension>* pattern.
159+
// This uses the redis KEYS command that might take a while but considering the typical size of the EXTENSION_JSON
160+
// cache its acceptable.
161+
if (cache instanceof RedisCacheWriter redisCache) {
162+
redisCache.clean(CACHE_LATEST_EXTENSION_VERSION, latestExtensionVersionCacheKey.generateWildcard(extension).getBytes());
163+
return;
164+
}
165+
149166
var targetPlatforms = new ArrayList<>(TargetPlatform.TARGET_PLATFORM_NAMES);
150167
targetPlatforms.add(null);
151168
for (var targetPlatform : targetPlatforms) {

‎server/src/main/java/org/eclipse/openvsx/cache/ExtensionJsonCacheKeyGenerator.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
* ****************************************************************************** */
1010
package org.eclipse.openvsx.cache;
1111

12+
import org.eclipse.openvsx.entities.Extension;
1213
import org.eclipse.openvsx.util.NamingUtil;
1314
import org.eclipse.openvsx.util.VersionAlias;
1415
import org.springframework.cache.interceptor.KeyGenerator;
@@ -28,4 +29,10 @@ public Object generate(Object target, Method method, Object... params) {
2829
public String generate(String namespaceName, String extensionName, String targetPlatform, String version) {
2930
return NamingUtil.toFileFormat(namespaceName, extensionName, targetPlatform, version);
3031
}
32+
33+
public String generateWildcard(Extension extension) {
34+
var extensionName = extension.getName();
35+
var namespaceName = extension.getNamespace().getName();
36+
return NamingUtil.toExtensionId(namespaceName, extensionName) + "*";
37+
}
3138
}

‎server/src/main/java/org/eclipse/openvsx/cache/LatestExtensionVersionCacheKeyGenerator.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,4 +56,10 @@ public String generate(Extension extension, String targetPlatform, boolean preRe
5656
return NamingUtil.toFileFormat(namespaceName, extensionName, targetPlatform, VersionAlias.LATEST) +
5757
",pre-release=" + preRelease + ",only-active=" + onlyActive + ",type=" + type;
5858
}
59+
60+
public String generateWildcard(Extension extension) {
61+
var extensionName = extension.getName();
62+
var namespaceName = extension.getNamespace().getName();
63+
return NamingUtil.toExtensionId(namespaceName, extensionName) + "*";
64+
}
5965
}

‎server/src/main/java/org/eclipse/openvsx/entities/DownloadCountProcessedItem.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/********************************************************************************
2+
* Copyright (c) 2025 Eclipse Foundation and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
********************************************************************************/
10+
package org.eclipse.openvsx.entities;
11+
12+
import jakarta.persistence.Entity;
13+
import jakarta.persistence.GeneratedValue;
14+
import jakarta.persistence.Id;
15+
import jakarta.persistence.SequenceGenerator;
16+
17+
import java.time.LocalDateTime;
18+
19+
@Entity
20+
public class DownloadCountProcessedItem {
21+
22+
@Id
23+
@GeneratedValue(generator = "downloadCountProcessedItemSeq")
24+
@SequenceGenerator(name = "downloadCountProcessedItemSeq", sequenceName = "download_count_processed_item_seq")
25+
private long id;
26+
27+
private String name;
28+
29+
private String storageType;
30+
31+
private LocalDateTime processedOn;
32+
33+
private int executionTime;
34+
35+
private boolean success;
36+
37+
public long getId() {
38+
return id;
39+
}
40+
41+
public void setId(long id) {
42+
this.id = id;
43+
}
44+
45+
public String getName() {
46+
return name;
47+
}
48+
49+
public void setName(String name) {
50+
this.name = name;
51+
}
52+
53+
public String getStorageType() {
54+
return storageType;
55+
}
56+
57+
public void setStorageType(String storageType) {
58+
this.storageType = storageType;
59+
}
60+
61+
public LocalDateTime getProcessedOn() {
62+
return processedOn;
63+
}
64+
65+
public void setProcessedOn(LocalDateTime processedOn) {
66+
this.processedOn = processedOn;
67+
}
68+
69+
public int getExecutionTime() {
70+
return executionTime;
71+
}
72+
73+
public void setExecutionTime(int executionTime) {
74+
this.executionTime = executionTime;
75+
}
76+
77+
public boolean isSuccess() {
78+
return success;
79+
}
80+
81+
public void setSuccess(boolean success) {
82+
this.success = success;
83+
}
84+
}

server/src/main/java/org/eclipse/openvsx/mirror/aop/AzureDownloadCountServiceAspect.java renamed to server/src/main/java/org/eclipse/openvsx/mirror/aop/DownloadCountServiceAspect.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
@Aspect
1818
@Component
1919
@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "true")
20-
public class AzureDownloadCountServiceAspect {
20+
public class DownloadCountServiceAspect {
2121

22-
@Around("execution(* org.eclipse.openvsx.storage.AzureDownloadCountService.isEnabled(..))")
22+
@Around("execution(* org.eclipse.openvsx.storage.log.*DownloadCountService.isEnabled(..))")
2323
public Object isEnabled() throws Throwable {
2424
return false;
2525
}

‎server/src/main/java/org/eclipse/openvsx/repositories/AzureDownloadCountProcessedItemRepository.java‎

Lines changed: 0 additions & 22 deletions
This file was deleted.

‎server/src/main/java/org/eclipse/openvsx/repositories/DownloadCountProcessedItemRepository.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/********************************************************************************
2+
* Copyright (c) 2021 Precies. Software and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
********************************************************************************/
10+
package org.eclipse.openvsx.repositories;
11+
12+
import org.eclipse.openvsx.entities.DownloadCountProcessedItem;
13+
import org.springframework.data.jpa.repository.Query;
14+
import org.springframework.data.repository.Repository;
15+
16+
import java.util.List;
17+
18+
public interface DownloadCountProcessedItemRepository extends Repository<DownloadCountProcessedItem, Long> {
19+
20+
@Query("select dc.name from DownloadCountProcessedItem dc where dc.success = true and dc.storageType = ?1 and dc.name in(?2)")
21+
List<String> findAllSucceededDownloadCountProcessedItemsByStorageTypeAndNameIn(String storageType, List<String> names);
22+
23+
@Query("select dc.name from DownloadCountProcessedItem dc where dc.success = false and dc.storageType = ?1 and dc.name in(?2)")
24+
List<String> findAllFailedDownloadCountProcessedItemsByStorageTypeAndNameIn(String storageType, List<String> names);
25+
}

‎server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public class RepositoryService {
4848
private final PersonalAccessTokenRepository tokenRepo;
4949
private final PersonalAccessTokenJooqRepository tokenJooqRepo;
5050
private final PersistedLogRepository persistedLogRepo;
51-
private final AzureDownloadCountProcessedItemRepository downloadCountRepo;
51+
private final DownloadCountProcessedItemRepository downloadCountRepo;
5252
private final ExtensionJooqRepository extensionJooqRepo;
5353
private final ExtensionVersionJooqRepository extensionVersionJooqRepo;
5454
private final FileResourceJooqRepository fileResourceJooqRepo;
@@ -73,7 +73,7 @@ public RepositoryService(
7373
PersonalAccessTokenRepository tokenRepo,
7474
PersonalAccessTokenJooqRepository tokenJooqRepo,
7575
PersistedLogRepository persistedLogRepo,
76-
AzureDownloadCountProcessedItemRepository downloadCountRepo,
76+
DownloadCountProcessedItemRepository downloadCountRepo,
7777
ExtensionJooqRepository extensionJooqRepo,
7878
ExtensionVersionJooqRepository extensionVersionJooqRepo,
7979
FileResourceJooqRepository fileResourceJooqRepo,
@@ -359,8 +359,12 @@ public Streamable<PersistedLog> findPersistedLogsAfter(LocalDateTime dateTime) {
359359
return persistedLogRepo.findByTimestampAfterOrderByTimestampAsc(dateTime);
360360
}
361361

362-
public List<String> findAllSucceededAzureDownloadCountProcessedItemsByNameIn(List<String> names) {
363-
return downloadCountRepo.findAllSucceededAzureDownloadCountProcessedItemsByNameIn(names);
362+
public List<String> findAllSucceededDownloadCountProcessedItemsByStorageTypeAndNameIn(String storageType, List<String> names) {
363+
return downloadCountRepo.findAllSucceededDownloadCountProcessedItemsByStorageTypeAndNameIn(storageType, names);
364+
}
365+
366+
public List<String> findAllFailedDownloadCountProcessedItemsByStorageTypeAndNameIn(String storageType, List<String> names) {
367+
return downloadCountRepo.findAllFailedDownloadCountProcessedItemsByStorageTypeAndNameIn(storageType, names);
364368
}
365369

366370
public List<Extension> findActiveExtensionsByPublicId(Collection<String> publicIds, String... namespacesToExclude) {

‎server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public AwsStorageService(FileCacheDurationConfig fileCacheDurationConfig, FilesC
8383
this.filesCacheKeyGenerator = filesCacheKeyGenerator;
8484
}
8585

86-
protected S3Client getS3Client() {
86+
public S3Client getS3Client() {
8787
if (s3Client == null) {
8888
var s3ClientBuilder = S3Client.builder()
8989
.defaultsMode(DefaultsMode.STANDARD)

‎server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.eclipse.openvsx.entities.Namespace;
2222
import org.eclipse.openvsx.repositories.RepositoryService;
2323
import org.eclipse.openvsx.search.SearchUtilService;
24+
import org.eclipse.openvsx.storage.log.DownloadCountService;
2425
import org.eclipse.openvsx.util.TempFile;
2526
import org.eclipse.openvsx.util.UrlUtil;
2627
import org.springframework.beans.factory.annotation.Value;
@@ -52,7 +53,7 @@ public class StorageUtilService implements IStorageService {
5253
private final AzureBlobStorageService azureStorage;
5354
private final LocalStorageService localStorage;
5455
private final AwsStorageService awsStorage;
55-
private final AzureDownloadCountService azureDownloadCountService;
56+
private final DownloadCountService downloadCountService;
5657
private final SearchUtilService search;
5758
private final CacheService cache;
5859
private final EntityManager entityManager;
@@ -73,7 +74,7 @@ public StorageUtilService(
7374
AzureBlobStorageService azureStorage,
7475
LocalStorageService localStorage,
7576
AwsStorageService awsStorage,
76-
AzureDownloadCountService azureDownloadCountService,
77+
DownloadCountService downloadCountService,
7778
SearchUtilService search,
7879
CacheService cache,
7980
EntityManager entityManager,
@@ -85,7 +86,7 @@ public StorageUtilService(
8586
this.azureStorage = azureStorage;
8687
this.localStorage = localStorage;
8788
this.awsStorage = awsStorage;
88-
this.azureDownloadCountService = azureDownloadCountService;
89+
this.downloadCountService = downloadCountService;
8990
this.search = search;
9091
this.cache = cache;
9192
this.entityManager = entityManager;
@@ -277,7 +278,7 @@ public Map<Long, Map<String, String>> getFileUrls(Collection<ExtensionVersion> e
277278

278279
@Transactional
279280
public void increaseDownloadCount(FileResource resource) {
280-
if(azureDownloadCountService.isEnabled()) {
281+
if(downloadCountService.isEnabled(resource)) {
281282
// don't count downloads twice
282283
return;
283284
}

0 commit comments

Comments
 (0)