Add Long-Running Scan Infrastructure for Async External Scanners (#1565)

* fix broken extension icons on scan cards

* Fix line endings

* Add long running scans. Refactored publish checks

---------

Co-authored-by: Alejandro Rivera <alejandro.rivera1996@gmail.com>

Author: janbro

server/build.gradle

@@ -44,7 +44,8 @@ def versions = [
     gatling: '3.14.9',
     loki4j: '1.4.2',
     jedis: '6.2.0',
-    re2j: '1.7'
+    re2j: '1.7',
+    jsonpath: '2.9.0'
 ]
 ext['junit-jupiter.version'] = versions.junit
 java {
@@ -139,6 +140,7 @@ dependencies {
     implementation "org.apache.httpcomponents.client5:httpclient5"
     implementation "org.apache.tika:tika-core:${versions.tika}"
     implementation "com.google.re2j:re2j:${versions.re2j}"
+    implementation "com.jayway.jsonpath:json-path:${versions.jsonpath}"
     implementation "com.github.loki4j:loki-logback-appender:${versions.loki4j}"
     implementation "io.micrometer:micrometer-tracing"
     implementation "io.micrometer:micrometer-tracing-bridge-otel"

server/src/dev/resources/application.yml

@@ -172,28 +172,37 @@ ovsx:
       template: 'revoked-access-tokens.html'
   scanning:
     enabled: true
-  similarity:
-    enabled: true
-    enforced: true
-    levenshtein-threshold: 0.2
-    skip-verified-publishers: true
-    check-against-verified-only: true
-    exclude-owner-namespaces: true
-    new-extensions-only: true
-  secret-scanning:
-    enabled: true
-    rules-path: 'classpath:scanning/secret-scanning-custom-rules.yaml'
-    inline-suppressions: 'secret-scanner:ignore,gitleaks:allow,nosecret,@suppress-secret'
-    auto-generate-rules: true
-    force-regenerate-rules: false
-    generated-rules-path: '/tmp/secret-scanning-rules-gitleaks.yaml'
-    max-file-size-bytes: 5242880
-    max-entry-count: 50000
-    max-total-uncompressed-bytes: 524288000
-    max-findings: 200
-    max-line-length: 10000
-    long-line-no-space-threshold: 1000
-    keyword-context-chars: 100
-    log-allowlisted-value-preview-length: 10
-    timeout-seconds: 5
-    timeout-check-every-n-lines: 100
+    # Shared archive limits for all scanning checks (secret detection, blocklist, etc.)
+    max-archive-size-bytes: 1073741824   # 1 GB total archive limit
+    max-single-file-bytes: 268435456     # 256 MB per-file limit
+    max-entry-count: 100000              # Max ZIP entries to process
+    blocklist-check:
+      enabled: true
+      enforced: false
+    similarity:
+      enabled: true
+      enforced: false
+      similarity-threshold: 0.2
+      skip-if-publisher-verified: false
+      only-protect-verified-names: false
+      allow-similarity-to-own-names: true
+      only-check-new-extensions: true
+    secret-detection:
+      enabled: true
+      enforced: false
+      rules-path: 'classpath:scanning/secret-detection-custom-rules.yaml'
+      suppression-markers: 'secret-detector:ignore,gitleaks:allow,nosecret,@suppress-secret'
+      gitleaks:
+        auto-fetch: true
+        force-refresh: true
+        output-path: '/tmp/secret-detection-rules-gitleaks.yaml'
+        scheduled-refresh: true
+        refresh-cron: '0 0 3 * * *'  # Daily at 3 AM
+        skip-rule-ids: 'generic-api-key'  # Rule IDs that produce too many false positives
+      max-findings: 200
+      minified-line-threshold: 10000
+      long-line-no-space-threshold: 1000
+      regex-context-chars: 100
+      debug-preview-chars: 10
+      timeout-seconds: 10
+      timeout-check-interval: 100

server/src/main/java/org/eclipse/openvsx/ExtensionService.java

@@ -22,6 +22,7 @@
 import org.eclipse.openvsx.json.TargetPlatformVersionJson;
 import org.eclipse.openvsx.publish.PublishExtensionVersionHandler;
 import org.eclipse.openvsx.repositories.RepositoryService;
+import org.eclipse.openvsx.scanning.ExtensionScanPersistenceService;
 import org.eclipse.openvsx.scanning.ExtensionScanService;
 import org.eclipse.openvsx.search.SearchUtilService;
 import org.eclipse.openvsx.util.ErrorResultException;
@@ -56,6 +57,7 @@ public class ExtensionService {
     private final PublishExtensionVersionHandler publishHandler;
     private final JobRequestScheduler scheduler;
     private final ExtensionScanService scanService;
+    private final ExtensionScanPersistenceService scanPersistenceService;
 
     @Value("${ovsx.publishing.require-license:false}")
     boolean requireLicense;
@@ -70,7 +72,8 @@ public ExtensionService(
             CacheService cache,
             PublishExtensionVersionHandler publishHandler,
             JobRequestScheduler scheduler,
-            ExtensionScanService scanService
+            ExtensionScanService scanService,
+            ExtensionScanPersistenceService scanPersistenceService
     ) {
         this.entityManager = entityManager;
         this.repositories = repositories;
@@ -79,6 +82,7 @@ public ExtensionService(
         this.publishHandler = publishHandler;
         this.scheduler = scheduler;
         this.scanService = scanService;
+        this.scanPersistenceService = scanPersistenceService;
     }
 
     @Transactional
@@ -111,9 +115,8 @@ private ExtensionVersion publishVersionWithScan(InputStream content, PersonalAcc
             scanService.runValidation(scan, extensionFile, token.getUser());
 
             doPublish(extensionFile, null, token, TimeUtil.getCurrentUTC(), true);
-
-            scanService.runScan(scan, extensionFile, token.getUser());
 
+            // Publish async handles requesting the longrunning scans
             publishHandler.publishAsync(extensionFile, this, scan);
             var download = extensionFile.getResource();
             publishHandler.schedulePublicIdJob(extensionFile.getResource());
@@ -290,6 +293,10 @@ protected ResultJson deleteExtension(ExtensionVersion extVersion) {
     }
 
     private void removeExtensionVersion(ExtensionVersion extVersion) {
+        // Clean up any pending scan jobs for this extension version
+        // to prevent "file not found" errors after deletion
+        scanPersistenceService.deleteScansForExtensionVersion(extVersion.getId());
+        
         repositories.findFiles(extVersion).map(RemoveFileJobRequest::new).forEach(scheduler::enqueue);
         repositories.deleteFiles(extVersion);
         entityManager.remove(extVersion);

server/src/main/java/org/eclipse/openvsx/UserAPI.java

@@ -11,9 +11,12 @@
 
 import jakarta.servlet.http.HttpServletRequest;
 import org.eclipse.openvsx.eclipse.EclipseService;
+import org.eclipse.openvsx.entities.ExtensionVersion;
 import org.eclipse.openvsx.entities.NamespaceMembership;
+import org.eclipse.openvsx.entities.ScanStatus;
 import org.eclipse.openvsx.entities.UserData;
 import org.eclipse.openvsx.json.*;
+import org.eclipse.openvsx.repositories.ExtensionScanRepository;
 import org.eclipse.openvsx.repositories.RepositoryService;
 import org.eclipse.openvsx.security.CodedAuthException;
 import org.eclipse.openvsx.storage.StorageUtilService;
@@ -54,21 +57,24 @@ public class UserAPI {
     private final StorageUtilService storageUtil;
     private final LocalRegistryService local;
     private final ExtensionService extensions;
+    private final ExtensionScanRepository scanRepository;
 
     public UserAPI(
             RepositoryService repositories,
             UserService users,
             EclipseService eclipse,
             StorageUtilService storageUtil,
             LocalRegistryService local,
-            ExtensionService extensions
+            ExtensionService extensions,
+            ExtensionScanRepository scanRepository
     ) {
         this.repositories = repositories;
         this.users = users;
         this.eclipse = eclipse;
         this.storageUtil = storageUtil;
         this.local = local;
         this.extensions = extensions;
+        this.scanRepository = scanRepository;
     }
 
     @GetMapping(
@@ -208,10 +214,107 @@ public List<ExtensionJson> getOwnExtensions() {
                     json.setPreview(latest.isPreview());
                     json.setActive(latest.getExtension().isActive());
                     json.setFiles(fileUrls.get(latest.getId()));
+                    
+                    // Add scan/review status information
+                    enrichWithReviewStatus(json, latest);
+                    
                     return json;
                 })
                 .toList();
     }
+    
+    /**
+     * Add review/scan status information to the extension JSON.
+     * 
+     * This shows users the current state of their extension in simple terms:
+     * - "published" - Extension is active and publicly available
+     * - "under_review" - Extension is being reviewed (validation, scanning, etc.)
+     * - "rejected" - Extension was blocked (quarantined or rejected)
+     */
+    private void enrichWithReviewStatus(ExtensionJson json, ExtensionVersion extVersion) {
+        // Look up scan by extension metadata (namespace, name, version, platform)
+        var ext = extVersion.getExtension();
+        var scanResult = scanRepository.findFirstByNamespaceNameAndExtensionNameAndExtensionVersionAndTargetPlatformOrderByStartedAtDesc(
+            ext.getNamespace().getName(),
+            ext.getName(),
+            extVersion.getVersion(),
+            extVersion.getTargetPlatform()
+        );
+
+        if (Boolean.TRUE.equals(json.getActive())) {
+            // Only mark published if scan result indicates PASSED or no scan result exists (scanning disabled / manual activation)
+            if (scanResult == null || scanResult.getStatus() == ScanStatus.PASSED) {
+                json.setReviewStatus("published");
+                return;
+            }
+        }
+        
+        if (scanResult == null) {
+            // No scan result found - show as under review
+            json.setReviewStatus("under_review");
+            json.setReviewMessage("Your extension is being reviewed.");
+            return;
+        }
+        
+        // Map internal status to simple user-facing status
+        switch (scanResult.getStatus()) {
+            case STARTED:
+                json.setReviewStatus("under_review");
+                json.setReviewMessage("Your extension is being reviewed.");
+                break;
+            case VALIDATING:
+                json.setReviewStatus("under_review");
+                json.setReviewMessage("Your extension is being reviewed.");
+                break;
+            case SCANNING:
+                json.setReviewStatus("under_review");
+                json.setReviewMessage("Your extension is being reviewed.");
+                break;
+            case QUARANTINED:
+                // Check if admin has made a decision on this quarantined scan
+                var adminDecision = repositories.findAdminScanDecisionByScanId(scanResult.getId());
+                if (adminDecision != null) {
+                    if (adminDecision.isAllowed()) {
+                        // Admin allowed the extension - show as published if active
+                        if (Boolean.TRUE.equals(json.getActive())) {
+                            json.setReviewStatus("published");
+                        } else {
+                            // Allowed but not yet active (edge case)
+                            json.setReviewStatus("under_review");
+                            json.setReviewMessage("Your extension has been approved and will be published shortly.");
+                        }
+                    } else {
+                        // Admin blocked the extension
+                        json.setReviewStatus("under_review");
+                        json.setReviewMessage("Your extension is being reviewed. Please contact support for details.");
+                    }
+                } else {
+                    // No admin decision yet - still under review
+                    json.setReviewStatus("under_review");
+                    if (scanResult.getErrorMessage() != null) {
+                        json.setReviewMessage(scanResult.getErrorMessage());
+                    } else {
+                        json.setReviewMessage("Your extension is being reviewed. Please contact support for details.");
+                    }
+                }
+                break;
+            case REJECTED:
+                json.setReviewStatus("rejected");
+                if (scanResult.getErrorMessage() != null) {
+                    json.setReviewMessage(scanResult.getErrorMessage());
+                } else {
+                    json.setReviewMessage("Your extension could not be published. Please contact support for details.");
+                }
+                break;
+            case ERRORED:
+                json.setReviewStatus("under_review");
+                json.setReviewMessage("Your extension could not be published. Please contact support for details.");
+                break;
+            default:
+                json.setReviewStatus("under_review");
+                json.setReviewMessage("Your extension is being reviewed. Please contact support for details.");
+        }
+    }
 
     @GetMapping(
             path = "/user/extension/{namespaceName}/{extensionName}",

server/src/main/java/org/eclipse/openvsx/admin/AdminService.java

@@ -22,6 +22,7 @@
 import org.eclipse.openvsx.mail.MailService;
 import org.eclipse.openvsx.migration.HandlerJobRequest;
 import org.eclipse.openvsx.repositories.RepositoryService;
+import org.eclipse.openvsx.scanning.ExtensionScanPersistenceService;
 import org.eclipse.openvsx.search.SearchUtilService;
 import org.eclipse.openvsx.storage.StorageUtilService;
 import org.eclipse.openvsx.util.*;
@@ -52,6 +53,7 @@ public class AdminService {
     private final CacheService cache;
     private final JobRequestScheduler scheduler;
     private final MailService mail;
+    private final ExtensionScanPersistenceService scanPersistenceService;
 
     public AdminService(
             RepositoryService repositories,
@@ -64,7 +66,8 @@ public AdminService(
             StorageUtilService storageUtil,
             CacheService cache,
             JobRequestScheduler scheduler,
-            MailService mail
+            MailService mail,
+            ExtensionScanPersistenceService scanPersistenceService
     ) {
         this.repositories = repositories;
         this.extensions = extensions;
@@ -77,6 +80,7 @@ public AdminService(
         this.cache = cache;
         this.scheduler = scheduler;
         this.mail = mail;
+        this.scanPersistenceService = scanPersistenceService;
     }
 
     @EventListener
@@ -240,6 +244,10 @@ protected ResultJson deleteExtension(ExtensionVersion extVersion, UserData admin
     }
 
     private void removeExtensionVersion(ExtensionVersion extVersion) {
+        // Clean up any pending scan jobs for this extension version
+        // to prevent "file not found" errors after deletion
+        scanPersistenceService.deleteScansForExtensionVersion(extVersion.getId());
+        
         repositories.findFiles(extVersion).map(RemoveFileJobRequest::new).forEach(scheduler::enqueue);
         repositories.deleteFiles(extVersion);
         entityManager.remove(extVersion);