Merge pull request #1100 from amvanbaren/remove-file-resources

Remove FileResource of type resource

Author: amvanbaren

server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java

@@ -318,25 +318,6 @@ public void getFileResources(ExtensionVersion extVersion, Consumer<TempFile> pro
         }
     }
 
-    public void processEachResource(ExtensionVersion extVersion, Consumer<TempFile> processor) {
-        readInputStream();
-        zipFile.stream()
-                .filter(zipEntry -> !zipEntry.isDirectory())
-                .forEach(zipEntry -> {
-                    try (var resourceFile = ArchiveUtil.readEntry(zipFile, zipEntry)) {
-                        var resource = new FileResource();
-                        resource.setExtension(extVersion);
-                        resource.setName(zipEntry.getName());
-                        resource.setType(FileResource.RESOURCE);
-                        resourceFile.setResource(resource);
-
-                        processor.accept(resourceFile);
-                    } catch (IOException | ErrorResultException exc) {
-                        logger.warn(exc.getMessage());
-                    }
-                });
-    }
-
     public FileResource getBinary(ExtensionVersion extVersion, String binaryName) {
         var binary = new FileResource();
         binary.setExtension(extVersion);

server/src/main/java/org/eclipse/openvsx/entities/FileResource.java

@@ -30,7 +30,6 @@ public class FileResource implements Serializable {
     public static final String README = "readme";
     public static final String LICENSE = "license";
     public static final String CHANGELOG = "changelog";
-    public static final String RESOURCE = "resource";
     public static final String VSIXMANIFEST = "vsixmanifest";
 
     // Storage types

server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java

@@ -43,7 +43,6 @@ public MigrationRunner(
     @Job(name = "Run migrations", retries = 0)
     public void run(HandlerJobRequest<?> jobRequest) throws Exception {
         orphanNamespaceMigration.fixOrphanNamespaces();
-        extractResourcesMigration();
         setPreReleaseMigration();
         renameDownloadsMigration();
         extractVsixManifestMigration();
@@ -53,12 +52,7 @@ public void run(HandlerJobRequest<?> jobRequest) throws Exception {
         checkPotentiallyMaliciousExtensionVersions();
         migrateLocalNamespaceLogos();
         migrateLocalFileResourceContent();
-    }
-
-    private void extractResourcesMigration() {
-        var jobName = "ExtractResourcesMigration";
-        var handler = ExtractResourcesJobRequestHandler.class;
-        repositories.findNotMigratedResources().forEach(item -> migrations.enqueueMigration(jobName, handler, item));
+        removeFileResourceTypeResource();
     }
 
     private void setPreReleaseMigration() {
@@ -114,4 +108,10 @@ private void migrateLocalFileResourceContent() {
         var handler = FileResourceContentJobRequestHandler.class;
         repositories.findNotMigratedLocalFileResourceContent().forEach(item -> migrations.enqueueMigration(jobName, handler, item));
     }
+
+    private void removeFileResourceTypeResource() {
+        var jobName = "RemoveFileResourceTypeResourceMigration";
+        var handler = RemoveFileResourceTypeResourceJobRequestHandler.class;
+        repositories.findNotMigratedFileResourceTypeResource().forEach(item -> migrations.enqueueMigration(jobName, handler, item));
+    }
 }

server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobService.java

@@ -0,0 +1,39 @@
+/** ******************************************************************************
+ * Copyright (c) 2025 Precies. Software OU and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ * ****************************************************************************** */
+package org.eclipse.openvsx.migration;
+
+import org.eclipse.openvsx.util.NamingUtil;
+import org.jobrunr.jobs.annotations.Job;
+import org.jobrunr.jobs.context.JobRunrDashboardLogger;
+import org.jobrunr.jobs.lambdas.JobRequestHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+@Component
+public class RemoveFileResourceTypeResourceJobRequestHandler implements JobRequestHandler<MigrationJobRequest> {
+
+    protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(RemoveFileResourceTypeResourceJobRequestHandler.class));
+
+    private final MigrationService migrations;
+
+    public RemoveFileResourceTypeResourceJobRequestHandler(MigrationService migrations) {
+        this.migrations = migrations;
+    }
+
+    @Override
+    @Job(name = "Remove FileResource of type 'resource'", retries = 3)
+    public void run(MigrationJobRequest jobRequest) throws Exception {
+        var resource = migrations.getResource(jobRequest);
+        logger.info("Removing file resource: {} {}", NamingUtil.toLogFormat(resource.getExtension()), resource.getName());
+        migrations.removeFile(resource);
+        migrations.deleteFileResource(resource);
+    }
+}

server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java

@@ -233,7 +233,6 @@ public void publishAsync(TempFile extensionFile, ExtensionService extensionServi
                 }
             }
 
-            processor.processEachResource(extVersion, consumer);
             processor.getFileResources(extVersion, consumer);
             try (var sha256File = processor.generateSha256Checksum(extVersion)) {
                 consumer.accept(sha256File);

server/src/main/java/org/eclipse/openvsx/migration/RemoveFileResourceTypeResourceJobRequestHandler.java

@@ -58,27 +58,6 @@ public List<FileResource> findAll(Collection<Long> extensionIds, Collection<Stri
                 .map(this::toFileResource);
     }
 
-    public List<FileResource> findAllResources(ExtensionVersion extVersion, String prefix) {
-        return dsl.select(
-                    FILE_RESOURCE.ID,
-                    FILE_RESOURCE.EXTENSION_ID,
-                    FILE_RESOURCE.NAME,
-                    FILE_RESOURCE.TYPE,
-                    FILE_RESOURCE.STORAGE_TYPE
-                )
-                .from(FILE_RESOURCE)
-                .where(FILE_RESOURCE.TYPE.eq(FileResource.RESOURCE))
-                .and(FILE_RESOURCE.EXTENSION_ID.eq(extVersion.getId()))
-                .and(FILE_RESOURCE.NAME.startsWith(prefix))
-                .fetch()
-                .map(row -> {
-                    var fileResource = toFileResource(row);
-                    fileResource.setStorageType(row.get(FILE_RESOURCE.STORAGE_TYPE));
-                    fileResource.setExtension(extVersion);
-                    return fileResource;
-                });
-    }
-
     private FileResource toFileResource(Record row) {
         var extVersion = new ExtensionVersion();
         extVersion.setId(row.get(FILE_RESOURCE.EXTENSION_ID));

server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java

@@ -267,18 +267,10 @@ public Streamable<FileResource> findFilesByType(String type) {
     }
 
     public FileResource findFileByType(ExtensionVersion extVersion, String type) {
-        if(FileResource.RESOURCE.equals(type)) {
-            throw new IllegalArgumentException("There are multiple files of type: " + FileResource.RESOURCE);
-        }
-
         return fileResourceRepo.findByExtensionAndType(extVersion, type);
     }
 
     public FileResource findFileByType(String namespace, String extension, String targetPlatform, String version, String type) {
-        if(FileResource.RESOURCE.equals(type)) {
-            throw new IllegalArgumentException("There are multiple files of type: " + FileResource.RESOURCE);
-        }
-
         return fileResourceJooqRepo.findByType(namespace, extension, targetPlatform, version, type);
     }
 
@@ -414,10 +406,6 @@ public List<FileResource> findFileResourcesByExtensionVersionIdAndType(Collectio
         return fileResourceJooqRepo.findAll(extensionVersionIds, types);
     }
 
-    public List<FileResource> findResourceFileResources(ExtensionVersion extVersion, String prefix) {
-        return fileResourceJooqRepo.findAllResources(extVersion, prefix);
-    }
-
     public List<NamespaceMembership> findNamespaceMemberships(Collection<Long> namespaceIds) {
         return membershipJooqRepo.findAllByNamespaceId(namespaceIds);
     }
@@ -482,10 +470,6 @@ public int countVersions(Extension extension) {
         return extensionVersionRepo.countByExtension(extension);
     }
 
-    public Streamable<MigrationItem> findNotMigratedResources() {
-        return findNotMigratedItems("V1_23__FileResource_Extract_Resources.sql");
-    }
-
     public Streamable<MigrationItem> findNotMigratedPreReleases() {
         return findNotMigratedItems("V1_26__Extension_Set_PreRelease.sql");
     }
@@ -518,6 +502,10 @@ public Iterable<MigrationItem> findNotMigratedLocalFileResourceContent() {
         return findNotMigratedItems("V1_48__Local_Storage_FileResource.sql");
     }
 
+    public Iterable<MigrationItem> findNotMigratedFileResourceTypeResource() {
+        return findNotMigratedItems("V1_50_FileResource_Remove_Resource.sql");
+    }
+
     private Streamable<MigrationItem> findNotMigratedItems(String migrationScript) {
         return migrationItemRepo.findByMigrationScriptAndMigrationScheduledFalseOrderById(migrationScript);
     }

server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java

@@ -0,0 +1,10 @@
+-- don't run removed migration
+UPDATE migration_item SET migration_scheduled = TRUE WHERE migration_script = 'V1_23__FileResource_Extract_Resources.sql';
+
+INSERT INTO migration_item(id, migration_script, entity_id, migration_scheduled)
+SELECT nextval('hibernate_sequence'), 'V1_50_FileResource_Remove_Resource.sql', fr.id, FALSE
+FROM file_resource fr
+JOIN extension_version ev ON ev.id = fr.extension_id
+JOIN extension e ON e.id = ev.extension_id
+WHERE fr.type = 'resource'
+ORDER BY e.download_count DESC;