From 2603a67c99e48210793c925ec39220f008c7b16d Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Tue, 4 Feb 2025 20:36:51 +0200 Subject: [PATCH] Remove FileResource of type resource --- .../eclipse/openvsx/ExtensionProcessor.java | 19 ------ .../openvsx/entities/FileResource.java | 1 - .../ExtractResourcesJobRequestHandler.java | 64 ------------------- .../migration/ExtractResourcesJobService.java | 35 ---------- .../openvsx/migration/MigrationRunner.java | 14 ++-- ...ResourceTypeResourceJobRequestHandler.java | 39 +++++++++++ .../PublishExtensionVersionHandler.java | 1 - .../FileResourceJooqRepository.java | 21 ------ .../repositories/RepositoryService.java | 20 ++---- .../V1_50_FileResource_Remove_Resource.sql | 10 +++ .../RepositoryServiceSmokeTest.java | 5 +- 11 files changed, 62 insertions(+), 167 deletions(-) delete mode 100644 server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java delete mode 100644 server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobService.java create mode 100644 server/src/main/java/org/eclipse/openvsx/migration/RemoveFileResourceTypeResourceJobRequestHandler.java create mode 100644 server/src/main/resources/db/migration/V1_50_FileResource_Remove_Resource.sql diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java index 9bf17f9ed..e14cfd3e7 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java @@ -318,25 +318,6 @@ public void getFileResources(ExtensionVersion extVersion, Consumer pro } } - public void processEachResource(ExtensionVersion extVersion, Consumer processor) { - readInputStream(); - zipFile.stream() - .filter(zipEntry -> !zipEntry.isDirectory()) - .forEach(zipEntry -> { - try (var resourceFile = ArchiveUtil.readEntry(zipFile, zipEntry)) { - var resource = new FileResource(); - resource.setExtension(extVersion); - resource.setName(zipEntry.getName()); - resource.setType(FileResource.RESOURCE); - resourceFile.setResource(resource); - - processor.accept(resourceFile); - } catch (IOException | ErrorResultException exc) { - logger.warn(exc.getMessage()); - } - }); - } - public FileResource getBinary(ExtensionVersion extVersion, String binaryName) { var binary = new FileResource(); binary.setExtension(extVersion); diff --git a/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java b/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java index 583678478..dfa7e5f0b 100644 --- a/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java +++ b/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java @@ -30,7 +30,6 @@ public class FileResource implements Serializable { public static final String README = "readme"; public static final String LICENSE = "license"; public static final String CHANGELOG = "changelog"; - public static final String RESOURCE = "resource"; public static final String VSIXMANIFEST = "vsixmanifest"; // Storage types diff --git a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java deleted file mode 100644 index 87fa811f9..000000000 --- a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java +++ /dev/null @@ -1,64 +0,0 @@ -/** ****************************************************************************** - * Copyright (c) 2022 Precies. Software and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - * ****************************************************************************** */ -package org.eclipse.openvsx.migration; - -import org.eclipse.openvsx.ExtensionProcessor; -import org.eclipse.openvsx.util.NamingUtil; -import org.jobrunr.jobs.annotations.Job; -import org.jobrunr.jobs.context.JobRunrDashboardLogger; -import org.jobrunr.jobs.lambdas.JobRequestHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -import java.nio.file.Files; - -@Component -@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true) -public class ExtractResourcesJobRequestHandler implements JobRequestHandler { - - protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(ExtractResourcesJobRequestHandler.class)); - - private final ExtractResourcesJobService service; - private final MigrationService migrations; - - public ExtractResourcesJobRequestHandler(ExtractResourcesJobService service, MigrationService migrations) { - this.service = service; - this.migrations = migrations; - } - - @Override - @Job(name = "Extract resources from published extension version", retries = 3) - public void run(MigrationJobRequest jobRequest) throws Exception { - var extVersion = migrations.getExtension(jobRequest.getEntityId()); - logger.info("Extracting resources for: {}", NamingUtil.toLogFormat(extVersion)); - - service.deleteResources(extVersion); - var download = migrations.getDownload(extVersion); - if(download == null) { - return; - } - - try(var extensionFile = migrations.getExtensionFile(download)) { - if(Files.size(extensionFile.getPath()) == 0) { - return; - } - try (var extProcessor = new ExtensionProcessor(extensionFile)) { - extProcessor.processEachResource(download.getExtension(), (tempFile) -> { - migrations.uploadFileResource(tempFile); - migrations.persistFileResource(tempFile.getResource()); - }); - } - } - - service.deleteWebResources(extVersion); - } -} diff --git a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobService.java b/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobService.java deleted file mode 100644 index 14a95db7d..000000000 --- a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobService.java +++ /dev/null @@ -1,35 +0,0 @@ -/** ****************************************************************************** - * Copyright (c) 2022 Precies. Software Ltd and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - * ****************************************************************************** */ -package org.eclipse.openvsx.migration; - -import jakarta.transaction.Transactional; -import org.eclipse.openvsx.entities.ExtensionVersion; -import org.eclipse.openvsx.repositories.RepositoryService; -import org.springframework.stereotype.Component; - -@Component -public class ExtractResourcesJobService { - - private final RepositoryService repositories; - - public ExtractResourcesJobService(RepositoryService repositories) { - this.repositories = repositories; - } - - @Transactional - public void deleteResources(ExtensionVersion extVersion) { - repositories.deleteFileResources(extVersion, "resource"); - } - - @Transactional - public void deleteWebResources(ExtensionVersion extVersion) { - repositories.deleteFileResources(extVersion, "web-resource"); - } -} diff --git a/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java b/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java index cbbc626ef..7dd108aee 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java @@ -43,7 +43,6 @@ public MigrationRunner( @Job(name = "Run migrations", retries = 0) public void run(HandlerJobRequest jobRequest) throws Exception { orphanNamespaceMigration.fixOrphanNamespaces(); - extractResourcesMigration(); setPreReleaseMigration(); renameDownloadsMigration(); extractVsixManifestMigration(); @@ -53,12 +52,7 @@ public void run(HandlerJobRequest jobRequest) throws Exception { checkPotentiallyMaliciousExtensionVersions(); migrateLocalNamespaceLogos(); migrateLocalFileResourceContent(); - } - - private void extractResourcesMigration() { - var jobName = "ExtractResourcesMigration"; - var handler = ExtractResourcesJobRequestHandler.class; - repositories.findNotMigratedResources().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); + removeFileResourceTypeResource(); } private void setPreReleaseMigration() { @@ -114,4 +108,10 @@ private void migrateLocalFileResourceContent() { var handler = FileResourceContentJobRequestHandler.class; repositories.findNotMigratedLocalFileResourceContent().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); } + + private void removeFileResourceTypeResource() { + var jobName = "RemoveFileResourceTypeResourceMigration"; + var handler = RemoveFileResourceTypeResourceJobRequestHandler.class; + repositories.findNotMigratedFileResourceTypeResource().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); + } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/RemoveFileResourceTypeResourceJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/RemoveFileResourceTypeResourceJobRequestHandler.java new file mode 100644 index 000000000..9faffc19c --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/migration/RemoveFileResourceTypeResourceJobRequestHandler.java @@ -0,0 +1,39 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Precies. Software OU and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.migration; + +import org.eclipse.openvsx.util.NamingUtil; +import org.jobrunr.jobs.annotations.Job; +import org.jobrunr.jobs.context.JobRunrDashboardLogger; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +@Component +public class RemoveFileResourceTypeResourceJobRequestHandler implements JobRequestHandler { + + protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(RemoveFileResourceTypeResourceJobRequestHandler.class)); + + private final MigrationService migrations; + + public RemoveFileResourceTypeResourceJobRequestHandler(MigrationService migrations) { + this.migrations = migrations; + } + + @Override + @Job(name = "Remove FileResource of type 'resource'", retries = 3) + public void run(MigrationJobRequest jobRequest) throws Exception { + var resource = migrations.getResource(jobRequest); + logger.info("Removing file resource: {} {}", NamingUtil.toLogFormat(resource.getExtension()), resource.getName()); + migrations.removeFile(resource); + migrations.deleteFileResource(resource); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java index 4f5288c2a..a6a23f4cb 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java @@ -233,7 +233,6 @@ public void publishAsync(TempFile extensionFile, ExtensionService extensionServi } } - processor.processEachResource(extVersion, consumer); processor.getFileResources(extVersion, consumer); try (var sha256File = processor.generateSha256Checksum(extVersion)) { consumer.accept(sha256File); diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java index 39833fc62..3aebdfb8c 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java @@ -58,27 +58,6 @@ public List findAll(Collection extensionIds, Collection findAllResources(ExtensionVersion extVersion, String prefix) { - return dsl.select( - FILE_RESOURCE.ID, - FILE_RESOURCE.EXTENSION_ID, - FILE_RESOURCE.NAME, - FILE_RESOURCE.TYPE, - FILE_RESOURCE.STORAGE_TYPE - ) - .from(FILE_RESOURCE) - .where(FILE_RESOURCE.TYPE.eq(FileResource.RESOURCE)) - .and(FILE_RESOURCE.EXTENSION_ID.eq(extVersion.getId())) - .and(FILE_RESOURCE.NAME.startsWith(prefix)) - .fetch() - .map(row -> { - var fileResource = toFileResource(row); - fileResource.setStorageType(row.get(FILE_RESOURCE.STORAGE_TYPE)); - fileResource.setExtension(extVersion); - return fileResource; - }); - } - private FileResource toFileResource(Record row) { var extVersion = new ExtensionVersion(); extVersion.setId(row.get(FILE_RESOURCE.EXTENSION_ID)); diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index 726c526df..d028647b3 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -267,18 +267,10 @@ public Streamable findFilesByType(String type) { } public FileResource findFileByType(ExtensionVersion extVersion, String type) { - if(FileResource.RESOURCE.equals(type)) { - throw new IllegalArgumentException("There are multiple files of type: " + FileResource.RESOURCE); - } - return fileResourceRepo.findByExtensionAndType(extVersion, type); } public FileResource findFileByType(String namespace, String extension, String targetPlatform, String version, String type) { - if(FileResource.RESOURCE.equals(type)) { - throw new IllegalArgumentException("There are multiple files of type: " + FileResource.RESOURCE); - } - return fileResourceJooqRepo.findByType(namespace, extension, targetPlatform, version, type); } @@ -414,10 +406,6 @@ public List findFileResourcesByExtensionVersionIdAndType(Collectio return fileResourceJooqRepo.findAll(extensionVersionIds, types); } - public List findResourceFileResources(ExtensionVersion extVersion, String prefix) { - return fileResourceJooqRepo.findAllResources(extVersion, prefix); - } - public List findNamespaceMemberships(Collection namespaceIds) { return membershipJooqRepo.findAllByNamespaceId(namespaceIds); } @@ -482,10 +470,6 @@ public int countVersions(Extension extension) { return extensionVersionRepo.countByExtension(extension); } - public Streamable findNotMigratedResources() { - return findNotMigratedItems("V1_23__FileResource_Extract_Resources.sql"); - } - public Streamable findNotMigratedPreReleases() { return findNotMigratedItems("V1_26__Extension_Set_PreRelease.sql"); } @@ -518,6 +502,10 @@ public Iterable findNotMigratedLocalFileResourceContent() { return findNotMigratedItems("V1_48__Local_Storage_FileResource.sql"); } + public Iterable findNotMigratedFileResourceTypeResource() { + return findNotMigratedItems("V1_50_FileResource_Remove_Resource.sql"); + } + private Streamable findNotMigratedItems(String migrationScript) { return migrationItemRepo.findByMigrationScriptAndMigrationScheduledFalseOrderById(migrationScript); } diff --git a/server/src/main/resources/db/migration/V1_50_FileResource_Remove_Resource.sql b/server/src/main/resources/db/migration/V1_50_FileResource_Remove_Resource.sql new file mode 100644 index 000000000..c9e5dd46a --- /dev/null +++ b/server/src/main/resources/db/migration/V1_50_FileResource_Remove_Resource.sql @@ -0,0 +1,10 @@ +-- don't run removed migration +UPDATE migration_item SET migration_scheduled = TRUE WHERE migration_script = 'V1_23__FileResource_Extract_Resources.sql'; + +INSERT INTO migration_item(id, migration_script, entity_id, migration_scheduled) +SELECT nextval('hibernate_sequence'), 'V1_50_FileResource_Remove_Resource.sql', fr.id, FALSE +FROM file_resource fr +JOIN extension_version ev ON ev.id = fr.extension_id +JOIN extension e ON e.id = ev.extension_id +WHERE fr.type = 'resource' +ORDER BY e.download_count DESC; \ No newline at end of file diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java index 6bfaaa39f..61d658a32 100644 --- a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java +++ b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java @@ -139,7 +139,6 @@ void testExecuteQueries() { () -> repositories.topMostDownloadedExtensions(1), () -> repositories.deleteFileResources(extVersion, "download"), () -> repositories.countActiveAccessTokens(userData), - () -> repositories.findNotMigratedResources(), () -> repositories.findNotMigratedPreReleases(), () -> repositories.findNotMigratedRenamedDownloads(), () -> repositories.findNotMigratedVsixManifests(), @@ -150,7 +149,6 @@ void testExecuteQueries() { () -> repositories.topNamespaceExtensions(1), () -> repositories.topNamespaceExtensionVersions(1), () -> repositories.findFileResourcesByExtensionVersionIdAndType(LONG_LIST, STRING_LIST), - () -> repositories.findResourceFileResources(extVersion, "prefix"), () -> repositories.findActiveExtensionVersions(LONG_LIST, "targetPlatform"), () -> repositories.findActiveExtension("name", "namespaceName"), () -> repositories.findActiveExtensionsById(LONG_LIST), @@ -228,7 +226,8 @@ void testExecuteQueries() { () -> repositories.findDeprecatedExtensions(extension), () -> repositories.findLatestReplacement(1L, null, false, false), () -> repositories.findNotMigratedLocalNamespaceLogos(), - () -> repositories.findNotMigratedLocalFileResourceContent() + () -> repositories.findNotMigratedLocalFileResourceContent(), + () -> repositories.findNotMigratedFileResourceTypeResource() ); // check that we did not miss anything