From 0035d128d821f6eb97c16ed1a9339234c05cc2d2 Mon Sep 17 00:00:00 2001 From: Muyang Ye Date: Tue, 5 Mar 2024 23:04:12 -0800 Subject: [PATCH] [#2181] Merge internalFilename and originalFilename (#2430) * Serialize non-primitive types and store in Influx * extract RawFieldSerializer * rename test * delete old * temp * redesign file api * fix * add comment * merge internalFilename and originalFilename * delete useless * fix bug * code review and unit test * Delete debug logs Co-authored-by: Tim <50115603+bossenti@users.noreply.github.com> * add log * undo --------- Co-authored-by: Tim <50115603+bossenti@users.noreply.github.com> --- .../dataimport/PerformImportGenerator.java | 4 +- .../dataimport/PreviewImportGenerator.java | 2 +- .../generator/ExportPackageGenerator.java | 2 +- .../export/resolver/FileResolver.java | 2 +- .../streampipes/model/file/FileMetadata.java | 19 +- .../streampipes/manager/file/FileHandler.java | 20 ++ .../streampipes/manager/file/FileManager.java | 22 +- .../rest/impl/PipelineElementFile.java | 2 +- .../core/migrations/AvailableMigrations.java | 4 +- .../v095/DuplicateFilesRenameMigration.java | 142 ----------- ...FilenamesAndRenameDuplicatesMigration.java | 228 ++++++++++++++++++ .../DuplicateFilesRenameMigrationTest.java | 120 --------- ...namesAndRenameDuplicatesMigrationTest.java | 106 ++++++++ .../src/lib/apis/files.service.ts | 2 +- .../src/lib/model/gen/streampipes-model.ts | 6 +- .../dialog/base-asset-links.directive.ts | 2 +- .../manage-asset-links-dialog.component.html | 2 +- .../manage-asset-links-dialog.component.ts | 2 +- .../static-file-input.component.html | 2 +- .../static-file-input.component.ts | 87 +++---- .../file-overview.component.html | 2 +- .../file-overview/file-overview.component.ts | 8 +- .../file-rename-dialog.component.html | 4 +- .../file-upload-dialog.component.ts | 2 +- 24 files changed, 426 insertions(+), 366 deletions(-) delete mode 100644 streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java create mode 100644 streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/MergeFilenamesAndRenameDuplicatesMigration.java delete mode 100644 streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java create mode 100644 streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/MergeFilenamesAndRenameDuplicatesMigrationTest.java diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java index d35b8dbf1c..f056e0fefb 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java @@ -135,8 +135,8 @@ protected void handleFile(String document, var fileMetadata = resolver.readDocument(document); resolver.writeDocument(document); byte[] file = zipContent.get( - fileMetadata.getInternalFilename().substring(0, fileMetadata.getInternalFilename().lastIndexOf("."))); - new FileHandler().storeFile(fileMetadata.getInternalFilename(), new ByteArrayInputStream(file)); + fileMetadata.getFilename().substring(0, fileMetadata.getFilename().lastIndexOf("."))); + new FileHandler().storeFile(fileMetadata.getFilename(), new ByteArrayInputStream(file)); } @Override diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java index 57ff84f809..55d3acf43e 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java @@ -116,7 +116,7 @@ protected void handleDataViewWidget(String document, String dataViewWidget) { protected void handleFile(String document, String fileMetadataId, Map zipContent) throws JsonProcessingException { - addExportItem(fileMetadataId, new FileResolver().readDocument(document).getOriginalFilename(), + addExportItem(fileMetadataId, new FileResolver().readDocument(document).getFilename(), importConfig::addFile); } diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java index e2365930ce..fc3663929f 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java @@ -121,7 +121,7 @@ public byte[] generateExportPackage() throws IOException { config.getFiles().forEach(item -> { var fileResolver = new FileResolver(); - String filename = fileResolver.findDocument(item.getResourceId()).getInternalFilename(); + String filename = fileResolver.findDocument(item.getResourceId()).getFilename(); addDoc(builder, item, new FileResolver(), manifest::addFile); try { builder.addBinary(filename, Files.readAllBytes(FileManager.getFile(filename).toPath())); diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java index a1493c9865..d30d66936a 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java @@ -44,7 +44,7 @@ public FileMetadata readDocument(String serializedDoc) throws JsonProcessingExce @Override public ExportItem convert(FileMetadata document) { - return new ExportItem(document.getFileId(), document.getOriginalFilename(), true); + return new ExportItem(document.getFileId(), document.getFilename(), true); } @Override diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java b/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java index 656ea6861c..f4ee123ecc 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java @@ -28,8 +28,7 @@ public class FileMetadata { private @SerializedName("_rev") String rev; - private String internalFilename; - private String originalFilename; + private String filename; private String filetype; private long createdAt; @@ -53,20 +52,12 @@ public void setRev(String rev) { this.rev = rev; } - public String getInternalFilename() { - return internalFilename; + public String getFilename() { + return filename; } - public void setInternalFilename(String internalFilename) { - this.internalFilename = internalFilename; - } - - public String getOriginalFilename() { - return originalFilename; - } - - public void setOriginalFilename(String originalFilename) { - this.originalFilename = originalFilename; + public void setFilename(String filename) { + this.filename = filename; } public long getCreatedAt() { diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileHandler.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileHandler.java index b5403a6f94..943da3d95f 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileHandler.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileHandler.java @@ -18,13 +18,19 @@ package org.apache.streampipes.manager.file; import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; public class FileHandler { + Logger logger = LoggerFactory.getLogger(FileHandler.class); + public void storeFile(String filename, InputStream fileInputStream) throws IOException { File targetFile = makeFile(filename); FileUtils.copyInputStreamToFile(fileInputStream, targetFile); @@ -39,6 +45,20 @@ public File getFile(String filename) { return FileUtils.getFile(makeFile(filename)); } + public void renameFile(String oldFilename, String newFilename) { + try { + var fileInputStream = new FileInputStream(getFile(oldFilename)); + deleteFile(oldFilename); + storeFile(newFilename, fileInputStream); + } catch (FileNotFoundException e) { + logger.error( + "Failed to find the old file locally with internalFilename as the identifier, this is most likely a mismatch " + + "between local file and FileMetadata stored in CouchDB. Raw exception message: " + e.getMessage()); + } catch (IOException e) { + logger.error("Failed to save renamed file locally: " + e.getMessage()); + } + } + private File makeFile(String filename) { File fileDir = new File(makeFileLocation()); if (!fileDir.exists()) { diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java index fa860de9f4..20fece68b9 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java @@ -29,7 +29,6 @@ import java.io.InputStream; import java.util.Arrays; import java.util.List; -import java.util.UUID; import java.util.stream.Collectors; public class FileManager { @@ -48,13 +47,13 @@ public static File getFileByOriginalName(String originalName) throws IOException var file = allFiles .stream() - .filter(fileMetadata -> fileMetadata.getOriginalFilename().equals(originalName)) + .filter(fileMetadata -> fileMetadata.getFilename().equals(originalName)) .findFirst(); if (file.isEmpty()){ throw new IOException("No file with original name '%s' found".formatted(originalName)); } - return new FileHandler().getFile(file.get().getInternalFilename()); + return new FileHandler().getFile(file.get().getFilename()); } /** @@ -74,16 +73,15 @@ public static FileMetadata storeFile(String user, fileInputStream = cleanFile(fileInputStream, filetype); - String internalFilename = makeInternalFilename(filetype); - FileMetadata fileMetadata = makeFileMetadata(user, filename, internalFilename, filetype); - new FileHandler().storeFile(internalFilename, fileInputStream); + FileMetadata fileMetadata = makeFileMetadata(user, filename, filetype); + new FileHandler().storeFile(filename, fileInputStream); storeFileMetadata(fileMetadata); return fileMetadata; } public static void deleteFile(String id) { FileMetadata fileMetadata = getFileMetadataStorage().getMetadataById(id); - new FileHandler().deleteFile(fileMetadata.getInternalFilename()); + new FileHandler().deleteFile(fileMetadata.getFilename()); getFileMetadataStorage().deleteFileMetadata(id); } @@ -118,24 +116,18 @@ private static IFileMetadataStorage getFileMetadataStorage() { } private static FileMetadata makeFileMetadata(String user, - String originalFilename, - String internalFilename, + String filename, String filetype) { FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setCreatedAt(System.currentTimeMillis()); fileMetadata.setCreatedByUser(user); fileMetadata.setFiletype(filetype); - fileMetadata.setInternalFilename(internalFilename); - fileMetadata.setOriginalFilename(originalFilename); + fileMetadata.setFilename(filename); return fileMetadata; } - private static String makeInternalFilename(String filetype) { - return UUID.randomUUID() + "." + filetype; - } - private static List filterFiletypes(List allFiles, String filetypes) { return allFiles .stream() diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java index 2a71a5a0a0..bcdfb5e5e8 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java @@ -135,7 +135,7 @@ private byte[] getFileContents(File file) throws IOException { public ResponseEntity> getAllOriginalFilenames() { return ok(FileManager.getAllFiles() .stream() - .map(fileMetadata -> fileMetadata.getOriginalFilename() + .map(fileMetadata -> fileMetadata.getFilename() .toLowerCase()) .toList()); } diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java index be23f8f0d5..2ee78fea02 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java @@ -25,7 +25,7 @@ import org.apache.streampipes.service.core.migrations.v090.UpdateUsernameViewMigration; import org.apache.streampipes.service.core.migrations.v093.AdapterMigration; import org.apache.streampipes.service.core.migrations.v093.StoreEmailTemplatesMigration; -import org.apache.streampipes.service.core.migrations.v095.DuplicateFilesRenameMigration; +import org.apache.streampipes.service.core.migrations.v095.MergeFilenamesAndRenameDuplicatesMigration; import java.util.Arrays; import java.util.List; @@ -40,7 +40,7 @@ public List getAvailableMigrations() { new UpdateUsernameViewMigration(), new AdapterMigration(), new StoreEmailTemplatesMigration(), - new DuplicateFilesRenameMigration() + new MergeFilenamesAndRenameDuplicatesMigration() ); } } diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java deleted file mode 100644 index e5e1f76470..0000000000 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.service.core.migrations.v095; - -import org.apache.streampipes.model.file.FileMetadata; -import org.apache.streampipes.service.core.migrations.Migration; -import org.apache.streampipes.storage.management.StorageDispatcher; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -public class DuplicateFilesRenameMigration implements Migration { - @Override - public boolean shouldExecute() { - return true; - } - - // Starting from v0.95, StreamPipes will use file name as the unique identifier of files - // This migration renames all the files that have duplicate names to ensure uniqueness - @Override - public void executeMigration() { - var fileMetadataStorage = StorageDispatcher.INSTANCE.getNoSqlStore() - .getFileMetadataStorage(); - - var filesToUpdate = getFilesToUpdate(fileMetadataStorage.getAllFileMetadataDescriptions()); - - filesToUpdate.forEach(fileMetadata -> fileMetadataStorage.updateFileMetadata(fileMetadata)); - } - - - /** - * Takes the list of files and groups all files with the same name together. - * The result is a list for each file name that has more than one file associated with it. - */ - private List> getListsOfFilesWithSameName(List filesWithOldName) { - var duplicateFileMap = filesWithOldName.stream() - .collect( - Collectors.groupingBy(file -> file.getOriginalFilename() - .toLowerCase())); - return duplicateFileMap.values() - .stream() - .filter(files -> files.size() > 1) - .toList(); - } - - @Override - public String getDescription() { - return "Rename files with duplicate names."; - } - - /** - * Takes the files searches for duplicates and renames them and returns the files that must be updated - */ - protected List getFilesToUpdate(List filesWithOldName) { - - var groupsOfFilesWithSameName = getListsOfFilesWithSameName(filesWithOldName); - - return groupsOfFilesWithSameName.stream() - .flatMap(filesWithSameName -> - renameFilesWithSameName(filesWithSameName).stream()) - .collect(Collectors.toList()); - } - - - /** - * Takes a list of files with the same name renames them and returns the updated list - */ - private List renameFilesWithSameName(List filesWithSameName) { - return IntStream.range(1, filesWithSameName.size()) - .mapToObj(i -> { - var metadata = filesWithSameName.get(i); - return renameFile(metadata, i); - }) - .toList(); - } - - - /** - * Takes a file and renames it with a new name based the number of occurrences of the file name - */ - private FileMetadata renameFile(FileMetadata fileMetadata, int index) { - var oldFilename = fileMetadata.getOriginalFilename(); - - var fileNameWithoutType = removeFileType(oldFilename); - var fileTypeSuffix = getFileType(oldFilename); - - var newFileName = createNewFileName(index, fileNameWithoutType, fileTypeSuffix); - - fileMetadata.setOriginalFilename(newFileName); - - return fileMetadata; - } - - /** - * Creates the new file name for a file with a duplicate name. - */ - private String createNewFileName( - int index, - String fileName, - String fileType - ) { - return String.format( - "%s(%d)%s", - fileName, - index + 1, - fileType - ); - } - - /** - * Returns file name without file type suffix. - */ - private String removeFileType(String fileName) { - var indexBeforeFileType = fileName.lastIndexOf('.'); - return fileName.substring(0, indexBeforeFileType); - } - - /** - * Returns the file type a given file name. - */ - private String getFileType(String fileName) { - var indexBeforeFileType = fileName.lastIndexOf('.'); - return fileName.substring(indexBeforeFileType); - } -} diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/MergeFilenamesAndRenameDuplicatesMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/MergeFilenamesAndRenameDuplicatesMigration.java new file mode 100644 index 0000000000..1330a109ce --- /dev/null +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/MergeFilenamesAndRenameDuplicatesMigration.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.service.core.migrations.v095; + +import org.apache.streampipes.manager.file.FileHandler; +import org.apache.streampipes.model.file.FileMetadata; +import org.apache.streampipes.service.core.migrations.Migration; +import org.apache.streampipes.storage.api.IFileMetadataStorage; +import org.apache.streampipes.storage.couchdb.utils.Utils; +import org.apache.streampipes.storage.management.StorageDispatcher; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.lightcouch.CouchDbClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; + +public class MergeFilenamesAndRenameDuplicatesMigration implements Migration { + + protected static final String ORIGINAL_FILENAME = "originalFilename"; + protected static final String INTERNAL_FILENAME = "internalFilename"; + protected static final String ID = "_id"; + protected static final String FILETYPE = "filetype"; + + private CouchDbClient couchDbClient; + + private ObjectMapper mapper = new ObjectMapper(); + + private IFileMetadataStorage fileMetadataStorage = + StorageDispatcher.INSTANCE.getNoSqlStore().getFileMetadataStorage(); + + private FileHandler fileHandler = new FileHandler(); + + Logger logger = LoggerFactory.getLogger(MergeFilenamesAndRenameDuplicatesMigration.class); + + protected Map> fileMetadataGroupedByOriginalName = new HashMap<>(); + + private boolean isTesting = false; + + public MergeFilenamesAndRenameDuplicatesMigration(boolean testing) { + isTesting = testing; + } + + public MergeFilenamesAndRenameDuplicatesMigration() { + couchDbClient = Utils.getCouchDbFileMetadataClient(); + } + + // Starting from v0.95, StreamPipes will use a single file name as the unique identifier of files instead of an + // internal filename and an original filename. This migration merges them and renames all the files that have + // duplicate names to ensure uniqueness + @Override + public boolean shouldExecute() { + return true; + } + + @Override + public void executeMigration() { + var couchDbRawFileMetadata = getCouchDbRawFileMetadata(getAllFileIds(fileMetadataStorage)); + getFileMetadataToUpdate(couchDbRawFileMetadata); + fileMetadataGroupedByOriginalName.forEach( + (originalFilename, fileMetadataList) -> update(originalFilename, fileMetadataList)); + } + + /** + * Gets all fileMetadata that need to be updated grouped by originalFilename + * key is (possibly) duplicated originalFilename and value is that file's FileMetadata list (if duplicated) + */ + protected void getFileMetadataToUpdate(List> couchDbRawFileMetadata) { + couchDbRawFileMetadata.forEach( + rawFileMetadata -> checkDuplicateOriginalFilename(rawFileMetadata)); + } + + /** + * Fetches all fileIds stored in CouchDB + */ + private List getAllFileIds(IFileMetadataStorage fileMetadataStorage) { + return fileMetadataStorage.getAllFileMetadataDescriptions().stream().map(fileMetadata -> fileMetadata.getFileId()) + .toList(); + } + + /** + * Takes the list of fileIds and searches for their raw metadata in CouchDB and returns them + */ + private List> getCouchDbRawFileMetadata(List fileIds) { + return fileIds.stream() + .map(fileId -> convertInputStreamToMap(couchDbClient.find(fileId))) + .toList(); + } + + /** + * Converts InputStream (as stored in CouchDB) to Map, if there's an error, constructs a new Map + */ + private Map convertInputStreamToMap(InputStream inputStream) { + try { + return mapper.readValue(inputStream, Map.class); + } catch (Exception e) { + Scanner scanner = new Scanner(inputStream).useDelimiter("\\A"); + String inputStreamString = scanner.hasNext() ? scanner.next() : ""; + logger.error( + "Failed to construct a Map from InputStream stored in CouchDB, the data for this file is likely corrupted, " + + "skipping it for migration.\nThe original debug message is: " + e.getMessage() + + "\nThe original InputStream is: " + inputStreamString); + return new HashMap<>(); + } + } + + /** + * Takes raw data stored in CouchDB and constructs fileMetadataGroupedByOriginalName, + * key is (possibly) duplicated originalFilename and value is that file's FileMetadata list (if duplicated) + */ + private void checkDuplicateOriginalFilename(Map rawFileMetadata) { + // If this file was already migrated or there was an error when converting InputStream to Map, skip it + if (rawFileMetadata.containsKey(ORIGINAL_FILENAME)) { + var originalFilename = rawFileMetadata.get(ORIGINAL_FILENAME).toString().toLowerCase(); + if (!fileMetadataGroupedByOriginalName.containsKey(originalFilename)) { + fileMetadataGroupedByOriginalName.put(originalFilename, new ArrayList<>()); + } + FileMetadata fileMetadata; + if (isTesting) { + fileMetadata = new FileMetadata(); + fileMetadata.setFileId(rawFileMetadata.get(ID).toString()); + fileMetadata.setFiletype(rawFileMetadata.get(FILETYPE).toString()); + } else { + fileMetadata = fileMetadataStorage.getMetadataById(rawFileMetadata.get(ID).toString()); + } + fileMetadataGroupedByOriginalName.get(originalFilename).add(fileMetadata); + } + } + + /** + * For each of the file, calls updateFileMetadata() and updateLocalFile() + */ + protected void update(String originalFilename, List fileMetadataList) { + var fileMetadata = fileMetadataList.get(0); + // just name the 1st one to its originalFilename + if (!isTesting) { + var internalFilename = getInternalFilenameFromFileMetadata(fileMetadata); + updateLocalFile(internalFilename, originalFilename); + } + updateFileMetadata(fileMetadata, originalFilename, isTesting); + for (int i = 1; i < fileMetadataList.size(); ++i) { + fileMetadata = fileMetadataList.get(i); + var newFilename = createNewFileName(i, removeFileType(originalFilename), fileMetadata.getFiletype()); + if (!isTesting) { + var internalFilename = getInternalFilenameFromFileMetadata(fileMetadata); + updateLocalFile(internalFilename, newFilename); + } + updateFileMetadata(fileMetadata, newFilename, isTesting); + } + } + + /** + * Updates FileMetadata: sets new merged filename to the given filename + */ + private void updateFileMetadata(FileMetadata fileMetadata, String filename, boolean isTesting) { + fileMetadata.setFilename(filename); + if (!isTesting) { + fileMetadataStorage.updateFileMetadata(fileMetadata); + } + } + + /** + * Updates the file stored locally: renames the file (i.e. before it's using internalFilename, now copy the + * InputStream stored and replace internalFilename with the given filename) + */ + private void updateLocalFile(String internalFilename, String filename) { + fileHandler.renameFile(internalFilename, filename); + } + + /** + * Gets the old internalFilename after merging + */ + private String getInternalFilenameFromFileMetadata(FileMetadata fileMetadata) { + return convertInputStreamToMap(couchDbClient.find(fileMetadata.getFileId())).get(INTERNAL_FILENAME).toString(); + } + + /** + * Creates the new file name for a file with a duplicate name. + */ + private String createNewFileName( + int index, + String fileName, + String fileType + ) { + return String.format( + "%s(%d).%s", + fileName, + index + 1, + fileType + ); + } + + /** + * Returns file name without file type suffix. + */ + private String removeFileType(String fileName) { + var indexBeforeFileType = fileName.lastIndexOf('.'); + return fileName.substring(0, indexBeforeFileType); + } + + @Override + public String getDescription() { + return "Merge internalFilename and originalFilename. Additionally, rename" + + "duplicate files to ensure uniqueness."; + } +} diff --git a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java deleted file mode 100644 index 18b35846f6..0000000000 --- a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.service.core.migrations.v095; - - -import org.apache.streampipes.model.file.FileMetadata; - -import org.junit.Before; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class DuplicateFilesRenameMigrationTest { - - private DuplicateFilesRenameMigration migration; - - private static final String FILE_NAME = "file.txt"; - private static final String NEW_FILE_NAME_2 = "file(2).txt"; - private static final String NEW_FILE_NAME_3 = "file(3).txt"; - - @Before - public void setUp() { - migration = new DuplicateFilesRenameMigration(); - } - - @Test - public void getFilesToUpdateHandlesNoDuplicates() { - var filesWithOldName = createFiles(List.of( - FILE_NAME - )); - var result = migration.getFilesToUpdate(filesWithOldName); - assertTrue(result.isEmpty()); - } - - @Test - public void getFilesToUpdateHandlesSingleDuplicate() { - var filesWithOldName = createFiles(List.of( - FILE_NAME, - FILE_NAME - )); - var result = migration.getFilesToUpdate(filesWithOldName); - assertEquals(1, result.size()); - assertEquals( - NEW_FILE_NAME_2, - result.get(0) - .getOriginalFilename() - ); - } - - @Test - public void getFilesToUpdateHandlesMultipleDuplicates() { - var filesWithOldName = createFiles(List.of( - FILE_NAME, - FILE_NAME, - FILE_NAME - )); - var result = migration.getFilesToUpdate(filesWithOldName); - assertEquals(2, result.size()); - assertEquals( - NEW_FILE_NAME_2, - result.get(0) - .getOriginalFilename() - ); - assertEquals( - NEW_FILE_NAME_3, - result.get(1) - .getOriginalFilename() - ); - } - - @Test - public void getFilesToUpdateHandlesMixedDuplicatesAndUniqueFiles() { - var filesWithOldName = createFiles(List.of( - FILE_NAME, - FILE_NAME, - "unique.txt" - )); - var result = migration.getFilesToUpdate(filesWithOldName); - assertEquals(1, result.size()); - assertEquals( - NEW_FILE_NAME_2, - result.get(0) - .getOriginalFilename() - ); - } - - /** - * Creates a list of FileMetadata objects from a list of file names. - */ - private List createFiles(List fileNames) { - return fileNames.stream() - .map(this::createFileMetadata) - .toList(); - } - - private FileMetadata createFileMetadata(String fileName) { - var fileMetadata = new FileMetadata(); - fileMetadata.setOriginalFilename(fileName); - return fileMetadata; - } -} \ No newline at end of file diff --git a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/MergeFilenamesAndRenameDuplicatesMigrationTest.java b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/MergeFilenamesAndRenameDuplicatesMigrationTest.java new file mode 100644 index 0000000000..b11fca54d0 --- /dev/null +++ b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/MergeFilenamesAndRenameDuplicatesMigrationTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.service.core.migrations.v095; + +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.streampipes.service.core.migrations.v095.MergeFilenamesAndRenameDuplicatesMigration.FILETYPE; +import static org.apache.streampipes.service.core.migrations.v095.MergeFilenamesAndRenameDuplicatesMigration.ID; +import static org.apache.streampipes.service.core.migrations.v095.MergeFilenamesAndRenameDuplicatesMigration.INTERNAL_FILENAME; +import static org.apache.streampipes.service.core.migrations.v095.MergeFilenamesAndRenameDuplicatesMigration.ORIGINAL_FILENAME; +import static org.junit.Assert.assertEquals; + +public class MergeFilenamesAndRenameDuplicatesMigrationTest { + + private static final Map RAW_FILEMETADATA_1 = new HashMap() { + { + put(ID, "id1"); + put(ORIGINAL_FILENAME, "file.txt"); + put(INTERNAL_FILENAME, "doesn't matter"); + put(FILETYPE, "txt"); + } + }; + + private static final Map RAW_FILEMETADATA_2 = new HashMap() { + { + put(ID, "id2"); + put(ORIGINAL_FILENAME, "FILE.txt"); + put(INTERNAL_FILENAME, "doesn't matter"); + put(FILETYPE, "TXT"); + } + }; + + private static final Map RAW_FILEMETADATA_3 = new HashMap() { + { + put(ID, "id2"); + put(ORIGINAL_FILENAME, "fIlE.TxT"); + put(INTERNAL_FILENAME, "doesn't matter"); + put(FILETYPE, "TxT"); + } + }; + + private static final Map RAW_FILEMETADATA_4 = new HashMap() { + { + put(ID, "id3"); + put(ORIGINAL_FILENAME, "file.csv"); + put(INTERNAL_FILENAME, "doesn't matter"); + put(FILETYPE, "csv"); + } + }; + + private List> couchDbRawFileMetadata; + + private MergeFilenamesAndRenameDuplicatesMigration migration; + + @Before + public void setUp() { + migration = new MergeFilenamesAndRenameDuplicatesMigration(true); + couchDbRawFileMetadata = new ArrayList<>() { + { + add(RAW_FILEMETADATA_1); + add(RAW_FILEMETADATA_2); + add(RAW_FILEMETADATA_3); + add(RAW_FILEMETADATA_4); + } + }; + } + + @Test + public void testMigration() { + // Test that the migration successfully groups FileMetadata by originalFilename + migration.getFileMetadataToUpdate(couchDbRawFileMetadata); + assertEquals(2, migration.fileMetadataGroupedByOriginalName.size()); + assertEquals(3, migration.fileMetadataGroupedByOriginalName.get("file.txt").size()); + assertEquals(1, migration.fileMetadataGroupedByOriginalName.get("file.csv").size()); + + // Test that the migration successfully renames duplicate files + migration.fileMetadataGroupedByOriginalName.forEach( + (originalFilename, fileMetadataList) -> migration.update(originalFilename, fileMetadataList)); + assertEquals("file.txt", migration.fileMetadataGroupedByOriginalName.get("file.txt").get(0).getFilename()); + assertEquals("file(2).TXT", migration.fileMetadataGroupedByOriginalName.get("file.txt").get(1).getFilename()); + assertEquals("file(3).TxT", migration.fileMetadataGroupedByOriginalName.get("file.txt").get(2).getFilename()); + assertEquals("file.csv", migration.fileMetadataGroupedByOriginalName.get("file.csv").get(0).getFilename()); + } +} \ No newline at end of file diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/files.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/files.service.ts index 2c7f759bb5..e2c1423150 100644 --- a/ui/projects/streampipes/platform-services/src/lib/apis/files.service.ts +++ b/ui/projects/streampipes/platform-services/src/lib/apis/files.service.ts @@ -91,7 +91,7 @@ export class FilesService { ); } - getAllOriginalFilenames(): Observable { + getAllFilenames(): Observable { return this.http.get( this.platformServicesCommons.apiBasePath + '/files/allFilenames', ); diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts index f1613ff344..6523bfe64a 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts @@ -1815,9 +1815,8 @@ export class FileMetadata { createdByUser: string; fileId: string; filetype: string; - internalFilename: string; + filename: string; lastModified: number; - originalFilename: string; rev: string; static fromData(data: FileMetadata, target?: FileMetadata): FileMetadata { @@ -1829,9 +1828,8 @@ export class FileMetadata { instance.createdByUser = data.createdByUser; instance.fileId = data.fileId; instance.filetype = data.filetype; - instance.internalFilename = data.internalFilename; + instance.filename = data.filename; instance.lastModified = data.lastModified; - instance.originalFilename = data.originalFilename; instance.rev = data.rev; return instance; } diff --git a/ui/src/app/assets/dialog/base-asset-links.directive.ts b/ui/src/app/assets/dialog/base-asset-links.directive.ts index 6ab3baa447..121d60fc27 100644 --- a/ui/src/app/assets/dialog/base-asset-links.directive.ts +++ b/ui/src/app/assets/dialog/base-asset-links.directive.ts @@ -98,7 +98,7 @@ export abstract class BaseAssetLinksDirective { a.measureName.localeCompare(b.measureName), ); this.files = files.sort((a, b) => - a.originalFilename.localeCompare(b.originalFilename), + a.filename.localeCompare(b.filename), ); this.adapters = adapters.sort((a, b) => a.name.localeCompare(b.name), diff --git a/ui/src/app/assets/dialog/manage-asset-links/manage-asset-links-dialog.component.html b/ui/src/app/assets/dialog/manage-asset-links/manage-asset-links-dialog.component.html index d27e5d4f43..1b48663ffd 100644 --- a/ui/src/app/assets/dialog/manage-asset-links/manage-asset-links-dialog.component.html +++ b/ui/src/app/assets/dialog/manage-asset-links/manage-asset-links-dialog.component.html @@ -337,7 +337,7 @@ selectLink( $event.checked, element.fileId, - element.originalFilename, + element.filename, 'file' ) " diff --git a/ui/src/app/assets/dialog/manage-asset-links/manage-asset-links-dialog.component.ts b/ui/src/app/assets/dialog/manage-asset-links/manage-asset-links-dialog.component.ts index 0d9b9dc1e9..356acde8fd 100644 --- a/ui/src/app/assets/dialog/manage-asset-links/manage-asset-links-dialog.component.ts +++ b/ui/src/app/assets/dialog/manage-asset-links/manage-asset-links-dialog.component.ts @@ -53,7 +53,7 @@ export class SpManageAssetLinksDialogComponent elementIdFunction = el => el.elementId; fileIdFunction = el => el.fileId; nameFunction = el => el.name; - filenameFunction = el => el.originalFilename; + filenameFunction = el => el.filename; measureNameFunction = el => el.measureName; constructor( diff --git a/ui/src/app/core-ui/static-properties/static-file-input/static-file-input.component.html b/ui/src/app/core-ui/static-properties/static-file-input/static-file-input.component.html index 9fe022a957..0ac88d7e42 100644 --- a/ui/src/app/core-ui/static-properties/static-file-input/static-file-input.component.html +++ b/ui/src/app/core-ui/static-properties/static-file-input/static-file-input.component.html @@ -62,7 +62,7 @@ *ngFor="let fileMetadata of fileMetadata" [value]="fileMetadata" > - {{ fileMetadata.originalFilename }} + {{ fileMetadata.filename }} diff --git a/ui/src/app/core-ui/static-properties/static-file-input/static-file-input.component.ts b/ui/src/app/core-ui/static-properties/static-file-input/static-file-input.component.ts index 079a7449f6..42b145e6ed 100644 --- a/ui/src/app/core-ui/static-properties/static-file-input/static-file-input.component.ts +++ b/ui/src/app/core-ui/static-properties/static-file-input/static-file-input.component.ts @@ -89,15 +89,14 @@ export class StaticFileInputComponent return validators; } - fetchFileMetadata(internalFilenameToSelect?: any) { + fetchFileMetadata(filenameToSelect?: any) { this.filesService .getFileMetadata(this.staticProperty.requiredFiletypes) .subscribe(fm => { this.fileMetadata = fm; - if (internalFilenameToSelect) { + if (filenameToSelect) { this.selectedFile = this.fileMetadata.find( - fmi => - fmi.internalFilename === internalFilenameToSelect, + fmi => fmi.filename === filenameToSelect, ); this.selectOption(this.selectedFile); this.emitUpdate(true); @@ -109,8 +108,7 @@ export class StaticFileInputComponent } else if (this.staticProperty.locationPath) { this.selectedFile = this.fileMetadata.find( fmi => - fmi.internalFilename === - this.staticProperty.locationPath, + fmi.filename === this.staticProperty.locationPath, ); } else { if (this.fileMetadata.length > 0) { @@ -136,64 +134,57 @@ export class StaticFileInputComponent upload() { if (this.selectedUploadFile !== undefined) { - this.filesService - .getAllOriginalFilenames() - .subscribe(allFileNames => { - if ( - !allFileNames.includes( - this.selectedUploadFile.name.toLowerCase(), - ) - ) { - this.uploadStatus = 0; - this.filesService - .uploadFile(this.selectedUploadFile) - .subscribe( - event => { - if ( - event.type === - HttpEventType.UploadProgress - ) { - this.uploadStatus = Math.round( - (100 * event.loaded) / event.total, - ); - } else if (event instanceof HttpResponse) { - const internalFilename = - event.body.internalFilename; - this.parentForm.controls[ - this.fieldName - ].setValue(internalFilename); - this.fetchFileMetadata( - internalFilename, - ); - } - }, - error => {}, - ); - } else { - this.openRenameDialog(); - } - }); + this.filesService.getAllFilenames().subscribe(allFileNames => { + if ( + !allFileNames.includes( + this.selectedUploadFile.name.toLowerCase(), + ) + ) { + this.uploadStatus = 0; + this.filesService + .uploadFile(this.selectedUploadFile) + .subscribe( + event => { + if ( + event.type === HttpEventType.UploadProgress + ) { + this.uploadStatus = Math.round( + (100 * event.loaded) / event.total, + ); + } else if (event instanceof HttpResponse) { + const filename = event.body.filename; + this.parentForm.controls[ + this.fieldName + ].setValue(filename); + this.fetchFileMetadata(filename); + } + }, + error => {}, + ); + } else { + this.openRenameDialog(); + } + }); } } selectOption(fileMetadata: FileMetadata) { - this.staticProperty.locationPath = fileMetadata.internalFilename; + this.staticProperty.locationPath = fileMetadata.filename; const valid: boolean = - fileMetadata.internalFilename !== '' || - fileMetadata.internalFilename !== undefined; + fileMetadata.filename !== '' || fileMetadata.filename !== undefined; this.updateEmitter.emit( new ConfigurationInfo(this.staticProperty.internalName, valid), ); } displayFn(fileMetadata: FileMetadata) { - return fileMetadata ? fileMetadata.originalFilename : ''; + return fileMetadata ? fileMetadata.filename : ''; } onStatusChange(status: any) {} onValueChange(value: any) { - this.staticProperty.locationPath = value.internalFilename; + this.staticProperty.locationPath = value.filename; this.parentForm.updateValueAndValidity(); } diff --git a/ui/src/app/files/components/file-overview/file-overview.component.html b/ui/src/app/files/components/file-overview/file-overview.component.html index 00ab831e65..ff6a4796cd 100644 --- a/ui/src/app/files/components/file-overview/file-overview.component.html +++ b/ui/src/app/files/components/file-overview/file-overview.component.html @@ -27,7 +27,7 @@ Filename -

{{ fileMetadata.originalFilename }}

+

{{ fileMetadata.filename }}

 diff --git a/ui/src/app/files/components/file-overview/file-overview.component.ts b/ui/src/app/files/components/file-overview/file-overview.component.ts index 472de9117d..dc8ca5c4fc 100644 --- a/ui/src/app/files/components/file-overview/file-overview.component.ts +++ b/ui/src/app/files/components/file-overview/file-overview.component.ts @@ -84,11 +84,9 @@ export class FileOverviewComponent implements OnInit { } downloadFile(fileMetadata: FileMetadata) { - this.filesService - .getFile(fileMetadata.internalFilename) - .subscribe(response => { - saveAs(response, fileMetadata.originalFilename); - }); + this.filesService.getFile(fileMetadata.filename).subscribe(response => { + saveAs(response, fileMetadata.filename); + }); } getFileColor(fileType: string) { diff --git a/ui/src/app/files/dialog/file-rename/file-rename-dialog.component.html b/ui/src/app/files/dialog/file-rename/file-rename-dialog.component.html index 121faeac02..8c849b8016 100644 --- a/ui/src/app/files/dialog/file-rename/file-rename-dialog.component.html +++ b/ui/src/app/files/dialog/file-rename/file-rename-dialog.component.html @@ -25,9 +25,7 @@

A file with the same name already exists. Please - give your file a new name or select one of the - existing files. it from existing files or rename - it: + select from existing files or rename it:

diff --git a/ui/src/app/files/dialog/file-upload/file-upload-dialog.component.ts b/ui/src/app/files/dialog/file-upload/file-upload-dialog.component.ts index 81a26890ad..328cfe8494 100644 --- a/ui/src/app/files/dialog/file-upload/file-upload-dialog.component.ts +++ b/ui/src/app/files/dialog/file-upload/file-upload-dialog.component.ts @@ -58,7 +58,7 @@ export class FileUploadDialogComponent { } store() { - this.filesService.getAllOriginalFilenames().subscribe(data => { + this.filesService.getAllFilenames().subscribe(data => { const allFileNames = new Set(data); this.duplicateFileNames = this.fileNames.filter(fileName => allFileNames.has(fileName.toLowerCase()),