[#2181] Merge internalFilename and originalFilename (#2430)

* Serialize non-primitive types and store in Influx * extract RawFieldSerializer * rename test * delete old * temp * redesign file api * fix * add comment * merge internalFilename and originalFilename * delete useless * fix bug * code review and unit test * Delete debug logs Co-authored-by: Tim <50115603+bossenti@users.noreply.github.com> * add log * undo --------- Co-authored-by: Tim <50115603+bossenti@users.noreply.github.com>
apache · Mar 6, 2024 · 0035d12 · 0035d12
1 parent 92048ed
commit 0035d12
Show file tree

Hide file tree

Showing 24 changed files with 426 additions and 366 deletions.
diff --git a/...export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java b/...export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
@@ -135,8 +135,8 @@ protected void handleFile(String document,
     var fileMetadata = resolver.readDocument(document);
     resolver.writeDocument(document);
     byte[] file = zipContent.get(
-        fileMetadata.getInternalFilename().substring(0, fileMetadata.getInternalFilename().lastIndexOf(".")));
-    new FileHandler().storeFile(fileMetadata.getInternalFilename(), new ByteArrayInputStream(file));
+        fileMetadata.getFilename().substring(0, fileMetadata.getFilename().lastIndexOf(".")));
+    new FileHandler().storeFile(fileMetadata.getFilename(), new ByteArrayInputStream(file));
   }
 
   @Override

diff --git a/...export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java b/...export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
@@ -116,7 +116,7 @@ protected void handleDataViewWidget(String document, String dataViewWidget) {
   protected void handleFile(String document,
                             String fileMetadataId,
                             Map<String, byte[]> zipContent) throws JsonProcessingException {
-    addExportItem(fileMetadataId, new FileResolver().readDocument(document).getOriginalFilename(),
+    addExportItem(fileMetadataId, new FileResolver().readDocument(document).getFilename(),
         importConfig::addFile);
   }
 

diff --git a/...-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java b/...-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
@@ -121,7 +121,7 @@ public byte[] generateExportPackage() throws IOException {
 
       config.getFiles().forEach(item -> {
         var fileResolver = new FileResolver();
-        String filename = fileResolver.findDocument(item.getResourceId()).getInternalFilename();
+        String filename = fileResolver.findDocument(item.getResourceId()).getFilename();
         addDoc(builder, item, new FileResolver(), manifest::addFile);
         try {
           builder.addBinary(filename, Files.readAllBytes(FileManager.getFile(filename).toPath()));

diff --git a/...mpipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java b/...mpipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
@@ -44,7 +44,7 @@ public FileMetadata readDocument(String serializedDoc) throws JsonProcessingExce
 
   @Override
   public ExportItem convert(FileMetadata document) {
-    return new ExportItem(document.getFileId(), document.getOriginalFilename(), true);
+    return new ExportItem(document.getFileId(), document.getFilename(), true);
   }
 
   @Override

diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java b/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java
@@ -28,8 +28,7 @@ public class FileMetadata {
 
   private @SerializedName("_rev") String rev;
 
-  private String internalFilename;
-  private String originalFilename;
+  private String filename;
   private String filetype;
 
   private long createdAt;
@@ -53,20 +52,12 @@ public void setRev(String rev) {
     this.rev = rev;
   }
 
-  public String getInternalFilename() {
-    return internalFilename;
+  public String getFilename() {
+    return filename;
   }
 
-  public void setInternalFilename(String internalFilename) {
-    this.internalFilename = internalFilename;
-  }
-
-  public String getOriginalFilename() {
-    return originalFilename;
-  }
-
-  public void setOriginalFilename(String originalFilename) {
-    this.originalFilename = originalFilename;
+  public void setFilename(String filename) {
+    this.filename = filename;
   }
 
   public long getCreatedAt() {

diff --git a/...es-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileHandler.java b/...es-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileHandler.java
@@ -18,13 +18,19 @@
 package org.apache.streampipes.manager.file;
 
 import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 
 public class FileHandler {
 
+  Logger logger = LoggerFactory.getLogger(FileHandler.class);
+
   public void storeFile(String filename, InputStream fileInputStream) throws IOException {
     File targetFile = makeFile(filename);
     FileUtils.copyInputStreamToFile(fileInputStream, targetFile);
@@ -39,6 +45,20 @@ public File getFile(String filename) {
     return FileUtils.getFile(makeFile(filename));
   }
 
+  public void renameFile(String oldFilename, String newFilename) {
+    try {
+      var fileInputStream = new FileInputStream(getFile(oldFilename));
+      deleteFile(oldFilename);
+      storeFile(newFilename, fileInputStream);
+    } catch (FileNotFoundException e) {
+      logger.error(
+          "Failed to find the old file locally with internalFilename as the identifier, this is most likely a mismatch "
+              + "between local file and FileMetadata stored in CouchDB. Raw exception message: " + e.getMessage());
+    } catch (IOException e) {
+      logger.error("Failed to save renamed file locally: " + e.getMessage());
+    }
+  }
+
   private File makeFile(String filename) {
     File fileDir = new File(makeFileLocation());
     if (!fileDir.exists()) {

diff --git a/...es-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java b/...es-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java
@@ -29,7 +29,6 @@
 import java.io.InputStream;
 import java.util.Arrays;
 import java.util.List;
-import java.util.UUID;
 import java.util.stream.Collectors;
 
 public class FileManager {
@@ -48,13 +47,13 @@ public static File getFileByOriginalName(String originalName) throws IOException
 
     var file = allFiles
             .stream()
-            .filter(fileMetadata -> fileMetadata.getOriginalFilename().equals(originalName))
+            .filter(fileMetadata -> fileMetadata.getFilename().equals(originalName))
             .findFirst();
 
     if (file.isEmpty()){
       throw new IOException("No file with original name '%s' found".formatted(originalName));
     }
-    return new FileHandler().getFile(file.get().getInternalFilename());
+    return new FileHandler().getFile(file.get().getFilename());
   }
 
   /**
@@ -74,16 +73,15 @@ public static FileMetadata storeFile(String user,
 
     fileInputStream = cleanFile(fileInputStream, filetype);
 
-    String internalFilename = makeInternalFilename(filetype);
-    FileMetadata fileMetadata = makeFileMetadata(user, filename, internalFilename, filetype);
-    new FileHandler().storeFile(internalFilename, fileInputStream);
+    FileMetadata fileMetadata = makeFileMetadata(user, filename, filetype);
+    new FileHandler().storeFile(filename, fileInputStream);
     storeFileMetadata(fileMetadata);
     return fileMetadata;
   }
 
   public static void deleteFile(String id) {
     FileMetadata fileMetadata = getFileMetadataStorage().getMetadataById(id);
-    new FileHandler().deleteFile(fileMetadata.getInternalFilename());
+    new FileHandler().deleteFile(fileMetadata.getFilename());
     getFileMetadataStorage().deleteFileMetadata(id);
   }
 
@@ -118,24 +116,18 @@ private static IFileMetadataStorage getFileMetadataStorage() {
   }
 
   private static FileMetadata makeFileMetadata(String user,
-                                               String originalFilename,
-                                               String internalFilename,
+                                               String filename,
                                                String filetype) {
 
     FileMetadata fileMetadata = new FileMetadata();
     fileMetadata.setCreatedAt(System.currentTimeMillis());
     fileMetadata.setCreatedByUser(user);
     fileMetadata.setFiletype(filetype);
-    fileMetadata.setInternalFilename(internalFilename);
-    fileMetadata.setOriginalFilename(originalFilename);
+    fileMetadata.setFilename(filename);
 
     return fileMetadata;
   }
 
-  private static String makeInternalFilename(String filetype) {
-    return UUID.randomUUID() + "." + filetype;
-  }
-
   private static List<FileMetadata> filterFiletypes(List<FileMetadata> allFiles, String filetypes) {
     return allFiles
         .stream()

diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java
@@ -135,7 +135,7 @@ private byte[] getFileContents(File file) throws IOException {
   public ResponseEntity<List<String>> getAllOriginalFilenames() {
     return ok(FileManager.getAllFiles()
                          .stream()
-                         .map(fileMetadata -> fileMetadata.getOriginalFilename()
+                         .map(fileMetadata -> fileMetadata.getFilename()
                                                           .toLowerCase())
                          .toList());
   }

diff --git a/...ore/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java b/...ore/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java
@@ -25,7 +25,7 @@
 import org.apache.streampipes.service.core.migrations.v090.UpdateUsernameViewMigration;
 import org.apache.streampipes.service.core.migrations.v093.AdapterMigration;
 import org.apache.streampipes.service.core.migrations.v093.StoreEmailTemplatesMigration;
-import org.apache.streampipes.service.core.migrations.v095.DuplicateFilesRenameMigration;
+import org.apache.streampipes.service.core.migrations.v095.MergeFilenamesAndRenameDuplicatesMigration;
 
 import java.util.Arrays;
 import java.util.List;
@@ -40,7 +40,7 @@ public List<Migration> getAvailableMigrations() {
         new UpdateUsernameViewMigration(),
         new AdapterMigration(),
         new StoreEmailTemplatesMigration(),
-        new DuplicateFilesRenameMigration()
+        new MergeFilenamesAndRenameDuplicatesMigration()
     );
   }
 }
diff --git a/...va/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java b/...va/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java