Draft commit for reducing read IOPS in temporary package parts.

poi-ooxml/src/main/java/org/apache/poi/openxml4j/opc/ZipPackage.java

@@ -27,7 +27,6 @@ Licensed to the Apache Software Foundation (ASF) under one or more
 import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -52,7 +51,6 @@ Licensed to the Apache Software Foundation (ASF) under one or more
 import org.apache.poi.openxml4j.util.ZipFileZipEntrySource;
 import org.apache.poi.openxml4j.util.ZipInputStreamZipEntrySource;
 import org.apache.poi.util.IOUtils;
-import org.apache.poi.util.TempFile;
 
 /**
  * Physical zip package.
@@ -62,6 +60,7 @@ public final class ZipPackage extends OPCPackage {
     private static final String SETTINGS_XML = "settings.xml";
     private static boolean useTempFilePackageParts = false;
     private static boolean encryptTempFilePackageParts = false;
+    private static boolean bufferTempFilePackageParts = false;
 
     private static final Logger LOG = LogManager.getLogger(ZipPackage.class);
 
@@ -85,6 +84,13 @@ public static void setEncryptTempFilePackageParts(boolean encryptTempFiles) {
         encryptTempFilePackageParts = encryptTempFiles;
     }
 
+    /**
+     * @param bufferTempFiles whether to buffer package part temp files
+     */
+    public static void setBufferTempFilePackageParts(boolean bufferTempFiles) {
+        bufferTempFilePackageParts = bufferTempFiles;
+    }
+
     /**
      * @return whether package part data is stored in temp files to save memory
      */
@@ -99,6 +105,13 @@ public static boolean encryptTempFilePackageParts() {
         return encryptTempFilePackageParts;
     }
 
+    /**
+     * @return whether package part temp files are buffered
+     */
+    public static boolean bufferTempFilePackageParts() {
+        return bufferTempFilePackageParts;
+    }
+
     /**
      * Constructor. Creates a new, empty ZipPackage.
      */
@@ -422,8 +435,16 @@ protected PackagePart createPartImpl(PackagePartName partName,
         try {
             if (useTempFilePackageParts) {
                 if (encryptTempFilePackageParts) {
-                    return new EncryptedTempFilePackagePart(this, partName, contentType, loadRelationships);
-                } else {
+                    if (bufferTempFilePackageParts) {
+                        return new BufferedEncryptedTempFilePackagePart(this, partName, contentType, loadRelationships);
+                    } else {
+                        return new EncryptedTempFilePackagePart(this, partName, contentType, loadRelationships);
+                    }
+                }
+                else if (bufferTempFilePackageParts) {
+                    return new BufferedTempFilePackagePart(this, partName, contentType, loadRelationships);
+                }
+                else {
                     return new TempFilePackagePart(this, partName, contentType, loadRelationships);
                 }
             } else {

poi-ooxml/src/main/java/org/apache/poi/openxml4j/opc/internal/BufferedEncryptedTempFilePackagePart.java

@@ -0,0 +1,157 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
+package org.apache.poi.openxml4j.opc.internal;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
+import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
+import org.apache.poi.openxml4j.opc.OPCPackage;
+import org.apache.poi.openxml4j.opc.PackagePart;
+import org.apache.poi.openxml4j.opc.PackagePartName;
+import org.apache.poi.openxml4j.opc.internal.marshallers.ZipPartMarshaller;
+import org.apache.poi.poifs.crypt.temp.EncryptedTempData;
+import org.apache.poi.util.Beta;
+import org.apache.poi.util.IOUtils;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * (Experimental) Buffered Encrypted Temp File version of a package part.
+ *
+ * @since POI 5.2.4
+ */
+@Beta
+public final class BufferedEncryptedTempFilePackagePart extends PackagePart {
+    private static int fileBufferSize = 1024 * 1024; // 1 MB
+    private static final Logger LOG = LogManager.getLogger(BufferedEncryptedTempFilePackagePart.class);
+
+    /**
+     * Storage for the part data.
+     */
+    private EncryptedTempData tempFile;
+
+    /**
+     * Constructor.
+     *
+     * @param pack
+     *            The owner package.
+     * @param partName
+     *            The part name.
+     * @param contentType
+     *            The content type.
+     * @throws InvalidFormatException
+     *             If the specified URI is not OPC compliant.
+     * @throws IOException
+     *             If temp file cannot be created.
+     */
+    public BufferedEncryptedTempFilePackagePart(OPCPackage pack, PackagePartName partName,
+                                                String contentType) throws InvalidFormatException, IOException {
+        this(pack, partName, contentType, true);
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param pack
+     *            The owner package.
+     * @param partName
+     *            The part name.
+     * @param contentType
+     *            The content type.
+     * @param loadRelationships
+     *            Specify if the relationships will be loaded.
+     * @throws InvalidFormatException
+     *             If the specified URI is not OPC compliant.
+     * @throws IOException
+     *             If temp file cannot be created.
+     */
+    public BufferedEncryptedTempFilePackagePart(OPCPackage pack, PackagePartName partName,
+                                                String contentType, boolean loadRelationships)
+            throws InvalidFormatException, IOException {
+        super(pack, partName, new ContentType(contentType), loadRelationships);
+        tempFile = new EncryptedTempData();
+    }
+
+    /**
+     * Allows configuration of read/write buffer sizes of temp file package parts
+     *
+     * @param bufferSize
+     *              Size of the buffer used for input/output streams.
+     */
+    public static void setBufferSize(int bufferSize) {
+        fileBufferSize = bufferSize;
+    }
+
+    @Override
+    protected InputStream getInputStreamImpl() throws IOException {
+        return new BufferedInputStream(tempFile.getInputStream(), fileBufferSize);
+    }
+
+    @Override
+    protected OutputStream getOutputStreamImpl() throws IOException {
+        return new BufferedOutputStream(tempFile.getOutputStream(), fileBufferSize);
+    }
+
+    @Override
+    public long getSize() {
+        return tempFile.getByteCount();
+    }
+
+    @Override
+    public void clear() {
+        try(OutputStream os = getOutputStreamImpl()) {
+            os.write(new byte[0]);
+        } catch (IOException e) {
+            LOG.atWarn().log("Failed to clear data in temp file", e);
+        }
+    }
+
+    @Override
+    public boolean save(OutputStream os) throws OpenXML4JException {
+        return new ZipPartMarshaller().marshall(this, os);
+    }
+
+    @Override
+    public boolean load(InputStream is) throws InvalidFormatException {
+       try (OutputStream os = getOutputStreamImpl()) {
+            IOUtils.copy(is, os);
+       } catch(IOException e) {
+            throw new InvalidFormatException(e.getMessage(), e);
+       }
+
+       // All done
+       return true;
+    }
+
+    @Override
+    public void close() {
+        tempFile.dispose();
+    }
+
+    @Override
+    public void flush() {
+        // Do nothing
+    }
+}

poi-ooxml/src/main/java/org/apache/poi/openxml4j/opc/internal/BufferedTempFilePackagePart.java

@@ -0,0 +1,160 @@
+/* ====================================================================
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+==================================================================== */
+
+package org.apache.poi.openxml4j.opc.internal;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
+import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
+import org.apache.poi.openxml4j.opc.OPCPackage;
+import org.apache.poi.openxml4j.opc.PackagePart;
+import org.apache.poi.openxml4j.opc.PackagePartName;
+import org.apache.poi.openxml4j.opc.internal.marshallers.ZipPartMarshaller;
+import org.apache.poi.util.Beta;
+import org.apache.poi.util.IOUtils;
+import org.apache.poi.util.TempFile;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * (Experimental) Buffered Temp File version of a package part.
+ *
+ * @since POI 5.4.2
+ */
+@Beta
+public final class BufferedTempFilePackagePart extends PackagePart {
+    private static int fileBufferSize = 1024 * 1024; // 1 MB
+    private static final Logger LOG = LogManager.getLogger(BufferedTempFilePackagePart.class);
+
+    /**
+     * Storage for the part data.
+     */
+    private File tempFile;
+
+    /**
+     * Constructor.
+     *
+     * @param pack
+     *            The owner package.
+     * @param partName
+     *            The part name.
+     * @param contentType
+     *            The content type.
+     * @throws InvalidFormatException
+     *             If the specified URI is not OPC compliant.
+     * @throws IOException
+     *             If temp file cannot be created.
+     */
+    public BufferedTempFilePackagePart(OPCPackage pack, PackagePartName partName,
+                                       String contentType) throws InvalidFormatException, IOException {
+        this(pack, partName, contentType, true);
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param pack
+     *            The owner package.
+     * @param partName
+     *            The part name.
+     * @param contentType
+     *            The content type.
+     * @param loadRelationships
+     *            Specify if the relationships will be loaded.
+     * @throws InvalidFormatException
+     *             If the specified URI is not OPC compliant.
+     * @throws IOException
+     *             If temp file cannot be created.
+     */
+    public BufferedTempFilePackagePart(OPCPackage pack, PackagePartName partName,
+                                       String contentType, boolean loadRelationships)
+            throws InvalidFormatException, IOException {
+        super(pack, partName, new ContentType(contentType), loadRelationships);
+        tempFile = TempFile.createTempFile("poi-package-part", ".tmp");
+    }
+
+    /**
+     * Allows configuration of read/write buffer sizes of temp file package parts
+     *
+     * @param bufferSize
+     *              Size of the buffer used for input/output streams.
+     */
+    public static void setBufferSize(int bufferSize) {
+        fileBufferSize = bufferSize;
+    }
+
+    @Override
+    protected InputStream getInputStreamImpl() throws IOException {
+        return new BufferedInputStream(new FileInputStream(tempFile), fileBufferSize);
+    }
+
+    @Override
+    protected OutputStream getOutputStreamImpl() throws IOException {
+        return new BufferedOutputStream(new FileOutputStream(tempFile), fileBufferSize);
+    }
+
+    @Override
+    public long getSize() {
+        return tempFile.length();
+    }
+
+    @Override
+    public void clear() {
+        try(OutputStream os = getOutputStreamImpl()) {
+            os.write(new byte[0]);
+        } catch (IOException e) {
+            LOG.atWarn().log("Failed to clear data in temp file", e);
+        }
+    }
+
+    @Override
+    public boolean save(OutputStream os) throws OpenXML4JException {
+        return new ZipPartMarshaller().marshall(this, os);
+    }
+
+    @Override
+    public boolean load(InputStream is) throws InvalidFormatException {
+       try (OutputStream os = getOutputStreamImpl()) {
+            IOUtils.copy(is, os);
+       } catch(IOException e) {
+            throw new InvalidFormatException(e.getMessage(), e);
+       }
+
+       // All done
+       return true;
+    }
+
+    @Override
+    public void close() {
+        if (!tempFile.delete()) {
+            LOG.atInfo().log("Failed to delete temp file; may already have been closed and deleted");
+        }
+    }
+
+    @Override
+    public void flush() {
+        // Do nothing
+    }
+}