Skip to content

Commit dffd0bb

Browse files
oleg-cherednikoleg-cherednik
committed
#20 Extract entries async
1 parent 41d6959 commit dffd0bb

File tree

6 files changed

+221
-70
lines changed

6 files changed

+221
-70
lines changed

‎src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/DataInputThreadLocal.java

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package ru.olegcherednik.zip4jvm.engine.unzip;
2+
3+
import ru.olegcherednik.zip4jvm.io.in.DataInput;
4+
5+
import lombok.RequiredArgsConstructor;
6+
import org.apache.commons.io.IOUtils;
7+
8+
import java.util.List;
9+
import java.util.concurrent.CopyOnWriteArrayList;
10+
import java.util.function.Supplier;
11+
12+
/**
13+
* @author Oleg Cherednik
14+
* @since 28.12.2024
15+
*/
16+
@RequiredArgsConstructor
17+
public class DataInputThreadLocal<T extends DataInput> extends ThreadLocal<T> {
18+
19+
private final Supplier<T> dataInputSup;
20+
private final List<T> dataInputs = new CopyOnWriteArrayList<>();
21+
22+
public void release() {
23+
dataInputs.forEach(IOUtils::closeQuietly);
24+
dataInputs.clear();
25+
}
26+
27+
// ---------- ThreadLocal ----------
28+
29+
@Override
30+
public T get() {
31+
T in = super.get();
32+
33+
if (in == null) {
34+
in = dataInputSup.get();
35+
set(in);
36+
dataInputs.add(in);
37+
}
38+
39+
return in;
40+
}
41+
42+
}

‎src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipEngine.java

+7-5
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
import ru.olegcherednik.zip4jvm.model.password.PasswordProvider;
2929
import ru.olegcherednik.zip4jvm.model.settings.UnzipSettings;
3030
import ru.olegcherednik.zip4jvm.model.src.SrcZip;
31+
import ru.olegcherednik.zip4jvm.utils.quitely.Quietly;
3132

32-
import java.io.IOException;
3333
import java.nio.file.Path;
3434
import java.util.Collection;
3535
import java.util.Collections;
@@ -47,7 +47,7 @@ public final class UnzipEngine implements ZipFile.Reader {
4747
public UnzipEngine(SrcZip srcZip, UnzipSettings settings) {
4848
PasswordProvider passwordProvider = settings.getPasswordProvider();
4949
zipModel = ZipModelBuilder.read(srcZip, settings.getCharsetCustomizer(), passwordProvider);
50-
unzipExtractEngine = new UnzipExtractEngine(passwordProvider, zipModel);
50+
unzipExtractEngine = new UnzipExtractAsyncEngine(passwordProvider, zipModel);
5151
}
5252

5353
// ---------- ZipFile.Reader ----------
@@ -105,9 +105,11 @@ public ZipFile.Entry next() {
105105
};
106106
}
107107

108-
public static RandomAccessDataInput createRandomAccessDataInput(SrcZip srcZip) throws IOException {
109-
return srcZip.isSolid() ? new SolidRandomAccessDataInput(srcZip)
110-
: new SplitRandomAccessDataInput(srcZip);
108+
// ---------- static ----------
109+
110+
public static RandomAccessDataInput createRandomAccessDataInput(SrcZip srcZip) {
111+
return Quietly.doRuntime(() -> srcZip.isSolid() ? new SolidRandomAccessDataInput(srcZip)
112+
: new SplitRandomAccessDataInput(srcZip));
111113
}
112114

113115
}

‎src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractAsyncEngine.java

+140
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package ru.olegcherednik.zip4jvm.engine.unzip;
20+
21+
import ru.olegcherednik.zip4jvm.exception.Zip4jvmException;
22+
import ru.olegcherednik.zip4jvm.io.in.file.consecutive.ConsecutiveAccessDataInput;
23+
import ru.olegcherednik.zip4jvm.model.ZipModel;
24+
import ru.olegcherednik.zip4jvm.model.entry.ZipEntry;
25+
import ru.olegcherednik.zip4jvm.model.password.PasswordProvider;
26+
27+
import java.io.IOException;
28+
import java.nio.file.Path;
29+
import java.util.Iterator;
30+
import java.util.LinkedList;
31+
import java.util.List;
32+
import java.util.Map;
33+
import java.util.Optional;
34+
import java.util.concurrent.CompletableFuture;
35+
import java.util.concurrent.Executor;
36+
import java.util.concurrent.ForkJoinPool;
37+
import java.util.concurrent.ForkJoinWorkerThread;
38+
import java.util.concurrent.atomic.AtomicInteger;
39+
40+
/**
41+
* @author Oleg Cherednik
42+
* @since 28.12.2024
43+
*/
44+
public class UnzipExtractAsyncEngine extends UnzipExtractEngine {
45+
46+
private static final int MIN_ENTRIES_AMOUNT = 100;
47+
48+
public UnzipExtractAsyncEngine(PasswordProvider passwordProvider, ZipModel zipModel) {
49+
super(passwordProvider, zipModel);
50+
}
51+
52+
// ---------- UnzipExtractEngine ----------
53+
54+
@Override
55+
protected void extractEntry(Path dstDir, Map<String, String> map) {
56+
if (map == null ? zipModel.getTotalEntries() <= MIN_ENTRIES_AMOUNT : map.size() <= MIN_ENTRIES_AMOUNT)
57+
super.extractEntry(dstDir, map);
58+
else
59+
extractEntryAsync(dstDir, map);
60+
}
61+
62+
// ----------
63+
64+
protected void extractEntryAsync(Path dstDir, Map<String, String> map) {
65+
List<CompletableFuture<Void>> tasks = new LinkedList<>();
66+
Iterator<ZipEntry> it = zipModel.absOffsAscIterator();
67+
DataInputThreadLocal<ConsecutiveAccessDataInput> threadLocalDataInput =
68+
new DataInputThreadLocal<>(this::createConsecutiveDataInput);
69+
70+
try {
71+
Executor executor = createExecutor();
72+
73+
while (it.hasNext()) {
74+
ZipEntry zipEntry = it.next();
75+
76+
if (map != null && !map.containsKey(zipEntry.getFileName()))
77+
continue;
78+
79+
String fileName = Optional.ofNullable(map)
80+
.map(m -> m.get(zipEntry.getFileName()))
81+
.orElse(zipEntry.getFileName());
82+
Path file = dstDir.resolve(fileName);
83+
84+
CompletableFuture<Void> task = CompletableFuture.runAsync(
85+
() -> {
86+
try {
87+
ConsecutiveAccessDataInput in = threadLocalDataInput.get();
88+
extractEntry(file, zipEntry, in);
89+
} catch (IOException e) {
90+
throw new Zip4jvmException(e);
91+
}
92+
},
93+
executor);
94+
95+
// CompletableFuture<Void> task = CompletableFuture.runAsync(() -> {
96+
// try {
97+
// ConsecutiveAccessDataInput in = threadLocalDataInput.get();
98+
// extractEntry(file, zipEntry, in);
99+
// } catch (IOException e) {
100+
// throw new Zip4jvmException(e);
101+
// }
102+
// });
103+
104+
tasks.add(task);
105+
}
106+
107+
tasks.forEach(CompletableFuture::join);
108+
} finally {
109+
threadLocalDataInput.release();
110+
}
111+
}
112+
113+
protected Executor createExecutor() {
114+
AtomicInteger counter = new AtomicInteger();
115+
116+
ForkJoinPool.ForkJoinWorkerThreadFactory factory = pool -> {
117+
ForkJoinWorkerThread thread = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool);
118+
thread.setName(String.format("zip4jvm-extract-%02d", counter.incrementAndGet()));
119+
return thread;
120+
};
121+
122+
return new ForkJoinPool(Runtime.getRuntime().availableProcessors(), factory, null, false);
123+
}
124+
125+
protected void extractEntry(Path file, ZipEntry zipEntry, ConsecutiveAccessDataInput in) throws IOException {
126+
in.seekForward(zipEntry.getLocalFileHeaderAbsOffs());
127+
128+
if (zipEntry.isSymlink())
129+
extractSymlink(file, zipEntry, in);
130+
else if (zipEntry.isDirectory())
131+
extractEmptyDirectory(file);
132+
else
133+
extractRegularFile(file, zipEntry, in);
134+
135+
// TODO attributes for directory should be set at the end (under Posix, it could have less privileges)
136+
setFileAttributes(file, zipEntry);
137+
setFileLastModifiedTime(file, zipEntry);
138+
}
139+
140+
}

‎src/main/java/ru/olegcherednik/zip4jvm/engine/unzip/UnzipExtractEngine.java

+23-20
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import ru.olegcherednik.zip4jvm.model.password.PasswordProvider;
3232
import ru.olegcherednik.zip4jvm.model.src.SrcZip;
3333
import ru.olegcherednik.zip4jvm.utils.ZipUtils;
34+
import ru.olegcherednik.zip4jvm.utils.quitely.Quietly;
3435
import ru.olegcherednik.zip4jvm.utils.time.DosTimestampConverterUtils;
3536

3637
import lombok.RequiredArgsConstructor;
@@ -105,24 +106,23 @@ protected List<ZipEntry> getEntriesByPrefix(String prefix) {
105106
.collect(Collectors.toList());
106107
}
107108

108-
// ----------
109-
110109
protected void extractEntry(Path dstDir, Map<String, String> map) {
111-
try (ConsecutiveAccessDataInput in = createConsecutiveDataInput(zipModel.getSrcZip())) {
110+
try (ConsecutiveAccessDataInput in = createConsecutiveDataInput()) {
112111
Iterator<ZipEntry> it = zipModel.absOffsAscIterator();
113112

114113
while (it.hasNext()) {
115114
ZipEntry zipEntry = it.next();
116115

117-
if (map == null || map.containsKey(zipEntry.getFileName())) {
118-
in.seekForward(zipEntry.getLocalFileHeaderAbsOffs());
116+
if (map != null && !map.containsKey(zipEntry.getFileName()))
117+
continue;
118+
119+
in.seekForward(zipEntry.getLocalFileHeaderAbsOffs());
119120

120-
String fileName = Optional.ofNullable(map)
121-
.map(m -> m.get(zipEntry.getFileName()))
122-
.orElse(zipEntry.getFileName());
123-
Path file = dstDir.resolve(fileName);
124-
extractEntry(file, zipEntry, in);
125-
}
121+
String fileName = Optional.ofNullable(map)
122+
.map(m -> m.get(zipEntry.getFileName()))
123+
.orElse(zipEntry.getFileName());
124+
Path file = dstDir.resolve(fileName);
125+
extractEntry(file, zipEntry, in);
126126
}
127127
} catch (IOException e) {
128128
throw new Zip4jvmException(e);
@@ -164,12 +164,23 @@ protected void extractRegularFile(Path file, ZipEntry zipEntry, DataInput in) th
164164
ZipUtils.copyLarge(zipEntry.createInputStream(in), getOutputStream(file));
165165
}
166166

167+
// ---------- static ----------
168+
169+
public ConsecutiveAccessDataInput createConsecutiveDataInput() {
170+
return Quietly.doRuntime(() -> {
171+
SrcZip srcZip = zipModel.getSrcZip();
172+
173+
return srcZip.isSolid() ? new SolidConsecutiveAccessDataInput(srcZip)
174+
: new SplitConsecutiveAccessDataInput(srcZip);
175+
});
176+
}
177+
167178
protected static void setFileAttributes(Path path, ZipEntry zipEntry) throws IOException {
168179
if (zipEntry.getExternalFileAttributes() != null)
169180
zipEntry.getExternalFileAttributes().apply(path);
170181
}
171182

172-
private static void setFileLastModifiedTime(Path path, ZipEntry zipEntry) throws IOException {
183+
protected static void setFileLastModifiedTime(Path path, ZipEntry zipEntry) throws IOException {
173184
long lastModifiedTime = DosTimestampConverterUtils.dosToJavaTime(zipEntry.getLastModifiedTime());
174185
Files.setLastModifiedTime(path, FileTime.fromMillis(lastModifiedTime));
175186
}
@@ -184,12 +195,4 @@ protected static OutputStream getOutputStream(Path file) throws IOException {
184195
return Files.newOutputStream(file);
185196
}
186197

187-
// ---------- static ----------
188-
189-
public static ConsecutiveAccessDataInput createConsecutiveDataInput(SrcZip srcZip) throws IOException {
190-
return srcZip.isSolid() ? new SolidConsecutiveAccessDataInput(srcZip)
191-
: new SplitConsecutiveAccessDataInput(srcZip);
192-
193-
}
194-
195198
}

‎src/main/java/ru/olegcherednik/zip4jvm/io/in/file/consecutive/SolidConsecutiveAccessDataInput.java

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public class SolidConsecutiveAccessDataInput extends BaseConsecutiveAccessDataIn
4242
private final InputStream in;
4343

4444
public SolidConsecutiveAccessDataInput(SrcZip srcZip) throws IOException {
45+
System.out.println(Thread.currentThread().getName());
4546
byteOrder = srcZip.getByteOrder();
4647
in = new BufferedInputStream(Files.newInputStream(srcZip.getDiskByNo(0).getPath()));
4748
}

‎src/test/java/ru/olegcherednik/zip4jvm/Foo.java

+8-45
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,9 @@
1818
*/
1919
package ru.olegcherednik.zip4jvm;
2020

21-
import ru.olegcherednik.zip4jvm.model.settings.ZipInfoSettings;
22-
2321
import java.io.IOException;
2422
import java.nio.file.Path;
2523
import java.nio.file.Paths;
26-
import java.util.concurrent.TimeUnit;
2724

2825
/**
2926
* @author Oleg Cherednik
@@ -34,51 +31,17 @@ public class Foo {
3431

3532
public static void main(String[] args) throws IOException {
3633
final long timeFrom = System.currentTimeMillis();
37-
int[][] token = new int[3][3];
38-
39-
// Path zip = Paths.get("d:/zip4jvm/zip64/split/ferdinand.zip");
40-
// Path zip = Paths.get("d:/zip4jvm/aaa/split/ducati.zip");
41-
42-
// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199.zip");
43-
// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199-ecd.zip");
44-
// Path zip = Paths.get("d:/zip4jvm/aaa/bikes.zip");
45-
// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199-dcl.zip");
46-
// Path zip = Paths.get("d:/zip4jvm/aaa/app.apk");
47-
// Path zip = Paths.get("d:/zip4jvm/aaa/android.apk");
48-
49-
// Path zip = Paths.get("d:/zip4jvm/aaa/ducati-panigale-1199.zip");
50-
// Path zip = Paths.get("d:/zip4jvm/zip64/bzip2-aes256-strong.zip");
5134

52-
// Path zip = Paths.get("d:/zip4jvm/zip64/bzip2-aes256-strong.zip");
53-
// Path zip = Paths.get("d:/Programming/GitHub/zip4jvm/src/test/resources/secure-zip/strong/store_solid_aes256_strong_ecd.zip");
35+
Path zip = Paths.get("d:/zip4jvm/zip64/multi/aes_10k.zip");
36+
Path dstDir = Paths.get("d:/zip4jvm/zip64/multi/out");
5437

55-
//Path zip = Paths.get("d:/zip4jvm/zip64/src.zip");
56-
// Path zip = Paths.get("d:/zip4jvm/scd/aes256bit.zip");
57-
// Path zip = Paths.get("d:/zip4jvm/scd/P1AA4B3C.zip");
58-
Path zip = Paths.get("d:/zip4jvm/scd/onetwo.zip");
59-
// Path zip = Paths.get("D:/Programming/GitHub/zip4jvm/src/test/resources/symlink/win/unique-symlink-target.zip");
60-
Path dstDir = Paths.get("d:/zip4jvm/scd/xxx");
61-
62-
// ZipIt.zip(zip).settings(settings).add(dirSrcData);
63-
64-
65-
// for (Path zip : Arrays.asList(zip1, zip2)) {
66-
// System.out.println(zip);
67-
// UnzipIt.zip(zip).dstDir(dstDir)
68-
// .settings(UnzipSettings.builder()
69-
// .password(password)
70-
// .build())
71-
// .extract();
72-
// ZipInfo.zip(zip).password("1".toCharArray()).printShortInfo();
73-
ZipInfo.zip(zip)
74-
.settings(ZipInfoSettings.builder()
75-
.copyPayload(true)
76-
.readEntries(true)
77-
.build())
78-
.password("1".toCharArray())
79-
.decompose(Paths.get(dstDir.toString(), zip.getFileName().toString()));
38+
UnzipIt.zip(zip).dstDir(dstDir).extract();
8039

8140
final long timeTo = System.currentTimeMillis();
82-
System.out.format("Time: %d sec", TimeUnit.MILLISECONDS.toSeconds(timeTo - timeFrom));
41+
long millis = timeTo - timeFrom;
42+
long minutes = (millis / 1000) / 60;
43+
int seconds = (int) ((millis / 1000) % 60);
44+
System.out.format("Time: %02d:%02d", minutes, seconds);
8345
}
46+
8447
}

0 commit comments

Comments
 (0)