Skip to content

Commit ddfa14a

Browse files
alvaswalvasw
committed
Set wasPersistableNetworkPayloadsTruncated if requester Version newer and capped
Cap items if requesterVersion is newer in HistoricalDataStoreService.getMapSinceVersion(...)
1 parent 97330ba commit ddfa14a

File tree

4 files changed

+46
-19
lines changed

4 files changed

+46
-19
lines changed

‎core/src/main/java/bisq/core/network/p2p/inventory/GetInventoryRequestHandler.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import java.util.HashMap;
5656
import java.util.LinkedList;
5757
import java.util.Map;
58+
import java.util.concurrent.atomic.AtomicBoolean;
5859

5960
import java.lang.management.ManagementFactory;
6061

@@ -101,7 +102,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) {
101102
// Data
102103
GetInventoryRequest getInventoryRequest = (GetInventoryRequest) networkEnvelope;
103104
Map<InventoryItem, Integer> dataObjects = new HashMap<>();
104-
p2PDataStorage.getMapForDataResponse(getInventoryRequest.getVersion(), Collections.emptySet())
105+
p2PDataStorage.getMapForDataResponse(getInventoryRequest.getVersion(), Collections.emptySet(), new AtomicBoolean(false))
105106
.values().stream()
106107
.map(e -> e.getClass().getSimpleName())
107108
.forEach(className -> addClassNameToMap(dataObjects, className));

‎core/src/test/java/bisq/core/account/witness/AccountAgeWitnessStorageServiceTest.java

+24-10
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import java.util.HashSet;
1616
import java.util.Map;
1717
import java.util.Set;
18+
import java.util.concurrent.atomic.AtomicBoolean;
1819

1920
import org.junit.jupiter.api.BeforeEach;
2021
import org.junit.jupiter.api.Test;
@@ -26,6 +27,7 @@
2627

2728

2829
public class AccountAgeWitnessStorageServiceTest {
30+
private final AtomicBoolean payloadsTruncated = new AtomicBoolean(false);
2931
private DummyHistoricalDataStoreService storageService;
3032

3133
@BeforeEach
@@ -37,8 +39,9 @@ void setup(@TempDir Path tempDirPath) {
3739
@Test
3840
void emptyStore() {
3941
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
40-
storageService.getMapSinceVersion(Version.VERSION, Collections.emptySet());
42+
storageService.getMapSinceVersion(Version.VERSION, Collections.emptySet(), payloadsTruncated);
4143
assertThat(mapSinceVersion, is(anEmptyMap()));
44+
assertThat(payloadsTruncated.get(), is(false));
4245
}
4346

4447
@Test
@@ -47,7 +50,7 @@ void testOnlyLiveData() {
4750
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessesToMap(liveDataMap, 2);
4851

4952
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
50-
storageService.getMapSinceVersion(Version.VERSION, Collections.emptySet());
53+
storageService.getMapSinceVersion(Version.VERSION, Collections.emptySet(), payloadsTruncated);
5154

5255
P2PDataStorage.ByteArray firstByteArray = new P2PDataStorage.ByteArray(new byte[]{0});
5356
P2PDataStorage.ByteArray secondByteArray = new P2PDataStorage.ByteArray(new byte[]{1});
@@ -57,6 +60,7 @@ void testOnlyLiveData() {
5760
secondByteArray, liveDataMap.get(secondByteArray));
5861

5962
assertThat(mapSinceVersion, is(expected));
63+
assertThat(payloadsTruncated.get(), is(false));
6064
}
6165

6266
@Test
@@ -70,10 +74,11 @@ void testOnlyStoreData() {
7074
storageService.setStoresByVersion(storeByVersion);
7175

7276
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
73-
storageService.getMapSinceVersion("1.7.0", Collections.emptySet());
77+
storageService.getMapSinceVersion("1.7.0", Collections.emptySet(), payloadsTruncated);
7478

7579
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>(versionStore.getMap());
7680
assertThat(mapSinceVersion, is(expected));
81+
assertThat(payloadsTruncated.get(), is(false));
7782
}
7883

7984
@Test
@@ -93,7 +98,7 @@ void testRequesterVersionNullCappedLiveData() {
9398
storageService.setStoresByVersion(storeByVersion);
9499

95100
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
96-
storageService.getMapSinceVersion(null, Collections.emptySet());
101+
storageService.getMapSinceVersion(null, Collections.emptySet(), payloadsTruncated);
97102

98103
int mapSize = mapSinceVersion.size();
99104
assertThat(mapSize, is(100));
@@ -102,6 +107,8 @@ void testRequesterVersionNullCappedLiveData() {
102107
var expected = liveDataMap.get(entry.getKey());
103108
assertThat(entry.getValue(), is(expected));
104109
}
110+
111+
assertThat(payloadsTruncated.get(), is(false));
105112
}
106113

107114
@Test
@@ -121,14 +128,15 @@ void testRequesterUncappedData() {
121128
storageService.setStoresByVersion(storeByVersion);
122129

123130
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
124-
storageService.getMapSinceVersion("1.2.3", Collections.emptySet());
131+
storageService.getMapSinceVersion("1.2.3", Collections.emptySet(), payloadsTruncated);
125132

126133
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>();
127134
expected.putAll(liveDataMap);
128135
expected.putAll(firstVersionStore.getMap());
129136
expected.putAll(secondVersionStore.getMap());
130137

131138
assertThat(mapSinceVersion, is(expected));
139+
assertThat(payloadsTruncated.get(), is(false));
132140
}
133141

134142
@Test
@@ -148,13 +156,14 @@ void testRequesterVersionIsOlder() {
148156
storageService.setStoresByVersion(storeByVersion);
149157

150158
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
151-
storageService.getMapSinceVersion("1.8.5", Collections.emptySet());
159+
storageService.getMapSinceVersion("1.8.5", Collections.emptySet(), payloadsTruncated);
152160

153161
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>();
154162
expected.putAll(liveDataMap);
155163
expected.putAll(secondVersionStore.getMap());
156164

157165
assertThat(mapSinceVersion, is(expected));
166+
assertThat(payloadsTruncated.get(), is(false));
158167
}
159168

160169
@Test
@@ -174,10 +183,11 @@ void testRequesterHasHistoricalDataStores() {
174183
storageService.setStoresByVersion(storeByVersion);
175184

176185
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
177-
storageService.getMapSinceVersion("1.9.5", Collections.emptySet());
186+
storageService.getMapSinceVersion("1.9.5", Collections.emptySet(), payloadsTruncated);
178187

179188
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>(liveDataMap);
180189
assertThat(mapSinceVersion, is(expected));
190+
assertThat(payloadsTruncated.get(), is(false));
181191
}
182192

183193
@Test
@@ -201,7 +211,7 @@ void testRequesterVersionIsNewerCapped() {
201211
Version.getMinorVersion(version) + "." + (Version.getPatchVersion(version) + 1);
202212

203213
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
204-
storageService.getMapSinceVersion(higherVersion, Collections.emptySet());
214+
storageService.getMapSinceVersion(higherVersion, Collections.emptySet(), payloadsTruncated);
205215

206216
int mapSize = mapSinceVersion.size();
207217
assertThat(mapSize, is(100));
@@ -210,6 +220,8 @@ void testRequesterVersionIsNewerCapped() {
210220
var expected = liveDataMap.get(entry.getKey());
211221
assertThat(entry.getValue(), is(expected));
212222
}
223+
224+
assertThat(payloadsTruncated.get(), is(true));
213225
}
214226

215227
@Test
@@ -240,7 +252,7 @@ void testKnownHashes() {
240252
Set<P2PDataStorage.ByteArray> knownHashes = Set.of(excludeFromLiveMap, excludeFromSecondStore);
241253

242254
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
243-
storageService.getMapSinceVersion("1.7.0", knownHashes);
255+
storageService.getMapSinceVersion("1.7.0", knownHashes, payloadsTruncated);
244256

245257
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>(liveDataMap);
246258
expected.putAll(firstVersionStore.getMap());
@@ -250,6 +262,7 @@ void testKnownHashes() {
250262
expected.remove(excludeFromSecondStore);
251263

252264
assertThat(mapSinceVersion, is(expected));
265+
assertThat(payloadsTruncated.get(), is(false));
253266
}
254267

255268
@Test
@@ -273,9 +286,10 @@ void testExcludeAllHashes() {
273286
knownHashes.addAll(secondStoreKeys);
274287

275288
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
276-
storageService.getMapSinceVersion("1.8.5", knownHashes);
289+
storageService.getMapSinceVersion("1.8.5", knownHashes, payloadsTruncated);
277290

278291
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = Collections.emptyMap();
279292
assertThat(mapSinceVersion, is(expected));
293+
assertThat(payloadsTruncated.get(), is(false));
280294
}
281295
}

‎p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java

+6-3
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,8 @@ public GetDataResponse buildGetDataResponse(
314314
// mapForDataResponse contains the filtered by version data from HistoricalDataStoreService as well as all other
315315
// maps of the remaining appendOnlyDataStoreServices.
316316
Map<ByteArray, PersistableNetworkPayload> mapForDataResponse =
317-
getMapForDataResponse(getDataRequest.getVersion(), excludedKeysAsByteArray);
317+
getMapForDataResponse(getDataRequest.getVersion(),
318+
excludedKeysAsByteArray, wasPersistableNetworkPayloadsTruncated);
318319

319320
// Give a bit of tolerance for message overhead
320321
double maxSize = Connection.getMaxPermittedMessageSize() * 0.6;
@@ -393,14 +394,16 @@ private Map<ByteArray, PersistableNetworkPayload> getMapForDataRequest() {
393394
return map;
394395
}
395396

396-
public Map<ByteArray, PersistableNetworkPayload> getMapForDataResponse(String requestersVersion, Set<P2PDataStorage.ByteArray> keysToExclude) {
397+
public Map<ByteArray, PersistableNetworkPayload> getMapForDataResponse(String requestersVersion,
398+
Set<P2PDataStorage.ByteArray> keysToExclude,
399+
AtomicBoolean arePayloadsTruncated) {
397400
Map<ByteArray, PersistableNetworkPayload> map = new HashMap<>();
398401
appendOnlyDataStoreService.getServices()
399402
.forEach(service -> {
400403
Map<ByteArray, PersistableNetworkPayload> serviceMap;
401404
if (service instanceof HistoricalDataStoreService) {
402405
var historicalDataStoreService = (HistoricalDataStoreService<? extends PersistableNetworkPayloadStore>) service;
403-
serviceMap = historicalDataStoreService.getMapSinceVersion(requestersVersion, keysToExclude);
406+
serviceMap = historicalDataStoreService.getMapSinceVersion(requestersVersion, keysToExclude, arePayloadsTruncated);
404407
} else {
405408
serviceMap = service.getMap();
406409
}

‎p2p/src/main/java/bisq/network/p2p/storage/persistence/HistoricalDataStoreService.java

+14-5
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.util.HashMap;
3232
import java.util.Map;
3333
import java.util.Set;
34+
import java.util.concurrent.atomic.AtomicBoolean;
3435
import java.util.concurrent.atomic.AtomicInteger;
3536
import java.util.stream.Collectors;
3637
import java.util.stream.Stream;
@@ -45,6 +46,7 @@
4546
*/
4647
@Slf4j
4748
public abstract class HistoricalDataStoreService<T extends PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> extends MapStoreService<T, PersistableNetworkPayload> {
49+
private static final int MAX_ITEMS = 100;
4850
protected ImmutableMap<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storesByVersion;
4951
// Cache to avoid that we have to recreate the historical data at each request
5052
private ImmutableMap<P2PDataStorage.ByteArray, PersistableNetworkPayload> allHistoricalPayloads;
@@ -66,27 +68,34 @@ public HistoricalDataStoreService(File storageDir, PersistenceManager<T> persist
6668
// We give back a map of our live map and all historical maps newer than the requested version.
6769
// If requestersVersion is null we return all historical data.
6870
public Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> getMapSinceVersion(String requestersVersion,
69-
Set<P2PDataStorage.ByteArray> keysToExclude) {
71+
Set<P2PDataStorage.ByteArray> keysToExclude,
72+
AtomicBoolean wasPersistableNetworkPayloadsTruncated) {
7073
// Get live data stream
7174
Stream<Map.Entry<P2PDataStorage.ByteArray, PersistableNetworkPayload>> liveDataStream =
7275
store.getMap().entrySet().stream();
7376

7477

7578
if (requestersVersion == null) {
7679
log.warn("The requester did not send a version but the field was added in v1.4.0. " +
77-
"Returning capped live data (100 items).");
80+
"Returning capped live data (" + MAX_ITEMS + " items).");
7881

7982
return liveDataStream
80-
.limit(100)
83+
.limit(MAX_ITEMS)
8184
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
8285
}
8386

8487
boolean isRequesterVersionNewer = Version.isNewVersion(requestersVersion, Version.VERSION);
8588
if (isRequesterVersionNewer) {
86-
log.warn("The requester's version is newer than ours. Returning capped live data (100 items).");
89+
log.warn("The requester's version is newer than ours. Returning capped live data (" +
90+
MAX_ITEMS + " items).");
91+
92+
var numberOfLiveData = store.getMap().size();
93+
if (numberOfLiveData > MAX_ITEMS) {
94+
wasPersistableNetworkPayloadsTruncated.set(true);
95+
}
8796

8897
return liveDataStream
89-
.limit(100)
98+
.limit(MAX_ITEMS)
9099
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
91100
}
92101

0 commit comments

Comments
 (0)