Merge pull request #692 from FgForrest/unified-merge

Unified merge

docker/Dockerfile

@@ -16,7 +16,7 @@ ENV SYSTEM_API_PORT="5555"
 
 ## folder with configuration files, all of them are applied one on top of another in alphabetical order
 ENV EVITA_BIN_DIR="$EVITA_HOME/bin/"
-ENV EVITA_CONFIG_DIR="$EVITA_HOME/conf/"
+ENV EVITA_CONFIG_DIR="$EVITA_HOME/config/"
 ENV EVITA_STORAGE_DIR="$EVITA_HOME/data/"
 ENV EVITA_EXPORT_DIR="$EVITA_HOME/export/"
 ENV EVITA_CERTIFICATE_DIR="$EVITA_HOME/certificates/"

evita_external_api/evita_external_api_core/src/main/java/io/evitadb/externalApi/api/system/model/HealthProblem.java → evita_api/src/main/java/io/evitadb/api/observability/HealthProblem.java

@@ -21,7 +21,7 @@
  *   limitations under the License.
  */
 
-package io.evitadb.externalApi.api.system.model;
+package io.evitadb.api.observability;
 
 /**
  * This enum represents the possible health problems that can be signaled by the server.

evita_engine/src/main/java/io/evitadb/store/spi/model/storageParts/RemovedStoragePart.java → evita_api/src/main/java/io/evitadb/api/observability/ReadinessState.java

@@ -6,7 +6,7 @@
  *             |  __/\ V /| | || (_| | |_| | |_) |
  *              \___| \_/ |_|\__\__,_|____/|____/
  *
- *   Copyright (c) 2023
+ *   Copyright (c) 2024
  *
  *   Licensed under the Business Source License, Version 1.1 (the "License");
  *   you may not use this file except in compliance with the License.
@@ -21,28 +21,35 @@
  *   limitations under the License.
  */
 
-package io.evitadb.store.spi.model.storageParts;
+package io.evitadb.api.observability;
 
-import io.evitadb.store.model.StoragePart;
-import io.evitadb.store.service.KeyCompressor;
-
-import javax.annotation.Nonnull;
-import java.io.Serial;
 
 /**
- * This class marks removed {@link StoragePart} in the transactional memory.
+ * Enum representing overall readiness state of the server.
+ *
+ * @author Jan Novotný (novotny@fg.cz), FG Forrest a.s. (c) 2024
  */
-public class RemovedStoragePart implements StoragePart {
-	public static final RemovedStoragePart INSTANCE = new RemovedStoragePart();
-	@Serial private static final long serialVersionUID = 2485318464734970542L;
+public enum ReadinessState {
 
-	@Override
-	public Long getStoragePartPK() {
-		throw new UnsupportedOperationException();
-	}
+	/**
+	 * At least one API is not ready.
+	 */
+	STARTING,
+	/**
+	 * All APIs are ready.
+	 */
+	READY,
+	/**
+	 * At least one API that was ready is not ready anymore.
+	 */
+	STALLING,
+	/**
+	 * Server is shutting down. None of the APIs are ready.
+	 */
+	SHUTDOWN,
+	/**
+	 * Unknown state - cannot determine the state of the APIs (should not happen).
+	 */
+	UNKNOWN
 
-	@Override
-	public long computeUniquePartIdAndSet(@Nonnull KeyCompressor keyCompressor) {
-		throw new UnsupportedOperationException();
-	}
 }

evita_api/src/main/java/module-info.java

@@ -21,13 +21,11 @@
  *   limitations under the License.
  */
 
-import io.evitadb.api.observability.trace.TracingContext;
-
 /**
  * Module contains external API of the evitaDB.
  */
 module evita.api {
-	uses TracingContext;
+	uses io.evitadb.api.observability.trace.TracingContext;
 
 	opens io.evitadb.api.configuration to com.fasterxml.jackson.databind;
 	opens io.evitadb.api.requestResponse.extraResult to com.graphqljava;
@@ -67,6 +65,7 @@
 	exports io.evitadb.api.requestResponse.schema.mutation.sortableAttributeCompound;
 	exports io.evitadb.api.requestResponse.data.annotation;
 	exports io.evitadb.api.requestResponse.transaction;
+	exports io.evitadb.api.observability;
 	exports io.evitadb.api.observability.trace;
 	exports io.evitadb.api.observability.annotation;
 

evita_engine/src/main/java/io/evitadb/core/Catalog.java

@@ -72,6 +72,7 @@
 import io.evitadb.api.requestResponse.system.TimeFlow;
 import io.evitadb.api.requestResponse.transaction.TransactionMutation;
 import io.evitadb.api.task.ServerTask;
+import io.evitadb.core.EntityCollection.EntityCollectionHeaderWithCollection;
 import io.evitadb.core.async.ObservableExecutorService;
 import io.evitadb.core.async.Scheduler;
 import io.evitadb.core.buffer.DataStoreChanges;
@@ -849,7 +850,7 @@ public boolean goLive() {
 
 			return true;
 		} finally {
-			goingLive.set(false);
+			this.goingLive.set(false);
 		}
 	}
 
@@ -920,7 +921,11 @@ public void terminate() {
 				// in warmup state try to persist all changes in volatile memory
 				if (warmingUpState) {
 					final long lastSeenVersion = entityCollection.getVersion();
-					entityHeaders.add(entityCollection.flush());
+					entityHeaders.add(
+						updateIndexIfNecessary(
+							entityCollection.flush()
+						)
+					);
 					changeOccurred = changeOccurred || entityCollection.getVersion() != lastSeenVersion;
 				}
 				// in all states terminate collection operations
@@ -933,7 +938,7 @@ public void terminate() {
 				this.persistenceService.storeHeader(
 					this.catalogId,
 					getCatalogState(),
-					this.versionId.getId(),
+					getVersion(),
 					this.entityTypeSequence.get(),
 					null,
 					entityHeaders,
@@ -1193,7 +1198,7 @@ public void flush(long catalogVersion, @Nonnull TransactionMutation lastProcesse
 		if (changeOccurred) {
 			this.persistenceService.flushTrappedUpdates(
 				catalogVersion,
-				this.dataStoreBuffer.getTrappedIndexChanges()
+				this.dataStoreBuffer.getTrappedChanges()
 			);
 			this.persistenceService.storeHeader(
 				this.catalogId,
@@ -1311,14 +1316,23 @@ void flush() {
 		final List<EntityCollectionHeader> entityHeaders = new ArrayList<>(this.entityCollections.size());
 		for (EntityCollection entityCollection : this.entityCollections.values()) {
 			final long lastSeenVersion = entityCollection.getVersion();
-			entityHeaders.add(entityCollection.flush());
+			entityHeaders.add(
+				updateIndexIfNecessary(
+					entityCollection.flush()
+				)
+			);
 			changeOccurred = changeOccurred || entityCollection.getVersion() != lastSeenVersion;
 		}
 
 		if (changeOccurred) {
 			this.persistenceService.flushTrappedUpdates(
 				0L,
-				this.dataStoreBuffer.getTrappedIndexChanges()
+				this.dataStoreBuffer.getTrappedChanges()
+			);
+			final CatalogHeader catalogHeader = this.persistenceService.getCatalogHeader(0L);
+			Assert.isPremiseValid(
+				catalogHeader != null && catalogHeader.catalogState() == CatalogState.WARMING_UP,
+				"Catalog header is expected to be present in the storage in WARMING_UP flag!"
 			);
 			this.persistenceService.storeHeader(
 				this.catalogId,
@@ -1333,6 +1347,27 @@ void flush() {
 		}
 	}
 
+	/**
+	 * Method transparently updates the contents of {@link #entityCollections} map with the new collection, if the
+	 * passed {@link EntityCollectionHeaderWithCollection} contains a different collection than the one stored in
+	 * the index.
+	 *
+	 * @param flushResult The result containing the header and the entity collection to potentially update.
+	 * @return The entity collection header from the flush result.
+	 */
+	@Nonnull
+	private EntityCollectionHeader updateIndexIfNecessary(
+		@Nonnull EntityCollectionHeaderWithCollection flushResult
+	) {
+		final EntityCollectionHeader header = flushResult.header();
+		this.entityCollections.computeIfPresent(
+			header.entityType(),
+			(entityType, entityCollection) -> entityCollection == flushResult.collection() ?
+				entityCollection : flushResult.collection()
+		);
+		return header;
+	}
+
 	/*
 		PRIVATE METHODS
 	 */
@@ -1484,7 +1519,11 @@ private CatalogSchemaContract removeEntitySchema(
 			final long catalogVersion = getVersion();
 			this.persistenceService.deleteEntityCollection(
 				catalogVersion,
-				catalogVersion > 0L ? collectionToRemove.flush(catalogVersion) : collectionToRemove.flush()
+				catalogVersion > 0L ?
+					collectionToRemove.flush(catalogVersion) :
+					updateIndexIfNecessary(
+						collectionToRemove.flush()
+					)
 			);
 		}
 		final CatalogSchemaContract result;
@@ -1591,7 +1630,9 @@ private void doReplaceEntityCollectionInternal(
 		entityCollectionToBeReplacedWith.updateSchema(getSchema(), modifyEntitySchemaName);
 		this.entityCollections.remove(entityCollectionNameToBeReplacedWith);
 		if (!transactionOpen) {
-			entityCollectionToBeReplacedWith.flush();
+			updateIndexIfNecessary(
+				entityCollectionToBeReplacedWith.flush()
+			);
 			final long catalogVersion = getVersion();
 			Assert.isPremiseValid(catalogVersion == 0L, "Catalog version is expected to be `0`!");
 			final EntityCollectionPersistenceService newPersistenceService = this.persistenceService.replaceCollectionWith(
@@ -1611,7 +1652,9 @@ private void doReplaceEntityCollectionInternal(
 					entityCollectionNameToBeReplacedWith, entityCollectionToBeReplacedWith
 				);
 				if (schemaUpdated) {
-					otherCollection.flush();
+					updateIndexIfNecessary(
+						otherCollection.flush()
+					);
 				}
 			}
 			// store catalog with a new file pointer