Merge pull request #684 from FgForrest/dev

Patch release

docker/Dockerfile

@@ -18,6 +18,7 @@ ENV SYSTEM_API_PORT="5555"
 ENV EVITA_BIN_DIR="$EVITA_HOME/bin/"
 ENV EVITA_CONFIG_DIR="$EVITA_HOME/conf/"
 ENV EVITA_STORAGE_DIR="$EVITA_HOME/data/"
+ENV EVITA_EXPORT_DIR="$EVITA_HOME/export/"
 ENV EVITA_CERTIFICATE_DIR="$EVITA_HOME/certificates/"
 ENV EVITA_LOG_FILE="$EVITA_HOME/logback.xml"
 
@@ -44,7 +45,7 @@ RUN apt update && apt install -y netcat-openbsd iproute2 iputils-ping mc tcpdump
 
 # Create necessary folders
 RUN set -ex \
-    && mkdir "$EVITA_HOME" "$EVITA_BIN_DIR" "$EVITA_CONFIG_DIR" "$EVITA_STORAGE_DIR" "$EVITA_CERTIFICATE_DIR" \
+    && mkdir "$EVITA_HOME" "$EVITA_BIN_DIR" "$EVITA_CONFIG_DIR" "$EVITA_STORAGE_DIR" "$EVITA_EXPORT_DIR" "$EVITA_CERTIFICATE_DIR" \
     && : ## end
 
 # Copy files

docker/README.MD

@@ -27,6 +27,7 @@ docker run -i --rm --net=host \
 ## Environment variables
 - **EVITA_CONFIG_FILE** - Path to configuration file. Default: `/evita/conf/evita-configuration.yaml`
 - **EVITA_STORAGE_DIR** - Path to storage directory. Default: `/evita/data`
+- **EVITA_EXPORT_DIR** - Path to export directory (temporary storage for backup / exports). Default: `/evita/export`
 - **EVITA_CERTIFICATE_DIR** - Path to directory with automatically generated server certificates. Default: `/evita/certificates`
 - **EVITA_JAVA_OPTS** - Java commandline options. Default: none (empty string)
 - **EVITA_ARGS** - Evita Server command-line arguments. Default: none (empty string) 

documentation/blog/en/07-advanced-features-on-developers-portal.md

@@ -51,7 +51,7 @@ Each code block is then rendered as an individual tab within the `<CodeTabsBlock
 <dependency>
     <groupId>io.evitadb</groupId>
     <artifactId>evita_test_support</artifactId>
-    <version>2024.8.4</version>
+    <version>2024.10.0</version>
     <scope>test</scope>
 </dependency>
 ```

documentation/blog/en/examples/string_contains/bom.gradle

@@ -21,4 +21,4 @@
  *   limitations under the License.
  */
 
-implementation 'io.evitadb:evita_external_api_graphql:2024.8.4'
+implementation 'io.evitadb:evita_external_api_graphql:2024.10.0'

documentation/blog/en/examples/string_contains/bom.maven

@@ -1,6 +1,6 @@
 <dependency>
     <groupId>io.evitadb</groupId>
     <artifactId>evita_external_api_graphql</artifactId>
-    <version>2024.8.4</version>
+    <version>2024.10.0</version>
     <type>pom</type>
 </dependency>

documentation/blog/en/examples/string_contains/model.xml

@@ -24,6 +24,6 @@
 <dependency>
 	<groupId>io.evitadb</groupId>
 	<artifactId>evita_external_api_graphql</artifactId>
-	<version>2024.8.4</version>
+	<version>2024.10.0</version>
 	<type>pom</type>
 </dependency>

documentation/publishing/root.cz/02-write-data/02-write-data.html

@@ -90,7 +90,7 @@ <h2>Zprovoznění Docker kontejneru</h2>
 |  __/\ V /| | || (_| | |_| | |_) |
  \___| \_/ |_|\__\__,_|____/|____/
 
-beta build 2024.8.4 (keep calm and report bugs 😉)
+beta build 2024.10.0 (keep calm and report bugs 😉)
 Visit us at: https://evitadb.io
 
 Log config used: META-INF/logback.xml (original file `/evita/logback.xml` doesn't exist)

documentation/user/en/get-started/query-our-dataset.md

@@ -83,7 +83,7 @@ When this procedure is completed you should see the similar output in the consol
 |  __/\ V /| | || (_| | |_| | |_) |
  \___| \_/ |_|\__\__,_|____/|____/
 
-beta build 2024.8.4 (keep calm and report bugs 😉)
+beta build 2024.10.0 (keep calm and report bugs 😉)
 Visit us at: https://evitadb.io
 
 19:45:37.088 INFO  i.e.s.c.DefaultCatalogPersistenceService - Catalog `evita` is being loaded and  it contains:
@@ -136,13 +136,13 @@ Open your Java IDE and add the following dependency to your project:
 <dependency>
     <groupId>io.evitadb</groupId>
     <artifactId>evita_java_driver</artifactId>
-    <version>2024.8.4</version>
+    <version>2024.10.0</version>
 </dependency>
 ```
 </CodeTabsBlock>
 <CodeTabsBlock>
 ```Gradle
-implementation 'io.evitadb:evita_java_driver:2024.8.4'
+implementation 'io.evitadb:evita_java_driver:2024.10.0'
 ```
 </CodeTabsBlock>
 </CodeTabs>

documentation/user/en/get-started/run-evitadb.md

@@ -63,14 +63,14 @@ To integrate evitaDB into your project, use the following steps:
 <dependency>
     <groupId>io.evitadb</groupId>
     <artifactId>evita_db</artifactId>
-    <version>2024.8.4</version>
+    <version>2024.10.0</version>
     <type>pom</type>
 </dependency>
 ```
 </CodeTabsBlock>
 <CodeTabsBlock>
 ```Gradle
-implementation 'io.evitadb:evita_db:2024.8.4'
+implementation 'io.evitadb:evita_db:2024.10.0'
 ```
 </CodeTabsBlock>
 </CodeTabs>
@@ -108,14 +108,14 @@ exception when you enable the corresponding API in evitaDB's configuration.
 <dependency>
     <groupId>io.evitadb</groupId>
     <artifactId>evita_external_api_grpc</artifactId>
-    <version>2024.8.4</version>
+    <version>2024.10.0</version>
     <type>pom</type>
 </dependency>
 ```
 </CodeTabsBlock>
 <CodeTabsBlock>
 ```Gradle
-implementation 'io.evitadb:evita_external_api_grpc:2024.8.4'
+implementation 'io.evitadb:evita_external_api_grpc:2024.10.0'
 ```
 </CodeTabsBlock>
 </CodeTabs>
@@ -128,14 +128,14 @@ implementation 'io.evitadb:evita_external_api_grpc:2024.8.4'
 <dependency>
     <groupId>io.evitadb</groupId>
     <artifactId>evita_external_api_graphql</artifactId>
-    <version>2024.8.4</version>
+    <version>2024.10.0</version>
     <type>pom</type>
 </dependency>
 ```
 </CodeTabsBlock>
 <CodeTabsBlock>
 ```Gradle
-implementation 'io.evitadb:evita_external_api_graphql:2024.8.4'
+implementation 'io.evitadb:evita_external_api_graphql:2024.10.0'
 ```
 </CodeTabsBlock>
 </CodeTabs>
@@ -148,14 +148,14 @@ implementation 'io.evitadb:evita_external_api_graphql:2024.8.4'
 <dependency>
     <groupId>io.evitadb</groupId>
     <artifactId>evita_external_api_rest</artifactId>
-    <version>2024.8.4</version>
+    <version>2024.10.0</version>
     <type>pom</type>
 </dependency>
 ```
 </CodeTabsBlock>
 <CodeTabsBlock>
 ```Gradle
-implementation 'io.evitadb:evita_external_api_rest:2024.8.4'
+implementation 'io.evitadb:evita_external_api_rest:2024.10.0'
 ```
 </CodeTabsBlock>
 </CodeTabs>
@@ -225,7 +225,7 @@ When you start the evitaDB server you should see the following information in th
 |  __/\ V /| | || (_| | |_| | |_) |
  \___| \_/ |_|\__\__,_|____/|____/
 
-beta build 2024.8.4 (keep calm and report bugs 😉)
+beta build 2024.10.0 (keep calm and report bugs 😉)
 Visit us at: https://evitadb.io
 
 Log config used: META-INF/logback.xml

documentation/user/en/operate/configure.md

@@ -35,14 +35,18 @@ server:                                           # [see Server configuration](#
   quiet: false
 
 storage:                                          # [see Storage configuration](#storage-configuration)
-  storageDirectory: null
+  storageDirectory: "./data"
+  exportDirectory: "./export"
   lockTimeoutSeconds: 60
   waitOnCloseSeconds: 60
   outputBufferSize: 4MB
   maxOpenedReadHandles: 12
   computeCRC32C: true
   minimalActiveRecordShare: 0.5
   fileSizeCompactionThresholdBytes: 100MB
+  timeTravelEnabled: true
+  exportDirectorySizeLimitBytes: 1G
+  exportFileHistoryExpirationSeconds: 7d
 
 transaction:                                      # [see Transaction configuration](#transaction-configuration)
   transactionWorkDirectory: /tmp/evitaDB/transaction
@@ -428,10 +432,17 @@ This section contains configuration options for the storage layer of the databas
 <dl>
     <dt>storageDirectory</dt>
     <dd>
-        <p>**Default:** `null`</p>
+        <p>**Default:** `./data`</p>
         <p>It defines the folder where evitaDB stores its catalog data. The path can be specified relative to the working
         directory of the application in absolute form (recommended).</p>
     </dd>
+    <dt>exportDirectory</dt>
+    <dd>
+        <p>**Default:** `./export`</p>
+        <p>It defines the folder where evitaDB stores its exported files. The path can be specified relative to the working
+        directory of the application in absolute form (recommended). Files are automatically removed according to limits
+        defined in `exportFileHistoryExpirationSeconds` and `exportDirectorySizeLimitBytes`.</p>
+    </dd>
     <dt>lockTimeoutSeconds</dt>
     <dd>
         <p>**Default:** `60`</p>
@@ -492,6 +503,18 @@ This section contains configuration options for the storage layer of the databas
         in the history covered by the WAL log. From the snapshot, the database can be restored to the exact point in 
         time with all the data available at that time.</p>
     </dd>
+    <dt>exportDirectorySizeLimitBytes</dt>
+    <dd>
+        <p>**Default:** `1G`</p>
+        <p>It specifies the maximum size of the export directory. If the size of the directory exceeds this limit, the 
+        oldest files are removed until the size of the directory is below the limit.</p>
+    </dd>
+    <dt>exportFileHistoryExpirationSeconds</dt>
+    <dd>
+        <p>**Default:** `7d`</p>
+        <p>It specifies the maximum age of the files in the export directory. If the age of the file exceeds this limit, 
+        the file is removed from the directory.</p>
+    </dd>
 </dl>
 
 ## Transaction configuration

documentation/user/en/operate/reference/metrics.md

@@ -3,6 +3,8 @@
 <UsedTerms>
   <h4>Labels used in metrics</h4>
   <dl>
+    <dt>api</dt>
+    <dd><strong>API type</strong>: The identification of the API being probed.</dd>
     <dt>buildType</dt>
     <dd><strong>Build type</strong>: Type of the instance build: NEW or REFRESH</dd>
     <dt>catalogName</dt>
@@ -31,6 +33,10 @@
     <dd><strong>Prefetched vs. non-prefetched query</strong>: Whether or not the query used a prefetch plan. Prefetch plan optimistically fetches queried entities in advance and executes directly on them (without accessing the indexes).</dd>
     <dt>procedureName</dt>
     <dd><strong>Procedure name</strong>: Name of the gRPC procedure that was called (the method name).</dd>
+    <dt>prospective</dt>
+    <dd><strong>Prospective (client/server)</strong>: Identifies whether the event represents whether event represents server or client view of readiness.
+Client view is the duration viewed from the HTTP client side affected by timeouts, server view is the real
+duration of the probe.</dd>
     <dt>recordType</dt>
     <dd><strong>Record type</strong>: Type of records that changed in the OffsetIndex.</dd>
     <dt>resolution</dt>
@@ -41,6 +47,8 @@
     <dd><strong>REST instance type</strong>: Domain of the REST API used in connection with this event/metric: SYSTEM, or CATALOG</dd>
     <dt>restOperationType</dt>
     <dd><strong>REST operation type</strong>: The type of operation that was executed. One of: QUERY, MUTATION.</dd>
+    <dt>result</dt>
+    <dd><strong>Result</strong>: The result of the readiness probe (ok, timeout, error).</dd>
     <dt>serviceName</dt>
     <dd><strong>Service name</strong>: Name of the gRPC service that was called (the name of the Java class).</dd>
     <dt>stage</dt>
@@ -50,6 +58,15 @@
   </dl>
 </UsedTerms>
 
+#### API
+
+<dl>
+  <dt><code>io_evitadb_external_api_readiness_duration_milliseconds</code> (HISTOGRAM)</dt>
+  <dd>Readiness probe duration<br/><br/><strong>Labels:</strong> <Term>api</Term>, <Term>prospective</Term>, <Term>result</Term><br/></dd>
+  <dt><code>io_evitadb_external_api_readiness_total</code> (COUNTER)</dt>
+  <dd>Readiness probe invoked total<br/><br/><strong>Labels:</strong> <Term>api</Term>, <Term>prospective</Term>, <Term>result</Term><br/></dd>
+</dl>
+
 #### API / GraphQL / Instance / Schema
 
 <dl>

documentation/user/en/use/api/write-tests.md

@@ -44,14 +44,14 @@ artifact into your project:
 <dependency>
     <groupId>io.evitadb</groupId>
     <artifactId>evita_test_support</artifactId>
-    <version>2024.8.4</version>
+    <version>2024.10.0</version>
     <scope>test</scope>
 </dependency>
 ```
 </CodeTabsBlock>
 <CodeTabsBlock>
 ```Gradle
-implementation 'io.evitadb:evita_test_support:2024.8.4'
+implementation 'io.evitadb:evita_test_support:2024.10.0'
 ```
 </CodeTabsBlock>
 </CodeTabs>

documentation/user/en/use/connectors/java.md

@@ -30,13 +30,13 @@ In order to use a Java remote client you need only to add following dependency t
 <dependency>
     <groupId>io.evitadb</groupId>
     <artifactId>evita_java_driver</artifactId>
-    <version>2024.8.4</version>
+    <version>2024.10.0</version>
 </dependency>
 ```
 </CodeTabsBlock>
 <CodeTabsBlock>
 ```Gradle
-implementation 'io.evitadb:evita_java_driver:2024.8.4'
+implementation 'io.evitadb:evita_java_driver:2024.10.0'
 ```
 </CodeTabsBlock>
 </CodeTabs>

evita_api/pom.xml

@@ -31,7 +31,7 @@
 	<parent>
 		<groupId>io.evitadb</groupId>
 		<artifactId>evita_root</artifactId>
-		<version>2024.10-SNAPSHOT</version>
+		<version>2024.11-SNAPSHOT</version>
 	</parent>
 	<dependencies>
 		<dependency>

evita_api/src/main/java/io/evitadb/api/configuration/EvitaConfiguration.java

@@ -42,10 +42,13 @@
  * This class is simple DTO object holding general options of the Evita shared for all catalogs (or better - catalog
  * agnostic).
  *
- * @param name    Name of the evitaDB instance. It's used for identification purposes only.
- * @param server  Contains server wide options.
- * @param storage This field contains all options related to underlying key-value store.
- * @param cache   Cache options contain settings crucial for Evita caching and cache invalidation.
+ * @param name      Name of the evitaDB instance with automatically added hash consisting of host name, path of the data
+ *                  directory and the timestamp of its creation. This hash allows to correctly distinguish instances even
+ *                  if the user leaves configuration defaults and doesn't bother changing the instance name.
+ *                  It's used for identification purposes only.
+ * @param server    Contains server wide options.
+ * @param storage   This field contains all options related to underlying key-value store.
+ * @param cache     Cache options contain settings crucial for Evita caching and cache invalidation.
  * @author Jan Novotný (novotny@fg.cz), FG Forrest a.s. (c) 2022
  */
 public record EvitaConfiguration(
@@ -119,6 +122,15 @@ public EvitaConfiguration() {
 		);
 	}
 
+	/**
+	 * Name of the evitaDB instance as it was provided in the configuration file without the automatically added hash.
+	 * @return Name of the evitaDB instance without appended hash.
+	 */
+	@Nonnull
+	public String plainName() {
+		return name.startsWith(DEFAULT_SERVER_NAME + "-") ? DEFAULT_SERVER_NAME : name;
+	}
+
 	/**
 	 * Standard builder pattern implementation.
 	 */

evita_api/src/main/java/io/evitadb/api/configuration/StorageOptions.java

@@ -87,18 +87,18 @@ public record StorageOptions(
 	long exportFileHistoryExpirationSeconds
 ) {
 
-	public static final int DEFAULT_OUTPUT_BUFFER_SIZE = 2_097_152;
+	public static final int DEFAULT_OUTPUT_BUFFER_SIZE = 2_097_152; // 2MB
 	public static final Path DEFAULT_DATA_DIRECTORY = Paths.get("").resolve("data");
 	public static final Path DEFAULT_EXPORT_DIRECTORY = Paths.get("").resolve("export");
 	public static final int DEFAULT_LOCK_TIMEOUT_SECONDS = 5;
 	public static final int DEFAULT_WAIT_ON_CLOSE_SECONDS = 5;
 	public static final int DEFAULT_MAX_OPENED_READ_HANDLES = Runtime.getRuntime().availableProcessors();
 	public static final boolean DEFAULT_COMPUTE_CRC = true;
 	public static final double DEFAULT_MINIMAL_ACTIVE_RECORD_SHARE = 0.5;
-	public static final long DEFAULT_MINIMAL_FILE_SIZE_COMPACTION_THRESHOLD = 104_857_600L;
+	public static final long DEFAULT_MINIMAL_FILE_SIZE_COMPACTION_THRESHOLD = 104_857_600L; // 100MB
 	public static final boolean DEFAULT_TIME_TRAVEL_ENABLED = true;
-	public static final long DEFAULT_EXPORT_DIRECTORY_SIZE_LIMIT_BYTES = 1_073_741_824L;
-	public static final long DEFAULT_EXPORT_FILE_HISTORY_EXPIRATION_SECONDS = 604_800L;
+	public static final long DEFAULT_EXPORT_DIRECTORY_SIZE_LIMIT_BYTES = 1_073_741_824L; // 1GB
+	public static final long DEFAULT_EXPORT_FILE_HISTORY_EXPIRATION_SECONDS = 604_800L; // 7 days
 
 	/**
 	 * Builder method is planned to be used only in tests.