Merge branch 'main' into darrenge/BDNPerf

.azure/pipelines/azure-pipelines-external-release.yml

@@ -1,9 +1,9 @@
 ######################################
 # NOTE: Before running this pipeline to generate a new nuget package, update the version string in two places
 #      1) update the name: string below (line 6)     -- this is the version for the nuget package (e.g. 1.0.0)
-#      2) update \libs\host\GarnetServer.cs readonly string version  (~line 53)   -- NOTE - these two values need to be the same 
+#      2) update \libs\host\GarnetServer.cs readonly string version  (~line 32)   -- NOTE - these two values need to be the same 
 ###################################### 
-name: 1.0.30
+name: 1.0.33
 trigger:
   branches:
     include:

Directory.Packages.props

@@ -21,10 +21,10 @@
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.0" />
     <PackageVersion Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" Version="8.0.1" />
     <PackageVersion Include="Microsoft.IdentityModel.Validators" Version="8.0.1" />
-    <PackageVersion Include="StackExchange.Redis" Version="2.8.0" />
+    <PackageVersion Include="StackExchange.Redis" Version="2.8.16" />
     <PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.1" />
     <PackageVersion Include="System.Interactive.Async" Version="6.0.1" />
-    <PackageVersion Include="System.Text.Json" Version="8.0.4" />
+    <PackageVersion Include="System.Text.Json" Version="8.0.5" />
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Hosting.WindowsServices" Version="8.0.0" />
   </ItemGroup>

Dockerfile

@@ -8,6 +8,7 @@ COPY libs/cluster/*.csproj libs/cluster/
 COPY libs/common/*.csproj libs/common/
 COPY libs/host/*.csproj libs/host/
 COPY libs/server/*.csproj libs/server/
+COPY libs/resources/*.csproj libs/resources/
 COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/
 COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/
 COPY main/GarnetServer/*.csproj main/GarnetServer/

Dockerfile.alpine

@@ -8,6 +8,7 @@ COPY libs/cluster/*.csproj libs/cluster/
 COPY libs/common/*.csproj libs/common/
 COPY libs/host/*.csproj libs/host/
 COPY libs/server/*.csproj libs/server/
+COPY libs/resources/*.csproj libs/resources/
 COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/
 COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/
 COPY main/GarnetServer/*.csproj main/GarnetServer/

Dockerfile.cbl-mariner

@@ -8,6 +8,7 @@ COPY libs/cluster/*.csproj libs/cluster/
 COPY libs/common/*.csproj libs/common/
 COPY libs/host/*.csproj libs/host/
 COPY libs/server/*.csproj libs/server/
+COPY libs/resources/*.csproj libs/resources/
 COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/
 COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/
 COPY main/GarnetServer/*.csproj main/GarnetServer/

Dockerfile.chiseled

@@ -12,6 +12,7 @@ COPY libs/cluster/*.csproj libs/cluster/
 COPY libs/common/*.csproj libs/common/
 COPY libs/host/*.csproj libs/host/
 COPY libs/server/*.csproj libs/server/
+COPY libs/resources/*.csproj libs/resources/
 COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/
 COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/
 COPY main/GarnetServer/*.csproj main/GarnetServer/

Dockerfile.ubuntu

@@ -8,6 +8,7 @@ COPY libs/cluster/*.csproj libs/cluster/
 COPY libs/common/*.csproj libs/common/
 COPY libs/host/*.csproj libs/host/
 COPY libs/server/*.csproj libs/server/
+COPY libs/resources/*.csproj libs/resources/
 COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/
 COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/
 COPY main/GarnetServer/*.csproj main/GarnetServer/

Garnet.nuspec

@@ -27,7 +27,7 @@
 	    <dependency id="Microsoft.IdentityModel.Validators" version="8.0.1" />
       <dependency id="Azure.Storage.Blobs" version="12.21.2" />
       <dependency id="NLua" version="1.7.3" />
-      <dependency id="System.Text.Json" version="8.0.4" />
+      <dependency id="System.Text.Json" version="8.0.5" />
     </dependencies>
     <readme>README.md</readme>
   </metadata>

benchmark/BDN.benchmark/Resp/RespParseStress.cs

@@ -31,6 +31,14 @@ public unsafe class RespParseStress
         byte[] setexRequestBuffer;
         byte* setexRequestBufferPointer;
 
+        static ReadOnlySpan<byte> SETNX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nNX\r\n"u8;
+        byte[] setnxRequestBuffer;
+        byte* setnxRequestBufferPointer;
+
+        static ReadOnlySpan<byte> SETXX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nXX\r\n"u8;
+        byte[] setxxRequestBuffer;
+        byte* setxxRequestBufferPointer;
+
         static ReadOnlySpan<byte> GET => "*2\r\n$3\r\nGET\r\n$1\r\nb\r\n"u8;
         byte[] getRequestBuffer;
         byte* getRequestBufferPointer;
@@ -91,6 +99,16 @@ public void GlobalSetup()
             for (int i = 0; i < batchSize; i++)
                 SETEX.CopyTo(new Span<byte>(setexRequestBuffer).Slice(i * SETEX.Length));
 
+            setnxRequestBuffer = GC.AllocateArray<byte>(SETNX.Length * batchSize, pinned: true);
+            setnxRequestBufferPointer = (byte*)Unsafe.AsPointer(ref setnxRequestBuffer[0]);
+            for (int i = 0; i < batchSize; i++)
+                SETNX.CopyTo(new Span<byte>(setnxRequestBuffer).Slice(i * SETNX.Length));
+
+            setxxRequestBuffer = GC.AllocateArray<byte>(SETXX.Length * batchSize, pinned: true);
+            setxxRequestBufferPointer = (byte*)Unsafe.AsPointer(ref setxxRequestBuffer[0]);
+            for (int i = 0; i < batchSize; i++)
+                SETXX.CopyTo(new Span<byte>(setxxRequestBuffer).Slice(i * SETXX.Length));
+
             getRequestBuffer = GC.AllocateArray<byte>(GET.Length * batchSize, pinned: true);
             getRequestBufferPointer = (byte*)Unsafe.AsPointer(ref getRequestBuffer[0]);
             for (int i = 0; i < batchSize; i++)
@@ -121,6 +139,9 @@ public void GlobalSetup()
             for (int i = 0; i < batchSize; i++)
                 HSETDEL.CopyTo(new Span<byte>(hSetDelRequestBuffer).Slice(i * HSETDEL.Length));
 
+            // Pre-populate raw string set with a single element
+            SlowConsumeMessage("*3\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n"u8);
+
             // Pre-populate sorted set with a single element to avoid repeatedly emptying it during the benchmark
             SlowConsumeMessage("*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8);
 
@@ -167,6 +188,18 @@ public void SetEx()
             _ = session.TryConsumeMessages(setexRequestBufferPointer, setexRequestBuffer.Length);
         }
 
+        [Benchmark]
+        public void SetNx()
+        {
+            _ = session.TryConsumeMessages(setnxRequestBufferPointer, setnxRequestBuffer.Length);
+        }
+
+        [Benchmark]
+        public void SetXx()
+        {
+            _ = session.TryConsumeMessages(setxxRequestBufferPointer, setxxRequestBuffer.Length);
+        }
+
         [Benchmark]
         public void Get()
         {

libs/cluster/Session/ClusterSlotCheck.cs → libs/cluster/Session/SlotVerification/RespClusterSlotVerify.cs

@@ -65,33 +65,6 @@ private void WriteClusterSlotVerificationMessage(ClusterConfig config, ClusterSl
                 SendAndReset(ref dcurr, ref dend);
         }
 
-        /// <summary>
-        /// Check if read or read/write is permitted on a single key and generate the appropriate response
-        ///         LOCAL   |   ~LOCAL  | MIGRATING EXISTS  |   MIGRATING ~EXISTS   |   IMPORTING ASKING    |   IMPORTING ~ASKING
-        /// R       OK      |   -MOVED  |   OK              |   -ASK                |   OK                  |   -MOVED
-        /// R/W     OK      |   -MOVED  |   -MIGRATING      |   -ASK                |   OK                  |   -MOVED
-        /// </summary>
-        /// <returns>True if redirect, False if can serve</returns>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public bool NetworkSingleKeySlotVerify(ReadOnlySpan<byte> key, bool readOnly, byte SessionAsking, ref byte* dcurr, ref byte* dend)
-        {
-            fixed (byte* keyPtr = key)
-            {
-                var keySlice = new ArgSlice(keyPtr, key.Length);
-                // If cluster is not enabled or a transaction is running skip slot check
-                if (!clusterProvider.serverOptions.EnableCluster || txnManager.state == TxnState.Running) return false;
-
-                var config = clusterProvider.clusterManager.CurrentConfig;
-                var vres = SingleKeySlotVerify(ref config, ref keySlice, readOnly, SessionAsking);
-
-                if (vres.state == SlotVerifiedState.OK)
-                    return false;
-                else
-                    WriteClusterSlotVerificationMessage(config, vres, ref dcurr, ref dend);
-                return true;
-            }
-        }
-
         /// <summary>
         /// Check if read/write is permitted on an array of keys and generate appropriate resp response.
         /// </summary>

libs/common/NumUtils.cs

@@ -13,6 +13,7 @@ namespace Garnet.common
     public static unsafe class NumUtils
     {
         public const int MaximumFormatInt64Length = 20;   // 19 + sign (i.e. -9223372036854775808)
+        public const int MaximumFormatDoubleLength = 310;   // (i.e. -1.7976931348623157E+308)
 
         /// <summary>
         /// Convert long number into sequence of ASCII bytes
@@ -75,6 +76,81 @@ public static unsafe void LongToBytes(long value, int length, ref byte* result)
             result += length;
         }
 
+        /// <summary>
+        /// Convert double number into sequence of ASCII bytes
+        /// </summary>
+        /// <param name="value">Value to convert</param>
+        /// <param name="dest">Span Byte</param>
+        /// <returns>Length of number in result</returns>
+        public static int DoubleToSpanByte(double value, Span<byte> dest)
+        {
+            int totalLen = NumOfCharInDouble(value, out var integerDigits, out var signSize, out var fractionalDigits);
+            bool isNegative = value < 0;
+            if (totalLen > dest.Length)
+                return 0;
+            fixed (byte* ptr = dest)
+            {
+                byte* curr = ptr;
+                DoubleToBytes(value, integerDigits, fractionalDigits, ref curr);
+            }
+
+            return totalLen;
+        }
+
+        /// <summary>
+        /// Convert double number into sequence of ASCII bytes
+        /// </summary>
+        /// <param name="value">Value to convert</param>
+        /// <param name="integerDigits">Number of digits in the integer part of the double value</param>
+        /// <param name="fractionalDigits">Number of digits in the fractional part of the double value</param>
+        /// <param name="result">Byte pointer, will be updated to point after the written number</param>
+        public static unsafe void DoubleToBytes(double value, int integerDigits, int fractionalDigits, ref byte* result)
+        {
+            Debug.Assert(!double.IsNaN(value) && !double.IsInfinity(value), "Cannot convert NaN or Infinity to bytes.");
+
+            if (value == 0)
+            {
+                *result++ = (byte)'0';
+                return;
+            }
+
+            bool isNegative = value < 0;
+            if (isNegative)
+            {
+                *result++ = (byte)'-';
+                value = -value;
+            }
+
+            result += integerDigits;
+            var integerPart = Math.Truncate(value);
+            double fractionalPart = fractionalDigits > 0 ? Math.Round(value - integerPart, fractionalDigits) : 0;
+
+            // Convert integer part
+            do
+            {
+                *--result = (byte)((byte)'0' + (integerPart % 10));
+                integerPart /= 10;
+            } while (integerPart >= 1);
+            result += integerDigits;
+
+            if (fractionalDigits > 0)
+            {
+                // Add decimal point
+                *result++ = (byte)'.';
+
+                // Convert fractional part
+                for (int i = 0; i < fractionalDigits; i++)
+                {
+                    fractionalPart *= 10;
+                    int digit = (int)fractionalPart;
+                    *result++ = (byte)((byte)'0' + digit);
+                    fractionalPart = Math.Round(fractionalPart - digit, fractionalDigits - i - 1);
+                }
+
+                result--; // Move back to the last digit
+            }
+        }
+
         /// <summary>
         /// Convert sequence of ASCII bytes into long number
         /// </summary>
@@ -142,6 +218,45 @@ public static bool TryBytesToLong(int length, byte* source, out long result)
             return true;
         }
 
+        /// <summary>
+        /// Convert sequence of ASCII bytes into double number
+        /// </summary>
+        /// <param name="source">Source bytes</param>
+        /// <param name="result">Double value extracted from sequence</param>
+        /// <returns>True if sequence contains only numeric digits, otherwise false</returns>
+        public static bool TryBytesToDouble(ReadOnlySpan<byte> source, out double result)
+        {
+            fixed (byte* ptr = source)
+                return TryBytesToDouble(source.Length, ptr, out result);
+        }
+
+        /// <summary>
+        /// Convert sequence of ASCII bytes into double number
+        /// </summary>
+        /// <param name="length">Length of number</param>
+        /// <param name="source">Source bytes</param>
+        /// <param name="result">Double value extracted from sequence</param>
+        /// <returns>True if sequence contains only numeric digits, otherwise false</returns>
+        public static bool TryBytesToDouble(int length, byte* source, out double result)
+        {
+            var fNeg = *source == '-';
+            var beg = fNeg ? source + 1 : source;
+            var len = fNeg ? length - 1 : length;
+            result = 0;
+
+            // Do not allow leading zeros
+            if (len > 1 && *beg == '0' && *(beg + 1) != '.')
+                return false;
+
+            // Parse number and check consumed bytes to avoid alphanumeric strings
+            if (!TryParse(new ReadOnlySpan<byte>(beg, len), out result))
+                return false;
+
+            // Negate if parsed value has a leading negative sign
+            result = fNeg ? -result : result;
+            return true;
+        }
+
         /// <summary>
         /// Convert sequence of ASCII bytes into ulong number
         /// </summary>
@@ -370,6 +485,38 @@ public static int NumDigitsInLong(long v, ref bool fNeg)
             return 19;
         }
 
+        /// <summary>
+        /// Return number of digits in given double number incluing the decimal part and `.` character
+        /// </summary>
+        /// <param name="v">Double value</param>
+        /// <returns>Number of digits in the integer part of the double value</returns>
+        public static int NumOfCharInDouble(double v, out int integerDigits, out byte signSize, out int fractionalDigits)
+        {
+            if (v == 0)
+            {
+                integerDigits = 1;
+                signSize = 0;
+                fractionalDigits = 0;
+                return 1;
+            }
+
+            Debug.Assert(!double.IsNaN(v) && !double.IsInfinity(v));
+
+            signSize = (byte)(v < 0 ? 1 : 0); // Add sign if the number is negative
+            v = Math.Abs(v);
+            integerDigits = (int)Math.Log10(v) + 1;
+
+            fractionalDigits = 0; // Max of 15 significant digits
+            while (fractionalDigits <= 14 && Math.Abs(v - Math.Round(v, fractionalDigits)) > 2 * Double.Epsilon) // 2 * Double.Epsilon is used to handle floating point errors
+            {
+                fractionalDigits++;
+            }
+
+            var dotSize = fractionalDigits != 0 ? 1 : 0; // Add decimal point if there are significant digits
+
+            return signSize + integerDigits + dotSize + fractionalDigits;
+        }
+
         /// <inheritdoc cref="Utf8Parser.TryParse(ReadOnlySpan{byte}, out int, out int, char)"/>
         public static bool TryParse(ReadOnlySpan<byte> source, out int value)
         {

libs/host/GarnetServer.cs

@@ -28,6 +28,9 @@ namespace Garnet
     /// </summary>
     public class GarnetServer : IDisposable
     {
+        // IMPORTANT: Keep the version in sync with .azure\pipelines\azure-pipelines-external-release.yml line ~6.
+        readonly string version = "1.0.33";
+
         internal GarnetProvider Provider;
 
         private readonly GarnetServerOptions opts;
@@ -51,9 +54,6 @@ public class GarnetServer : IDisposable
         /// </summary>
         protected StoreWrapper storeWrapper;
 
-        // IMPORTANT: Keep the version in sync with .azure\pipelines\azure-pipelines-external-release.yml line ~6.
-        readonly string version = "1.0.30";
-
         /// <summary>
         /// Resp protocol version
         /// </summary>

libs/resources/Garnet.resources.csproj

@@ -1,16 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
+    <SignAssembly>true</SignAssembly>
+    <AssemblyOriginatorKeyFile>../../Garnet.snk</AssemblyOriginatorKeyFile>
+    <DelaySign>false</DelaySign>
+    <EmbedUntrackedSources>true</EmbedUntrackedSources>
   </PropertyGroup>
 
-  <ItemGroup>
-    <None Remove="RespCommandsDocs.json" />
-    <None Remove="RespCommandsInfo.json" />
-  </ItemGroup>
-
   <ItemGroup>
     <EmbeddedResource Include="RespCommandsDocs.json" />
     <EmbeddedResource Include="RespCommandsInfo.json" />

libs/resources/ResourceUtils.cs

@@ -0,0 +1,9 @@
+namespace Garnet.resources
+{
+    /// <summary>
+    /// Dummy class for externally referencing this assembly
+    /// </summary>
+    public class ResourceUtils
+    {
+    }
+}