From 648e9bb91c74e8b4e138ed1ce81f368a6c146070 Mon Sep 17 00:00:00 2001
From: Lukas Sismis <lsismis@oisf.net>
Date: Thu, 25 Jul 2024 10:34:21 +0200
Subject: [PATCH 1/8] hashlittle: add a safe variant of hashlittle2 function

This variant of hashlittle2() ensures that it avoids
accesses beyond the last byte of the string, which will
cause warnings from tools like Valgrind or Address
Sanitizer.
---
 src/util-hash-lookup3.c | 204 ++++++++++++++++++++++++++++++++++++++++
 src/util-hash-lookup3.h |   5 +
 2 files changed, 209 insertions(+)

diff --git a/src/util-hash-lookup3.c b/src/util-hash-lookup3.c
index 2354c183d219..c2b0fe275828 100644
--- a/src/util-hash-lookup3.c
+++ b/src/util-hash-lookup3.c
@@ -805,7 +805,211 @@ void hashlittle2(
   *pc=c; *pb=b;
 }
 
+/*
+ * hashlittle2: return 2 32-bit hash values
+ *
+ * This is identical to hashlittle(), except it returns two 32-bit hash
+ * values instead of just one.  This is good enough for hash table
+ * lookup with 2^^64 buckets, or if you want a second hash if you're not
+ * happy with the first, or if you want a probably-unique 64-bit ID for
+ * the key.  *pc is better mixed than *pb, so use *pc first.  If you want
+ * a 64-bit value do something like "*pc + (((uint64_t)*pb)<<32)".
+ */
+void hashlittle2_safe(const void *key, /* the key to hash */
+        size_t length,                 /* length of the key */
+        uint32_t *pc,                  /* IN: primary initval, OUT: primary hash */
+        uint32_t *pb)                  /* IN: secondary initval, OUT: secondary hash */
+{
+    uint32_t a, b, c; /* internal state */
+    union {
+        const void *ptr;
+        size_t i;
+    } u; /* needed for Mac Powerbook G4 */
+
+    /* Set up the internal state */
+    a = b = c = 0xdeadbeef + ((uint32_t)length) + *pc;
+    c += *pb;
+
+    u.ptr = key;
+    if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) {
+        const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */
+
+        /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
+        while (length > 12) {
+            a += k[0];
+            b += k[1];
+            c += k[2];
+            mix(a, b, c);
+            length -= 12;
+            k += 3;
+        }
+
+        /*----------------------------- handle the last (probably partial) block */
+        /*
+         * Note that unlike hashlittle() above, we use the "safe" version of this
+         * block that is #ifdef VALGRIND above, in order to avoid warnings from
+         * Valgrind or Address Sanitizer.
+         */
+        const uint8_t *k8 = (const uint8_t *)k;
+        switch (length) {
+            case 12:
+                c += k[2];
+                b += k[1];
+                a += k[0];
+                break;
+            case 11:
+                c += ((uint32_t)k8[10]) << 16; /* fall through */
+            case 10:
+                c += ((uint32_t)k8[9]) << 8; /* fall through */
+            case 9:
+                c += k8[8]; /* fall through */
+            case 8:
+                b += k[1];
+                a += k[0];
+                break;
+            case 7:
+                b += ((uint32_t)k8[6]) << 16; /* fall through */
+            case 6:
+                b += ((uint32_t)k8[5]) << 8; /* fall through */
+            case 5:
+                b += k8[4]; /* fall through */
+            case 4:
+                a += k[0];
+                break;
+            case 3:
+                a += ((uint32_t)k8[2]) << 16; /* fall through */
+            case 2:
+                a += ((uint32_t)k8[1]) << 8; /* fall through */
+            case 1:
+                a += k8[0];
+                break;
+            case 0:
+                *pc = c;
+                *pb = b;
+                return; /* zero length strings require no mixing */
+        }
+
+    } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) {
+        const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */
+        const uint8_t *k8;
+
+        /*--------------- all but last block: aligned reads and different mixing */
+        while (length > 12) {
+            a += k[0] + (((uint32_t)k[1]) << 16);
+            b += k[2] + (((uint32_t)k[3]) << 16);
+            c += k[4] + (((uint32_t)k[5]) << 16);
+            mix(a, b, c);
+            length -= 12;
+            k += 6;
+        }
+
+        /*----------------------------- handle the last (probably partial) block */
+        k8 = (const uint8_t *)k;
+        switch (length) {
+            case 12:
+                c += k[4] + (((uint32_t)k[5]) << 16);
+                b += k[2] + (((uint32_t)k[3]) << 16);
+                a += k[0] + (((uint32_t)k[1]) << 16);
+                break;
+            case 11:
+                c += ((uint32_t)k8[10]) << 16; /* fall through */
+            case 10:
+                c += k[4];
+                b += k[2] + (((uint32_t)k[3]) << 16);
+                a += k[0] + (((uint32_t)k[1]) << 16);
+                break;
+            case 9:
+                c += k8[8]; /* fall through */
+            case 8:
+                b += k[2] + (((uint32_t)k[3]) << 16);
+                a += k[0] + (((uint32_t)k[1]) << 16);
+                break;
+            case 7:
+                b += ((uint32_t)k8[6]) << 16; /* fall through */
+            case 6:
+                b += k[2];
+                a += k[0] + (((uint32_t)k[1]) << 16);
+                break;
+            case 5:
+                b += k8[4]; /* fall through */
+            case 4:
+                a += k[0] + (((uint32_t)k[1]) << 16);
+                break;
+            case 3:
+                a += ((uint32_t)k8[2]) << 16; /* fall through */
+            case 2:
+                a += k[0];
+                break;
+            case 1:
+                a += k8[0];
+                break;
+            case 0:
+                *pc = c;
+                *pb = b;
+                return; /* zero length strings require no mixing */
+        }
+
+    } else { /* need to read the key one byte at a time */
+        const uint8_t *k = (const uint8_t *)key;
+
+        /*--------------- all but the last block: affect some 32 bits of (a,b,c) */
+        while (length > 12) {
+            a += k[0];
+            a += ((uint32_t)k[1]) << 8;
+            a += ((uint32_t)k[2]) << 16;
+            a += ((uint32_t)k[3]) << 24;
+            b += k[4];
+            b += ((uint32_t)k[5]) << 8;
+            b += ((uint32_t)k[6]) << 16;
+            b += ((uint32_t)k[7]) << 24;
+            c += k[8];
+            c += ((uint32_t)k[9]) << 8;
+            c += ((uint32_t)k[10]) << 16;
+            c += ((uint32_t)k[11]) << 24;
+            mix(a, b, c);
+            length -= 12;
+            k += 12;
+        }
+
+        /*-------------------------------- last block: affect all 32 bits of (c) */
+        switch (length) /* all the case statements fall through */
+        {
+            case 12:
+                c += ((uint32_t)k[11]) << 24; /* fall through */
+            case 11:
+                c += ((uint32_t)k[10]) << 16; /* fall through */
+            case 10:
+                c += ((uint32_t)k[9]) << 8; /* fall through */
+            case 9:
+                c += k[8]; /* fall through */
+            case 8:
+                b += ((uint32_t)k[7]) << 24; /* fall through */
+            case 7:
+                b += ((uint32_t)k[6]) << 16; /* fall through */
+            case 6:
+                b += ((uint32_t)k[5]) << 8; /* fall through */
+            case 5:
+                b += k[4]; /* fall through */
+            case 4:
+                a += ((uint32_t)k[3]) << 24; /* fall through */
+            case 3:
+                a += ((uint32_t)k[2]) << 16; /* fall through */
+            case 2:
+                a += ((uint32_t)k[1]) << 8; /* fall through */
+            case 1:
+                a += k[0];
+                break;
+            case 0:
+                *pc = c;
+                *pb = b;
+                return; /* zero length strings require no mixing */
+        }
+    }
 
+    final(a, b, c);
+    *pc = c;
+    *pb = b;
+}
 
 /*
  * hashbig():
diff --git a/src/util-hash-lookup3.h b/src/util-hash-lookup3.h
index 62d3d14270b1..ab7f6d3d88f3 100644
--- a/src/util-hash-lookup3.h
+++ b/src/util-hash-lookup3.h
@@ -62,6 +62,11 @@ void hashlittle2(const void *key,       /* the key to hash */
                  uint32_t   *pc,        /* IN: primary initval, OUT: primary hash */
                  uint32_t   *pb);       /* IN: secondary initval, OUT: secondary hash */
 
+/* A variant of hashlittle2() that ensures avoids accesses beyond the last byte
+ * of the string, which will cause warnings from tools like Valgrind or Address
+ * Sanitizer. */
+void hashlittle2_safe(const void *key, size_t length, uint32_t *pc, uint32_t *pb);
+
 uint32_t hashbig( const void *key, size_t length, uint32_t initval);
 
 #endif /* SURICATA_UTIL_HASH_LOOKUP3_H */

From 4b321e94b51708da93eb1bb65bcdeab754efde7e Mon Sep 17 00:00:00 2001
From: Lukas Sismis <lsismis@oisf.net>
Date: Sat, 5 Oct 2024 20:27:19 +0200
Subject: [PATCH 2/8] util-hash: add iterator for hash function

---
 src/util-hash.c | 15 +++++++++++++++
 src/util-hash.h |  1 +
 2 files changed, 16 insertions(+)

diff --git a/src/util-hash.c b/src/util-hash.c
index ea04f2d7bb94..bdfc1c6f74ff 100644
--- a/src/util-hash.c
+++ b/src/util-hash.c
@@ -208,6 +208,21 @@ void *HashTableLookup(HashTable *ht, void *data, uint16_t datalen)
     return NULL;
 }
 
+// CallbackFn is an iterator, first argument is the data, second is user auxilary data
+void HashTableIterate(HashTable *ht, void (*CallbackFn)(void *, void *), void *aux)
+{
+    if (ht == NULL || CallbackFn == NULL)
+        return;
+
+    for (uint32_t i = 0; i < ht->array_size; i++) {
+        HashTableBucket *hashbucket = ht->array[i];
+        while (hashbucket != NULL) {
+            CallbackFn(hashbucket->data, aux);
+            hashbucket = hashbucket->next;
+        }
+    }
+}
+
 uint32_t HashTableGenericHash(HashTable *ht, void *data, uint16_t datalen)
 {
      uint8_t *d = (uint8_t *)data;
diff --git a/src/util-hash.h b/src/util-hash.h
index ec96f5bcac71..40241b3db71a 100644
--- a/src/util-hash.h
+++ b/src/util-hash.h
@@ -51,6 +51,7 @@ void HashTableFree(HashTable *);
 int HashTableAdd(HashTable *, void *, uint16_t);
 int HashTableRemove(HashTable *, void *, uint16_t);
 void *HashTableLookup(HashTable *, void *, uint16_t);
+void HashTableIterate(HashTable *ht, void (*CallbackFn)(void *, void *), void *aux);
 uint32_t HashTableGenericHash(HashTable *, void *, uint16_t);
 char HashTableDefaultCompare(void *, uint16_t, void *, uint16_t);
 

From 6073d14075de0f2a6aad4647f8010a6312eb4ed1 Mon Sep 17 00:00:00 2001
From: Lukas Sismis <lsismis@oisf.net>
Date: Wed, 24 Jul 2024 22:45:20 +0200
Subject: [PATCH 3/8] util-path: remove dead code

---
 src/util-path.c | 36 ------------------------------------
 src/util-path.h |  1 -
 2 files changed, 37 deletions(-)

diff --git a/src/util-path.c b/src/util-path.c
index 34c4cc75ca79..356c4a7727b9 100644
--- a/src/util-path.c
+++ b/src/util-path.c
@@ -117,42 +117,6 @@ char *PathMergeAlloc(const char *const dir, const char *const fname)
     return ret;
 }
 
-/**
- * \brief Wrapper to join a directory and filename and resolve using realpath
- *   _fullpath is used for WIN32
- *
- * \param out_buf output buffer.  Up to PATH_MAX will be written.  Unchanged on exit failure.
- * \param buf_size length of output buffer, must be PATH_MAX
- * \param dir the directory
- * \param fname the filename
- *
- * \retval 0 on success
- * \retval -1 on failure
- */
-int PathJoin(char *out_buf, size_t buf_size, const char *const dir, const char *const fname)
-{
-    SCEnter();
-    if (buf_size != PATH_MAX) {
-        return -1;
-    }
-    if (PathMerge(out_buf, buf_size, dir, fname) != 0) {
-        SCLogError("Could not join filename to path");
-        return -1;
-    }
-    char *tmp_buf = SCRealPath(out_buf, NULL);
-    if (tmp_buf == NULL) {
-        SCLogError("Error resolving path: %s", strerror(errno));
-        return -1;
-    }
-    memset(out_buf, 0, buf_size);
-    size_t ret = strlcpy(out_buf, tmp_buf, buf_size);
-    free(tmp_buf);
-    if (ret >= buf_size) {
-        return -1;
-    }
-    return 0;
-}
-
 /**
  * \brief Wrapper around SCMkDir with default mode arguments.
  */
diff --git a/src/util-path.h b/src/util-path.h
index fee58eabd3c4..b2b2624908dd 100644
--- a/src/util-path.h
+++ b/src/util-path.h
@@ -51,7 +51,6 @@ int PathIsAbsolute(const char *);
 int PathIsRelative(const char *);
 int PathMerge(char *out_buf, size_t buf_size, const char *const dir, const char *const fname);
 char *PathMergeAlloc(const char *const dir, const char *const fname);
-int PathJoin(char *out_buf, size_t buf_len, const char *const dir, const char *const fname);
 int SCDefaultMkDir(const char *path);
 int SCCreateDirectoryTree(const char *path, const bool final);
 bool SCPathExists(const char *path);

From ce6123c3f305062996e390a15bb2e18dfdea6c8c Mon Sep 17 00:00:00 2001
From: Lukas Sismis <lsismis@oisf.net>
Date: Wed, 24 Jul 2024 22:46:15 +0200
Subject: [PATCH 4/8] detect-engine: remove commented out code

---
 src/detect-engine.c | 9 ---------
 src/detect-engine.h | 1 -
 2 files changed, 10 deletions(-)

diff --git a/src/detect-engine.c b/src/detect-engine.c
index c34c305d4a15..887bf46ce5c8 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -3001,15 +3001,6 @@ static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
     return 0;
 }
 
-/*
- * getting & (re)setting the internal sig i
- */
-
-//inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *de_ctx)
-//{
-//    return de_ctx->signum;
-//}
-
 void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
 {
     de_ctx->signum = 0;
diff --git a/src/detect-engine.h b/src/detect-engine.h
index 3552f161399e..6240a4492e77 100644
--- a/src/detect-engine.h
+++ b/src/detect-engine.h
@@ -99,7 +99,6 @@ void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, i
 
 TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **);
 TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
-//inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *);
 /* faster as a macro than a inline function on my box -- VJ */
 #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum)
 void DetectEngineResetMaxSigId(DetectEngineCtx *);

From 15b4559505c76af103a38d2b5e04073db3f78acb Mon Sep 17 00:00:00 2001
From: Lukas Sismis <lsismis@oisf.net>
Date: Mon, 28 Oct 2024 15:41:07 +0100
Subject: [PATCH 5/8] util-mpm-hs: refactor Hyperscan Pattern DB initialization
 to smaller functions

---
 src/Makefile.am        |   2 +
 src/util-mpm-hs-core.c |  83 +++++++++++++
 src/util-mpm-hs-core.h |  93 ++++++++++++++
 src/util-mpm-hs.c      | 268 +++++++++++++++++++++++++----------------
 src/util-mpm-hs.h      |  42 -------
 5 files changed, 343 insertions(+), 145 deletions(-)
 create mode 100644 src/util-mpm-hs-core.c
 create mode 100644 src/util-mpm-hs-core.h

diff --git a/src/Makefile.am b/src/Makefile.am
index 6970d709f35c..680206d2a84f 100755
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -536,6 +536,7 @@ noinst_HEADERS = \
 	util-mpm-ac-ks.h \
 	util-mpm.h \
 	util-mpm-hs.h \
+	util-mpm-hs-core.h \
 	util-optimize.h \
 	util-pages.h \
 	util-path.h \
@@ -1088,6 +1089,7 @@ libsuricata_c_a_SOURCES = \
 	util-mpm-ac-ks-small.c \
 	util-mpm.c \
 	util-mpm-hs.c \
+	util-mpm-hs-core.c \
 	util-pages.c \
 	util-path.c \
 	util-pidfile.c \
diff --git a/src/util-mpm-hs-core.c b/src/util-mpm-hs-core.c
new file mode 100644
index 000000000000..b8663ae220d2
--- /dev/null
+++ b/src/util-mpm-hs-core.c
@@ -0,0 +1,83 @@
+/* Copyright (C) 2007-2024 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jim Xu <jim.xu@windriver.com>
+ * \author Justin Viiret <justin.viiret@intel.com>
+ * \author Lukas Sismis <lsismis@oisf.net>
+ *
+ * MPM pattern matcher core function for the Hyperscan regex matcher.
+ */
+
+#include "suricata-common.h"
+#include "suricata.h"
+#include "util-mpm-hs-core.h"
+
+#ifdef BUILD_HYPERSCAN
+
+#include <hs.h>
+
+/**
+ * Translates Hyperscan error codes to human-readable messages.
+ *
+ * \param error_code
+ *      The error code returned by a Hyperscan function.
+ * \return
+ *      A string describing the error.
+ */
+const char *HSErrorToStr(hs_error_t error_code)
+{
+    switch (error_code) {
+        case HS_SUCCESS:
+            return "HS_SUCCESS: The engine completed normally";
+        case HS_INVALID:
+            return "HS_INVALID: A parameter passed to this function was invalid";
+        case HS_NOMEM:
+            return "HS_NOMEM: A memory allocation failed";
+        case HS_SCAN_TERMINATED:
+            return "HS_SCAN_TERMINATED: The engine was terminated by callback";
+        case HS_COMPILER_ERROR:
+            return "HS_COMPILER_ERROR: The pattern compiler failed";
+        case HS_DB_VERSION_ERROR:
+            return "HS_DB_VERSION_ERROR: The given database was built for a different version of "
+                   "Hyperscan";
+        case HS_DB_PLATFORM_ERROR:
+            return "HS_DB_PLATFORM_ERROR: The given database was built for a different platform "
+                   "(i.e., CPU type)";
+        case HS_DB_MODE_ERROR:
+            return "HS_DB_MODE_ERROR: The given database was built for a different mode of "
+                   "operation";
+        case HS_BAD_ALIGN:
+            return "HS_BAD_ALIGN: A parameter passed to this function was not correctly aligned";
+        case HS_BAD_ALLOC:
+            return "HS_BAD_ALLOC: The memory allocator did not return correctly aligned memory";
+        case HS_SCRATCH_IN_USE:
+            return "HS_SCRATCH_IN_USE: The scratch region was already in use";
+        case HS_ARCH_ERROR:
+            return "HS_ARCH_ERROR: Unsupported CPU architecture";
+        case HS_INSUFFICIENT_SPACE:
+            return "HS_INSUFFICIENT_SPACE: Provided buffer was too small";
+        case HS_UNKNOWN_ERROR:
+            return "HS_UNKNOWN_ERROR: Unexpected internal error";
+        default:
+            return "Unknown error code";
+    }
+}
+
+#endif /* BUILD_HYPERSCAN */
diff --git a/src/util-mpm-hs-core.h b/src/util-mpm-hs-core.h
new file mode 100644
index 000000000000..fc7c2d302848
--- /dev/null
+++ b/src/util-mpm-hs-core.h
@@ -0,0 +1,93 @@
+/* Copyright (C) 2007-2024 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jim Xu <jim.xu@windriver.com>
+ * \author Justin Viiret <justin.viiret@intel.com>
+ * \author Lukas Sismis <lsismis@oisf.net>
+ *
+ * MPM pattern matcher core function for the Hyperscan regex matcher.
+ */
+
+#ifndef SURICATA_UTIL_MPM_HS_CORE__H
+#define SURICATA_UTIL_MPM_HS_CORE__H
+
+#include "suricata-common.h"
+#include "suricata.h"
+
+#ifdef BUILD_HYPERSCAN
+#include <hs.h>
+
+typedef struct SCHSPattern_ {
+    /* length of the pattern */
+    uint16_t len;
+    /* flags describing the pattern */
+    uint8_t flags;
+    /* holds the original pattern that was added */
+    uint8_t *original_pat;
+    /* pattern id */
+    uint32_t id;
+
+    uint16_t offset;
+    uint16_t depth;
+
+    /* sid(s) for this pattern */
+    uint32_t sids_size;
+    SigIntId *sids;
+
+    /* only used at ctx init time, when this structure is part of a hash
+     * table. */
+    struct SCHSPattern_ *next;
+} SCHSPattern;
+
+typedef struct SCHSCtx_ {
+    /* hash used during ctx initialization */
+    SCHSPattern **init_hash;
+
+    /* pattern database and pattern arrays. */
+    void *pattern_db;
+
+    /* size of database, for accounting. */
+    size_t hs_db_size;
+} SCHSCtx;
+
+typedef struct SCHSThreadCtx_ {
+    /* Hyperscan scratch space region for this thread, capable of handling any
+     * database that has been compiled. */
+    void *scratch;
+
+    /* size of scratch space, for accounting. */
+    size_t scratch_size;
+} SCHSThreadCtx;
+
+typedef struct PatternDatabase_ {
+    SCHSPattern **parray;
+    hs_database_t *hs_db;
+    uint32_t pattern_cnt;
+
+    /* Reference count: number of MPM contexts using this pattern database. */
+    uint32_t ref_cnt;
+    /* Signals if the matcher has loaded/saved the pattern database to disk */
+    bool cached;
+} PatternDatabase;
+
+const char *HSErrorToStr(hs_error_t error_code);
+
+#endif /* BUILD_HYPERSCAN */
+#endif /* SURICATA_UTIL_MPM_HS_CORE__H */
diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c
index 82b91b4ec0e8..da02e2194dfe 100644
--- a/src/util-mpm-hs.c
+++ b/src/util-mpm-hs.c
@@ -38,6 +38,7 @@
 #include "util-unittest-helper.h"
 #include "util-memcmp.h"
 #include "util-mpm-hs.h"
+#include "util-mpm-hs-core.h"
 #include "util-memcpy.h"
 #include "util-hash.h"
 #include "util-hash-lookup3.h"
@@ -379,7 +380,7 @@ typedef struct SCHSCompileData_ {
     unsigned int pattern_cnt;
 } SCHSCompileData;
 
-static SCHSCompileData *SCHSAllocCompileData(unsigned int pattern_cnt)
+static SCHSCompileData *CompileDataAlloc(unsigned int pattern_cnt)
 {
     SCHSCompileData *cd = SCCalloc(pattern_cnt, sizeof(SCHSCompileData));
     if (cd == NULL) {
@@ -422,7 +423,7 @@ static SCHSCompileData *SCHSAllocCompileData(unsigned int pattern_cnt)
     return NULL;
 }
 
-static void SCHSFreeCompileData(SCHSCompileData *cd)
+static void CompileDataFree(SCHSCompileData *cd)
 {
     if (cd == NULL) {
         return;
@@ -445,15 +446,6 @@ static void SCHSFreeCompileData(SCHSCompileData *cd)
     SCFree(cd);
 }
 
-typedef struct PatternDatabase_ {
-    SCHSPattern **parray;
-    hs_database_t *hs_db;
-    uint32_t pattern_cnt;
-
-    /* Reference count: number of MPM contexts using this pattern database. */
-    uint32_t ref_cnt;
-} PatternDatabase;
-
 static uint32_t SCHSPatternHash(const SCHSPattern *p, uint32_t hash)
 {
     BUG_ON(p->original_pat == NULL);
@@ -570,38 +562,20 @@ static PatternDatabase *PatternDatabaseAlloc(uint32_t pattern_cnt)
     return pd;
 }
 
-/**
- * \brief Process the patterns added to the mpm, and create the internal tables.
- *
- * \param mpm_ctx Pointer to the mpm context.
- */
-int SCHSPreparePatterns(MpmCtx *mpm_ctx)
+static int HSCheckPatterns(MpmCtx *mpm_ctx, SCHSCtx *ctx)
 {
-    SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx;
-
     if (mpm_ctx->pattern_cnt == 0 || ctx->init_hash == NULL) {
         SCLogDebug("no patterns supplied to this mpm_ctx");
         return 0;
     }
+    return 1;
+}
 
-    hs_error_t err;
-    hs_compile_error_t *compile_err = NULL;
-    SCHSCompileData *cd = NULL;
-    PatternDatabase *pd = NULL;
-
-    cd = SCHSAllocCompileData(mpm_ctx->pattern_cnt);
-    if (cd == NULL) {
-        goto error;
-    }
-
-    pd = PatternDatabaseAlloc(mpm_ctx->pattern_cnt);
-    if (pd == NULL) {
-        goto error;
-    }
-
-    /* populate the pattern array with the patterns in the hash */
+static void HSPatternArrayPopulate(SCHSCtx *ctx, PatternDatabase *pd)
+{
     for (uint32_t i = 0, p = 0; i < INIT_HASH_SIZE; i++) {
-        SCHSPattern *node = ctx->init_hash[i], *nnode = NULL;
+        SCHSPattern *node = ctx->init_hash[i];
+        SCHSPattern *nnode = NULL;
         while (node != NULL) {
             nnode = node->next;
             node->next = NULL;
@@ -609,105 +583,207 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx)
             node = nnode;
         }
     }
+}
 
+static void HSPatternArrayInit(SCHSCtx *ctx, PatternDatabase *pd)
+{
+    HSPatternArrayPopulate(ctx, pd);
     /* we no longer need the hash, so free its memory */
     SCFree(ctx->init_hash);
     ctx->init_hash = NULL;
+}
 
-    /* Serialise whole database compilation as a relatively easy way to ensure
-     * dedupe is safe. */
-    SCMutexLock(&g_db_table_mutex);
-
-    /* Init global pattern database hash if necessary. */
+static int HSGlobalPatternDatabaseInit(void)
+{
     if (g_db_table == NULL) {
         g_db_table = HashTableInit(INIT_DB_HASH_SIZE, PatternDatabaseHash,
                                    PatternDatabaseCompare,
                                    PatternDatabaseTableFree);
         if (g_db_table == NULL) {
-            SCMutexUnlock(&g_db_table_mutex);
-            goto error;
+            return -1;
         }
     }
+    return 0;
+}
 
+static void HSLogCompileError(hs_compile_error_t *compile_err)
+{
+    SCLogError("failed to compile hyperscan database");
+    if (compile_err) {
+        SCLogError("compile error: %s", compile_err->message);
+        hs_free_compile_error(compile_err);
+    }
+}
+
+static int HSScratchAlloc(const hs_database_t *db)
+{
+    SCMutexLock(&g_scratch_proto_mutex);
+    hs_error_t err = hs_alloc_scratch(db, &g_scratch_proto);
+    SCMutexUnlock(&g_scratch_proto_mutex);
+    if (err != HS_SUCCESS) {
+        SCLogError("failed to allocate scratch");
+        return -1;
+    }
+    return 0;
+}
+
+static int PatternDatabaseGetSize(PatternDatabase *pd, size_t *db_size)
+{
+    hs_error_t err = hs_database_size(pd->hs_db, db_size);
+    if (err != HS_SUCCESS) {
+        SCLogError("failed to query database size: %s", HSErrorToStr(err));
+        return -1;
+    }
+    return 0;
+}
+
+static void SCHSCleanupOnError(PatternDatabase *pd, SCHSCompileData *cd)
+{
+    if (pd) {
+        PatternDatabaseFree(pd);
+    }
+    if (cd) {
+        CompileDataFree(cd);
+    }
+}
+
+static int CompileDataExtensionsInit(hs_expr_ext_t **ext, const SCHSPattern *p)
+{
+    if (p->flags & (MPM_PATTERN_FLAG_OFFSET | MPM_PATTERN_FLAG_DEPTH)) {
+        *ext = SCCalloc(1, sizeof(hs_expr_ext_t));
+        if ((*ext) == NULL) {
+            return -1;
+        }
+        if (p->flags & MPM_PATTERN_FLAG_OFFSET) {
+            (*ext)->flags |= HS_EXT_FLAG_MIN_OFFSET;
+            (*ext)->min_offset = p->offset + p->len;
+        }
+        if (p->flags & MPM_PATTERN_FLAG_DEPTH) {
+            (*ext)->flags |= HS_EXT_FLAG_MAX_OFFSET;
+            (*ext)->max_offset = p->offset + p->depth;
+        }
+    }
+
+    return 0;
+}
+
+/**
+ * \brief Initialize the pattern database - try to get existing pd
+ * from the global hash table, or load it from disk if caching is enabled.
+ *
+ * \param PatternDatabase* [in/out] Pointer to the pattern database to use.
+ * \param SCHSCompileData* [in] Pointer to the compile data.
+ * \retval 0 On success, negative value on failure.
+ */
+static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd)
+{
     /* Check global hash table to see if we've seen this pattern database
      * before, and reuse the Hyperscan database if so. */
-    PatternDatabase *pd_cached = HashTableLookup(g_db_table, pd, 1);
-
+    PatternDatabase *pd_cached = HashTableLookup(g_db_table, *pd, 1);
     if (pd_cached != NULL) {
         SCLogDebug("Reusing cached database %p with %" PRIu32
                    " patterns (ref_cnt=%" PRIu32 ")",
                    pd_cached->hs_db, pd_cached->pattern_cnt,
                    pd_cached->ref_cnt);
         pd_cached->ref_cnt++;
-        ctx->pattern_db = pd_cached;
-        SCMutexUnlock(&g_db_table_mutex);
-        PatternDatabaseFree(pd);
-        SCHSFreeCompileData(cd);
+        PatternDatabaseFree(*pd);
+        CompileDataFree(cd);
+        *pd = pd_cached;
         return 0;
     }
 
-    BUG_ON(ctx->pattern_db != NULL); /* already built? */
+    return -1; // not cached
+}
 
+static int PatternDatabaseCompile(PatternDatabase *pd, SCHSCompileData *cd)
+{
     for (uint32_t i = 0; i < pd->pattern_cnt; i++) {
         const SCHSPattern *p = pd->parray[i];
-
         cd->ids[i] = i;
         cd->flags[i] = HS_FLAG_SINGLEMATCH;
         if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
             cd->flags[i] |= HS_FLAG_CASELESS;
         }
-
         cd->expressions[i] = HSRenderPattern(p->original_pat, p->len);
+        if (CompileDataExtensionsInit(&cd->ext[i], p) != 0) {
+            return -1;
+        }
+    }
 
-        if (p->flags & (MPM_PATTERN_FLAG_OFFSET | MPM_PATTERN_FLAG_DEPTH)) {
-            cd->ext[i] = SCCalloc(1, sizeof(hs_expr_ext_t));
-            if (cd->ext[i] == NULL) {
-                SCMutexUnlock(&g_db_table_mutex);
-                goto error;
-            }
+    hs_compile_error_t *compile_err = NULL;
+    hs_error_t err = hs_compile_ext_multi((const char *const *)cd->expressions, cd->flags, cd->ids,
+            (const hs_expr_ext_t *const *)cd->ext, cd->pattern_cnt, HS_MODE_BLOCK, NULL, &pd->hs_db,
+            &compile_err);
+    if (err != HS_SUCCESS) {
+        HSLogCompileError(compile_err);
+        return -1;
+    }
 
-            if (p->flags & MPM_PATTERN_FLAG_OFFSET) {
-                cd->ext[i]->flags |= HS_EXT_FLAG_MIN_OFFSET;
-                cd->ext[i]->min_offset = p->offset + p->len;
-            }
-            if (p->flags & MPM_PATTERN_FLAG_DEPTH) {
-                cd->ext[i]->flags |= HS_EXT_FLAG_MAX_OFFSET;
-                cd->ext[i]->max_offset = p->offset + p->depth;
-            }
-        }
+    if (HSScratchAlloc(pd->hs_db) != 0) {
+        return -1;
     }
 
-    BUG_ON(mpm_ctx->pattern_cnt == 0);
+    if (HashTableAdd(g_db_table, pd, 1) < 0) {
+        return -1;
+    }
+    pd->ref_cnt = 1;
+    return 0;
+}
 
-    err = hs_compile_ext_multi((const char *const *)cd->expressions, cd->flags,
-                               cd->ids, (const hs_expr_ext_t *const *)cd->ext,
-                               cd->pattern_cnt, HS_MODE_BLOCK, NULL, &pd->hs_db,
-                               &compile_err);
+/**
+ * \brief Process the patterns added to the mpm, and create the internal tables.
+ *
+ * \param mpm_ctx Pointer to the mpm context.
+ */
+int SCHSPreparePatterns(MpmCtx *mpm_ctx)
+{
+    SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx;
 
-    if (err != HS_SUCCESS) {
-        SCLogError("failed to compile hyperscan database");
-        if (compile_err) {
-            SCLogError("compile error: %s", compile_err->message);
-        }
-        hs_free_compile_error(compile_err);
+    if (HSCheckPatterns(mpm_ctx, ctx) == 0) {
+        return 0;
+    }
+
+    SCHSCompileData *cd = CompileDataAlloc(mpm_ctx->pattern_cnt);
+    PatternDatabase *pd = PatternDatabaseAlloc(mpm_ctx->pattern_cnt);
+    if (cd == NULL || pd == NULL) {
+        goto error;
+    }
+
+    HSPatternArrayInit(ctx, pd);
+    /* Serialise whole database compilation as a relatively easy way to ensure
+     * dedupe is safe. */
+    SCMutexLock(&g_db_table_mutex);
+    if (HSGlobalPatternDatabaseInit() == -1) {
         SCMutexUnlock(&g_db_table_mutex);
         goto error;
     }
 
-    ctx->pattern_db = pd;
+    if (PatternDatabaseGetCached(&pd, cd) == 0 && pd != NULL) {
+        ctx->pattern_db = pd;
+        if (PatternDatabaseGetSize(pd, &ctx->hs_db_size) != 0) {
+            SCMutexUnlock(&g_db_table_mutex);
+            goto error;
+        }
 
-    SCMutexLock(&g_scratch_proto_mutex);
-    err = hs_alloc_scratch(pd->hs_db, &g_scratch_proto);
-    SCMutexUnlock(&g_scratch_proto_mutex);
-    if (err != HS_SUCCESS) {
-        SCLogError("failed to allocate scratch");
+        if (pd->ref_cnt == 1) {
+            // freshly allocated
+            mpm_ctx->memory_cnt++;
+            mpm_ctx->memory_size += ctx->hs_db_size;
+        }
+        SCMutexUnlock(&g_db_table_mutex);
+        return 0;
+    }
+
+    BUG_ON(ctx->pattern_db != NULL); /* already built? */
+    BUG_ON(mpm_ctx->pattern_cnt == 0);
+
+    if (PatternDatabaseCompile(pd, cd) != 0) {
         SCMutexUnlock(&g_db_table_mutex);
         goto error;
     }
 
-    err = hs_database_size(pd->hs_db, &ctx->hs_db_size);
-    if (err != HS_SUCCESS) {
-        SCLogError("failed to query database size");
+    ctx->pattern_db = pd;
+    if (PatternDatabaseGetSize(pd, &ctx->hs_db_size) != 0) {
         SCMutexUnlock(&g_db_table_mutex);
         goto error;
     }
@@ -715,26 +791,12 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx)
     mpm_ctx->memory_cnt++;
     mpm_ctx->memory_size += ctx->hs_db_size;
 
-    SCLogDebug("Built %" PRIu32 " patterns into a database of size %" PRIuMAX
-               " bytes", mpm_ctx->pattern_cnt, (uintmax_t)ctx->hs_db_size);
-
-    /* Cache this database globally for later. */
-    pd->ref_cnt = 1;
-    int r = HashTableAdd(g_db_table, pd, 1);
     SCMutexUnlock(&g_db_table_mutex);
-    if (r < 0)
-        goto error;
-
-    SCHSFreeCompileData(cd);
+    CompileDataFree(cd);
     return 0;
 
 error:
-    if (pd) {
-        PatternDatabaseFree(pd);
-    }
-    if (cd) {
-        SCHSFreeCompileData(cd);
-    }
+    SCHSCleanupOnError(pd, cd);
     return -1;
 }
 
diff --git a/src/util-mpm-hs.h b/src/util-mpm-hs.h
index 09deb5ff99f9..487abf8fa3ab 100644
--- a/src/util-mpm-hs.h
+++ b/src/util-mpm-hs.h
@@ -27,48 +27,6 @@
 #ifndef SURICATA_UTIL_MPM_HS__H
 #define SURICATA_UTIL_MPM_HS__H
 
-typedef struct SCHSPattern_ {
-    /* length of the pattern */
-    uint16_t len;
-    /* flags describing the pattern */
-    uint8_t flags;
-    /* holds the original pattern that was added */
-    uint8_t *original_pat;
-    /* pattern id */
-    uint32_t id;
-
-    uint16_t offset;
-    uint16_t depth;
-
-    /* sid(s) for this pattern */
-    uint32_t sids_size;
-    SigIntId *sids;
-
-    /* only used at ctx init time, when this structure is part of a hash
-     * table. */
-    struct SCHSPattern_ *next;
-} SCHSPattern;
-
-typedef struct SCHSCtx_ {
-    /* hash used during ctx initialization */
-    SCHSPattern **init_hash;
-
-    /* pattern database and pattern arrays. */
-    void *pattern_db;
-
-    /* size of database, for accounting. */
-    size_t hs_db_size;
-} SCHSCtx;
-
-typedef struct SCHSThreadCtx_ {
-    /* Hyperscan scratch space region for this thread, capable of handling any
-     * database that has been compiled. */
-    void *scratch;
-
-    /* size of scratch space, for accounting. */
-    size_t scratch_size;
-} SCHSThreadCtx;
-
 void MpmHSRegister(void);
 
 void MpmHSGlobalCleanup(void);

From 7f8306e6905f476a5ed18050afb9aa176ce8a9ea Mon Sep 17 00:00:00 2001
From: Lukas Sismis <lsismis@oisf.net>
Date: Mon, 28 Oct 2024 15:44:09 +0100
Subject: [PATCH 6/8] util-mpm: prepare MPM codebase for ruleset caching

---
 Makefile.am                |  1 +
 configure.ac               |  3 +++
 src/detect-engine-loader.c |  5 +++++
 src/detect-engine.c        | 18 ++++++++++++++++++
 src/detect-engine.h        |  2 ++
 src/util-mpm-ac-ks.c       |  1 +
 src/util-mpm-ac.c          |  1 +
 src/util-mpm-hs.c          |  1 +
 src/util-mpm.h             |  1 +
 suricata.yaml.in           |  4 ++++
 10 files changed, 37 insertions(+)

diff --git a/Makefile.am b/Makefile.am
index 20e50bdc4a03..7c44caca59e6 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -37,6 +37,7 @@ install-conf:
 	install -d "$(DESTDIR)$(e_rundir)"
 	install -m 770 -d "$(DESTDIR)$(e_localstatedir)"
 	install -m 770 -d "$(DESTDIR)$(e_datadir)"
+	install -m 660 -d "$(DESTDIR)$(e_hscachedir)"
 
 install-rules:
 if INSTALL_SURICATA_UPDATE
diff --git a/configure.ac b/configure.ac
index 93e728fc2246..5cf6b4ac1c2f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2456,6 +2456,7 @@ if test "$WINDOWS_PATH" = "yes"; then
 
     e_sysconfdir="${e_winbase}\\\\"
     e_defaultruledir="$e_winbase\\\\rules\\\\"
+    e_hscachedir="$e_winbase\\\\cache\\\\hs\\\\"
     e_magic_file="$e_winbase\\\\magic.mgc"
     e_logdir="$e_winbase\\\\log"
     e_logfilesdir="$e_logdir\\\\files"
@@ -2477,6 +2478,7 @@ else
     EXPAND_VARIABLE(sysconfdir, e_sysconfdir, "/suricata/")
     EXPAND_VARIABLE(localstatedir, e_localstatedir, "/run/suricata")
     EXPAND_VARIABLE(datadir, e_datarulesdir, "/suricata/rules")
+    EXPAND_VARIABLE(localstatedir, e_hscachedir, "/lib/suricata/cache/hs")
     EXPAND_VARIABLE(localstatedir, e_datadir, "/lib/suricata/data")
     EXPAND_VARIABLE(localstatedir, e_defaultruledir, "/lib/suricata/rules")
 
@@ -2490,6 +2492,7 @@ AC_SUBST(e_logcertsdir)
 AC_SUBST(e_sysconfdir)
 AC_DEFINE_UNQUOTED([CONFIG_DIR],["$e_sysconfdir"],[Our CONFIG_DIR])
 AC_SUBST(e_localstatedir)
+AC_SUBST(e_hscachedir)
 AC_SUBST(e_datadir)
 AC_DEFINE_UNQUOTED([DATA_DIR],["$e_datadir"],[Our DATA_DIR])
 AC_SUBST(e_magic_file)
diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c
index 950812a187c9..698fa28d85db 100644
--- a/src/detect-engine-loader.c
+++ b/src/detect-engine-loader.c
@@ -33,6 +33,7 @@
 #include "tm-threads.h"
 #include "queue.h"
 
+#include "detect-engine.h"
 #include "detect-engine-loader.h"
 #include "detect-engine-build.h"
 #include "detect-engine-analyzer.h"
@@ -402,6 +403,10 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, bool sig_file_exc
 
     ret = 0;
 
+    if (DetectEngineMpmCachingEnabled() && mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) {
+        mpm_table[de_ctx->mpm_matcher].CacheRuleset();
+    }
+
  end:
     gettimeofday(&de_ctx->last_reload, NULL);
     if (SCRunmodeGet() == RUNMODE_ENGINE_ANALYSIS) {
diff --git a/src/detect-engine.c b/src/detect-engine.c
index 887bf46ce5c8..c47653dca6a2 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -3001,6 +3001,24 @@ static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
     return 0;
 }
 
+bool DetectEngineMpmCachingEnabled(void)
+{
+    const char *strval = NULL;
+    if (ConfGet("detect.sgh-mpm-caching", &strval) != 1)
+        return false;
+
+    int sgh_mpm_caching = 0;
+    (void)ConfGetBool("detect.sgh-mpm-caching", &sgh_mpm_caching);
+    return (bool)sgh_mpm_caching;
+}
+
+const char *DetectEngineMpmCachingGetPath(void)
+{
+    const char *strval = NULL;
+    ConfGet("detect.sgh-mpm-caching-path", &strval);
+    return strval;
+}
+
 void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
 {
     de_ctx->signum = 0;
diff --git a/src/detect-engine.h b/src/detect-engine.h
index 6240a4492e77..305f77910ee6 100644
--- a/src/detect-engine.h
+++ b/src/detect-engine.h
@@ -99,6 +99,8 @@ void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, i
 
 TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **);
 TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
+bool DetectEngineMpmCachingEnabled(void);
+const char *DetectEngineMpmCachingGetPath(void);
 /* faster as a macro than a inline function on my box -- VJ */
 #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum)
 void DetectEngineResetMaxSigId(DetectEngineCtx *);
diff --git a/src/util-mpm-ac-ks.c b/src/util-mpm-ac-ks.c
index 60bb1b7a9e42..b8443276b729 100644
--- a/src/util-mpm-ac-ks.c
+++ b/src/util-mpm-ac-ks.c
@@ -1403,6 +1403,7 @@ void MpmACTileRegister(void)
     mpm_table[MPM_AC_KS].AddPattern = SCACTileAddPatternCS;
     mpm_table[MPM_AC_KS].AddPatternNocase = SCACTileAddPatternCI;
     mpm_table[MPM_AC_KS].Prepare = SCACTilePreparePatterns;
+    mpm_table[MPM_AC_KS].CacheRuleset = NULL;
     mpm_table[MPM_AC_KS].Search = SCACTileSearch;
     mpm_table[MPM_AC_KS].PrintCtx = SCACTilePrintInfo;
 #ifdef UNITTESTS
diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c
index 6e115acaa305..38b061ef53b6 100644
--- a/src/util-mpm-ac.c
+++ b/src/util-mpm-ac.c
@@ -1102,6 +1102,7 @@ void MpmACRegister(void)
     mpm_table[MPM_AC].AddPattern = SCACAddPatternCS;
     mpm_table[MPM_AC].AddPatternNocase = SCACAddPatternCI;
     mpm_table[MPM_AC].Prepare = SCACPreparePatterns;
+    mpm_table[MPM_AC].CacheRuleset = NULL;
     mpm_table[MPM_AC].Search = SCACSearch;
     mpm_table[MPM_AC].PrintCtx = SCACPrintInfo;
 #ifdef UNITTESTS
diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c
index da02e2194dfe..80d495e820ae 100644
--- a/src/util-mpm-hs.c
+++ b/src/util-mpm-hs.c
@@ -1110,6 +1110,7 @@ void MpmHSRegister(void)
     mpm_table[MPM_HS].AddPattern = SCHSAddPatternCS;
     mpm_table[MPM_HS].AddPatternNocase = SCHSAddPatternCI;
     mpm_table[MPM_HS].Prepare = SCHSPreparePatterns;
+    mpm_table[MPM_HS].CacheRuleset = NULL;
     mpm_table[MPM_HS].Search = SCHSSearch;
     mpm_table[MPM_HS].PrintCtx = SCHSPrintInfo;
     mpm_table[MPM_HS].PrintThreadCtx = SCHSPrintSearchStats;
diff --git a/src/util-mpm.h b/src/util-mpm.h
index 0bcd87899b6c..5df4bc0c8de2 100644
--- a/src/util-mpm.h
+++ b/src/util-mpm.h
@@ -164,6 +164,7 @@ typedef struct MpmTableElmt_ {
     int  (*AddPattern)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t);
     int  (*AddPatternNocase)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t);
     int  (*Prepare)(struct MpmCtx_ *);
+    int (*CacheRuleset)(void);
     /** \retval cnt number of patterns that matches: once per pattern max. */
     uint32_t (*Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t);
     void (*PrintCtx)(struct MpmCtx_ *);
diff --git a/suricata.yaml.in b/suricata.yaml.in
index 7dd7cb588675..706de93a6481 100644
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -1699,6 +1699,10 @@ detect:
     toclient-groups: 3
     toserver-groups: 25
   sgh-mpm-context: auto
+  # Cache MPM contexts to the disk to avoid rule compilation at the startup.
+  # Cache files are created in the standard library directory.
+  sgh-mpm-caching: yes
+  sgh-mpm-caching-path: @e_hscachedir@
   inspection-recursion-limit: 3000
   # maximum number of times a tx will get logged for a stream-only rule match
   # stream-tx-log-limit: 4

From fff39ecb47b40db0283a4af1b7deaea0bc5f881c Mon Sep 17 00:00:00 2001
From: Lukas Sismis <lsismis@oisf.net>
Date: Mon, 28 Oct 2024 15:46:17 +0100
Subject: [PATCH 7/8] hyperscan: add caching mechanism for hyperscan contexts

Cache Hyperscan serialized databases to disk to prevent compilation
of the same databases when Suricata is run again with the same
ruleset.
Hyperscan binary files are stored per rulegroup in the designated
folder, by default in the cached library folder.
Since caching is per signature group heads,
some chunk of the ruleset can change and it still can reuse part of
the unchanged signature groups.

Loading *fresh* ET Open ruleset:  19 seconds
Loading *cached* ET Open ruleset: 07 seconds

Ticket: 7170
---
 doc/userguide/performance/hyperscan.rst |  26 ++-
 src/Makefile.am                         |   2 +
 src/util-mpm-hs-cache.c                 | 252 ++++++++++++++++++++++++
 src/util-mpm-hs-cache.h                 |  37 ++++
 src/util-mpm-hs-core.h                  |   6 +
 src/util-mpm-hs.c                       |  43 +++-
 6 files changed, 364 insertions(+), 2 deletions(-)
 create mode 100644 src/util-mpm-hs-cache.c
 create mode 100644 src/util-mpm-hs-cache.h

diff --git a/doc/userguide/performance/hyperscan.rst b/doc/userguide/performance/hyperscan.rst
index 055fa7f21b75..9b39bc22e95b 100644
--- a/doc/userguide/performance/hyperscan.rst
+++ b/doc/userguide/performance/hyperscan.rst
@@ -81,4 +81,28 @@ if it is present on the system in case of the "auto" setting.
 
 
 If the current suricata installation does not have hyperscan
-support, refer to :ref:`installation`
\ No newline at end of file
+support, refer to :ref:`installation`
+
+Hyperscan caching
+~~~~~~~~~~~~~~~~~
+
+Upon startup, Hyperscan compiles and optimizes the ruleset into its own
+internal structure. Suricata optimizes the startup process by saving
+the Hyperscan internal structures to disk and loading them on the next start.
+This prevents the recompilation of the ruleset and results in faster
+initialization. If the ruleset is changed, new necessary cache files are
+automatically created.
+
+To enable this function, in `suricata.yaml` configure:
+
+::
+
+  # Cache MPM contexts to the disk to avoid rule compilation at the startup.
+  # Cache files are created in the standard library directory.
+  sgh-mpm-caching: yes
+  sgh-mpm-caching-path: /var/lib/suricata/cache/hs
+
+
+**Note**:
+You might need to create and adjust permissions to the default caching folder
+path, especially if you are running Suricata as a non-root user.
diff --git a/src/Makefile.am b/src/Makefile.am
index 680206d2a84f..3598b42d39d5 100755
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -536,6 +536,7 @@ noinst_HEADERS = \
 	util-mpm-ac-ks.h \
 	util-mpm.h \
 	util-mpm-hs.h \
+	util-mpm-hs-cache.h \
 	util-mpm-hs-core.h \
 	util-optimize.h \
 	util-pages.h \
@@ -1089,6 +1090,7 @@ libsuricata_c_a_SOURCES = \
 	util-mpm-ac-ks-small.c \
 	util-mpm.c \
 	util-mpm-hs.c \
+	util-mpm-hs-cache.c \
 	util-mpm-hs-core.c \
 	util-pages.c \
 	util-path.c \
diff --git a/src/util-mpm-hs-cache.c b/src/util-mpm-hs-cache.c
new file mode 100644
index 000000000000..39111b2e7752
--- /dev/null
+++ b/src/util-mpm-hs-cache.c
@@ -0,0 +1,252 @@
+/* Copyright (C) 2007-2024 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jim Xu <jim.xu@windriver.com>
+ * \author Justin Viiret <justin.viiret@intel.com>
+ *
+ * MPM pattern matcher that calls the Hyperscan regex matcher.
+ */
+
+#include "suricata-common.h"
+#include "suricata.h"
+#include "detect-engine.h"
+#include "util-debug.h"
+#include "util-hash-lookup3.h"
+#include "util-mpm-hs-core.h"
+#include "util-mpm-hs-cache.h"
+#include "util-path.h"
+
+#ifdef BUILD_HYPERSCAN
+
+#include <hs.h>
+
+static const char *HSCacheConstructFPath(uint64_t hs_db_hash)
+{
+    static char hash_file_path[PATH_MAX];
+
+    char hash_file_path_suffix[] = "_v1.hs";
+    char filename[PATH_MAX];
+    uint64_t r =
+            snprintf(filename, sizeof(filename), "%020lu%s", hs_db_hash, hash_file_path_suffix);
+    if (r != (uint64_t)(20 + strlen(hash_file_path_suffix)))
+        return NULL;
+
+    r = PathMerge(
+            hash_file_path, sizeof(hash_file_path), DetectEngineMpmCachingGetPath(), filename);
+    if (r)
+        return NULL;
+
+    return hash_file_path;
+}
+
+static char *HSReadStream(const char *file_path, size_t *buffer_sz)
+{
+    FILE *file = fopen(file_path, "rb");
+    if (!file) {
+        SCLogDebug("Failed to open file %s: %s", file_path, strerror(errno));
+        return NULL;
+    }
+
+    // Seek to the end of the file to determine its size
+    fseek(file, 0, SEEK_END);
+    long file_sz = ftell(file);
+    if (file_sz < 0) {
+        SCLogDebug("Failed to determine file size of %s: %s", file_path, strerror(errno));
+        fclose(file);
+        return NULL;
+    }
+
+    char *buffer = (char *)SCCalloc(file_sz, sizeof(char));
+    if (!buffer) {
+        SCLogWarning("Failed to allocate memory");
+        fclose(file);
+        return NULL;
+    }
+
+    // Rewind file pointer and read the file into the buffer
+    rewind(file);
+    size_t bytes_read = fread(buffer, 1, file_sz, file);
+    if (bytes_read != (size_t)file_sz) {
+        SCLogDebug("Failed to read the entire file %s: %s", file_path, strerror(errno));
+        SCFree(buffer);
+        fclose(file);
+        return NULL;
+    }
+
+    *buffer_sz = file_sz;
+    fclose(file);
+    return buffer;
+}
+
+/**
+ * Function to hash the searched pattern, only things relevant to Hyperscan
+ * compilation are hashed.
+ */
+static void SCHSCachePatternHash(const SCHSPattern *p, uint32_t *h1, uint32_t *h2)
+{
+    BUG_ON(p->original_pat == NULL);
+    hashlittle2_safe(&p->len, sizeof(p->len), h1, h2);
+    hashlittle2_safe(&p->flags, sizeof(p->flags), h1, h2);
+    hashlittle2_safe(p->original_pat, p->len, h1, h2);
+    hashlittle2_safe(&p->offset, sizeof(p->offset), h1, h2);
+    hashlittle2_safe(&p->depth, sizeof(p->depth), h1, h2);
+}
+
+int HSLoadCache(hs_database_t **hs_db, uint64_t hs_db_hash)
+{
+    const char *hash_file_static = HSCacheConstructFPath(hs_db_hash);
+    if (hash_file_static == NULL)
+        return -1;
+
+    SCLogDebug("Loading the cached HS DB from %s", hash_file_static);
+    if (!SCPathExists(hash_file_static))
+        return -1;
+
+    FILE *db_cache = fopen(hash_file_static, "r");
+    char *buffer = NULL;
+    int ret = 0;
+    if (db_cache) {
+        size_t buffer_size;
+        buffer = HSReadStream(hash_file_static, &buffer_size);
+        if (!buffer) {
+            SCLogWarning("Hyperscan cached DB file %s cannot be read", hash_file_static);
+            ret = -1;
+            goto freeup;
+        }
+
+        hs_error_t error = hs_deserialize_database(buffer, buffer_size, hs_db);
+        if (error != HS_SUCCESS) {
+            SCLogWarning("Failed to deserialize Hyperscan database of %s: %s", hash_file_static,
+                    HSErrorToStr(error));
+            ret = -1;
+            goto freeup;
+        }
+
+        ret = 0;
+        goto freeup;
+    }
+
+freeup:
+    if (db_cache)
+        fclose(db_cache);
+    if (buffer)
+        SCFree(buffer);
+    return ret;
+}
+
+static int HSSaveCache(hs_database_t *hs_db, uint64_t hs_db_hash)
+{
+    static bool notified = false;
+    char *db_stream = NULL;
+    size_t db_size;
+    int ret = -1;
+
+    hs_error_t err = hs_serialize_database(hs_db, &db_stream, &db_size);
+    if (err != HS_SUCCESS) {
+        SCLogWarning("Failed to serialize Hyperscan database: %s", HSErrorToStr(err));
+        goto cleanup;
+    }
+
+    const char *hash_file_static = HSCacheConstructFPath(hs_db_hash);
+    SCLogDebug("Caching the compiled HS at %s", hash_file_static);
+    if (SCPathExists(hash_file_static)) {
+        // potentially signs that it might not work as expected as we got into
+        // hash collision. If this happens with older and not used caches it is
+        // fine.
+        // It is problematic when one ruleset yields two colliding MPM groups.
+        SCLogWarning("Overwriting cache file %s. If the problem persists consider switching off "
+                     "the caching",
+                hash_file_static);
+    }
+
+    if (SCCreateDirectoryTree(DetectEngineMpmCachingGetPath(), true) != 0) {
+        if (!notified) {
+            SCLogWarning("Failed to create Hyperscan cache folder, make sure "
+                         "the  parent folder is writeable "
+                         "or adjust sgh-mpm-caching-path setting (%s)",
+                    DetectEngineMpmCachingGetPath());
+            notified = true;
+        }
+        return -1;
+    }
+
+    FILE *db_cache_out = fopen(hash_file_static, "w");
+    if (!db_cache_out) {
+        if (!notified) {
+            SCLogWarning("Failed to create Hyperscan cache file, make sure the folder exist and is "
+                         "writable or adjust sgh-mpm-caching-path setting (%s)",
+                    hash_file_static);
+            notified = true;
+        }
+        goto cleanup;
+    }
+    size_t r = fwrite(db_stream, sizeof(db_stream[0]), db_size, db_cache_out);
+    if (r > 0 && (size_t)r != db_size) {
+        SCLogWarning("Failed to write to file: %s", hash_file_static);
+        if (r != db_size) {
+            // possibly a corrupted DB cache was created
+            r = remove(hash_file_static);
+            if (r != 0) {
+                SCLogWarning("Failed to remove corrupted cache file: %s", hash_file_static);
+            }
+        }
+    }
+    ret = fclose(db_cache_out);
+    if (ret != 0) {
+        SCLogWarning("Failed to close file: %s", hash_file_static);
+        goto cleanup;
+    }
+
+    ret = 0;
+cleanup:
+    if (db_stream)
+        SCFree(db_stream);
+    return ret;
+}
+
+uint64_t HSHashDb(const PatternDatabase *pd)
+{
+    uint64_t cached_hash = 0;
+    uint32_t *hash = (uint32_t *)(&cached_hash);
+    hashword2(&pd->pattern_cnt, 1, &hash[0], &hash[1]);
+    for (uint32_t i = 0; i < pd->pattern_cnt; i++) {
+        SCHSCachePatternHash(pd->parray[i], &hash[0], &hash[1]);
+    }
+
+    return cached_hash;
+}
+
+void HSSaveCacheIterator(void *data, void *aux)
+{
+    PatternDatabase *pd = (PatternDatabase *)data;
+    PatternDatabaseCache *pd_stats = (PatternDatabaseCache *)aux;
+    pd_stats->hs_dbs_cnt++;
+    if (pd->cached) {
+        pd_stats->hs_dbs_cache_loaded_cnt++;
+        return;
+    }
+
+    if (HSSaveCache(pd->hs_db, HSHashDb(pd)) == 0) {
+        pd->cached = true; // for rule reloads
+        pd_stats->hs_dbs_cache_saved_cnt++;
+    }
+}
+
+#endif /* BUILD_HYPERSCAN */
diff --git a/src/util-mpm-hs-cache.h b/src/util-mpm-hs-cache.h
new file mode 100644
index 000000000000..cc17d9846cc7
--- /dev/null
+++ b/src/util-mpm-hs-cache.h
@@ -0,0 +1,37 @@
+/* Copyright (C) 2024 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Lukas Sismis <lsismis@oisf.net>
+ *
+ * Hyperscan caching logic for faster database compilation.
+ */
+
+#ifndef SURICATA_UTIL_MPM_HS_CACHE__H
+#define SURICATA_UTIL_MPM_HS_CACHE__H
+
+#include "util-mpm-hs-core.h"
+
+#ifdef BUILD_HYPERSCAN
+int HSLoadCache(hs_database_t **hs_db, uint64_t hs_db_hash);
+uint64_t HSHashDb(const PatternDatabase *pd);
+void HSSaveCacheIterator(void *data, void *aux);
+#endif /* BUILD_HYPERSCAN */
+
+#endif /* SURICATA_UTIL_MPM_HS_CACHE__H */
diff --git a/src/util-mpm-hs-core.h b/src/util-mpm-hs-core.h
index fc7c2d302848..77d6082877db 100644
--- a/src/util-mpm-hs-core.h
+++ b/src/util-mpm-hs-core.h
@@ -87,6 +87,12 @@ typedef struct PatternDatabase_ {
     bool cached;
 } PatternDatabase;
 
+typedef struct PatternDatabaseCache_ {
+    uint32_t hs_dbs_cnt;
+    uint32_t hs_dbs_cache_loaded_cnt;
+    uint32_t hs_dbs_cache_saved_cnt;
+} PatternDatabaseCache;
+
 const char *HSErrorToStr(hs_error_t error_code);
 
 #endif /* BUILD_HYPERSCAN */
diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c
index 80d495e820ae..050ce2784e4b 100644
--- a/src/util-mpm-hs.c
+++ b/src/util-mpm-hs.c
@@ -33,16 +33,19 @@
 #include "detect-engine-build.h"
 
 #include "conf.h"
+#include "util-conf.h"
 #include "util-debug.h"
 #include "util-unittest.h"
 #include "util-unittest-helper.h"
 #include "util-memcmp.h"
 #include "util-mpm-hs.h"
+#include "util-mpm-hs-cache.h"
 #include "util-mpm-hs-core.h"
 #include "util-memcpy.h"
 #include "util-hash.h"
 #include "util-hash-lookup3.h"
 #include "util-hyperscan.h"
+#include "util-path.h"
 
 #ifdef BUILD_HYPERSCAN
 
@@ -551,6 +554,7 @@ static PatternDatabase *PatternDatabaseAlloc(uint32_t pattern_cnt)
     pd->pattern_cnt = pattern_cnt;
     pd->ref_cnt = 0;
     pd->hs_db = NULL;
+    pd->cached = false;
 
     /* alloc the pattern array */
     pd->parray = (SCHSPattern **)SCCalloc(pd->pattern_cnt, sizeof(SCHSPattern *));
@@ -690,6 +694,26 @@ static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd)
         CompileDataFree(cd);
         *pd = pd_cached;
         return 0;
+    } else if (DetectEngineMpmCachingEnabled()) {
+        pd_cached = *pd;
+        uint64_t db_lookup_hash = HSHashDb(pd_cached);
+        if (HSLoadCache(&pd_cached->hs_db, db_lookup_hash) == 0) {
+            pd_cached->ref_cnt = 1;
+            pd_cached->cached = true;
+            if (HSScratchAlloc(pd_cached->hs_db) != 0) {
+                goto recover;
+            }
+            if (HashTableAdd(g_db_table, pd_cached, 1) < 0) {
+                goto recover;
+            }
+            CompileDataFree(cd);
+            return 0;
+
+        recover:
+            pd_cached->ref_cnt = 0;
+            pd_cached->cached = false;
+            return -1;
+        }
     }
 
     return -1; // not cached
@@ -800,6 +824,23 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx)
     return -1;
 }
 
+/**
+ * \brief Cache the loaded ruleset
+ *
+ */
+static int SCHSCacheRuleset(void)
+{
+    SCLogDebug("Caching the loaded ruleset ");
+    PatternDatabaseCache pd_stats = { 0 };
+    SCMutexLock(&g_db_table_mutex);
+    HashTableIterate(g_db_table, HSSaveCacheIterator, &pd_stats);
+    SCMutexUnlock(&g_db_table_mutex);
+    SCLogInfo("%u rule groups cached (%u newly cached) of total %u groups",
+            pd_stats.hs_dbs_cache_loaded_cnt + pd_stats.hs_dbs_cache_saved_cnt,
+            pd_stats.hs_dbs_cache_saved_cnt, pd_stats.hs_dbs_cnt);
+    return 0;
+}
+
 /**
  * \brief Init the mpm thread context.
  *
@@ -1110,7 +1151,7 @@ void MpmHSRegister(void)
     mpm_table[MPM_HS].AddPattern = SCHSAddPatternCS;
     mpm_table[MPM_HS].AddPatternNocase = SCHSAddPatternCI;
     mpm_table[MPM_HS].Prepare = SCHSPreparePatterns;
-    mpm_table[MPM_HS].CacheRuleset = NULL;
+    mpm_table[MPM_HS].CacheRuleset = SCHSCacheRuleset;
     mpm_table[MPM_HS].Search = SCHSSearch;
     mpm_table[MPM_HS].PrintCtx = SCHSPrintInfo;
     mpm_table[MPM_HS].PrintThreadCtx = SCHSPrintSearchStats;

From 3e98865982abe6e9f84d2a8edb6b5445535a3fc8 Mon Sep 17 00:00:00 2001
From: Lukas Sismis <lsismis@oisf.net>
Date: Mon, 28 Oct 2024 17:23:32 +0100
Subject: [PATCH 8/8] detect-engine: make DetectEngineMpmCachingEnabled a
 private function

---
 src/app-layer-detect-proto.c |  2 +-
 src/app-layer-ftp.c          |  3 +-
 src/app-layer-smtp.c         |  2 +-
 src/detect-engine-loader.c   |  2 +-
 src/detect-engine-mpm.c      | 22 +++++------
 src/detect-engine.c          | 37 ++++++++++---------
 src/detect-engine.h          |  1 -
 src/detect.h                 |  3 ++
 src/util-mpm-ac-ks.c         | 60 +++++++++++++++---------------
 src/util-mpm-ac.c            | 62 +++++++++++++++----------------
 src/util-mpm-hs-cache.c      |  6 ++-
 src/util-mpm-hs-core.h       |  4 +-
 src/util-mpm-hs.c            | 71 ++++++++++++++++++------------------
 src/util-mpm.h               |  2 +-
 14 files changed, 143 insertions(+), 134 deletions(-)

diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c
index a65a98c88a41..2498fd1c15fe 100644
--- a/src/app-layer-detect-proto.c
+++ b/src/app-layer-detect-proto.c
@@ -1290,7 +1290,7 @@ static int AppLayerProtoDetectPMPrepareMpm(AppLayerProtoDetectPMCtx *ctx)
     int ret = 0;
     MpmCtx *mpm_ctx = &ctx->mpm_ctx;
 
-    if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx) < 0)
+    if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx, false) < 0)
         goto error;
 
     goto end;
diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c
index a1a99d4bd701..c19a86b70920 100644
--- a/src/app-layer-ftp.c
+++ b/src/app-layer-ftp.c
@@ -1281,8 +1281,7 @@ static void FTPSetMpmState(void)
                        i /*  id */, i /* rule id */ , 0 /* no flags */);
     }
 
-    mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx);
-
+    mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx, false);
 }
 
 static void FTPFreeMpmState(void)
diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index ec4799605cbd..9afb34ade724 100644
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -1632,7 +1632,7 @@ static void SMTPSetMpmState(void)
                         i /* pattern id */, i /* rule id */ , 0 /* no flags */);
     }
 
-    mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx);
+    mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx, false);
 }
 
 static void SMTPFreeMpmState(void)
diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c
index 698fa28d85db..82a3d3463cc8 100644
--- a/src/detect-engine-loader.c
+++ b/src/detect-engine-loader.c
@@ -403,7 +403,7 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, bool sig_file_exc
 
     ret = 0;
 
-    if (DetectEngineMpmCachingEnabled() && mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) {
+    if (de_ctx->mpm_cache_to_disk && mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) {
         mpm_table[de_ctx->mpm_matcher].CacheRuleset();
     }
 
diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index 5e8687e34686..acdc47b87f7e 100644
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -295,7 +295,7 @@ int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
             MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
                 }
             }
         }
@@ -524,7 +524,7 @@ int DetectMpmPrepareFrameMpms(DetectEngineCtx *de_ctx)
             SCLogDebug("%s: %d mpm_Ctx %p", am->name, r, mpm_ctx);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
                     SCLogDebug("%s: %d", am->name, r);
                 }
             }
@@ -689,7 +689,7 @@ int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx)
             MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, 0);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
                     SCLogDebug("%s: %d", am->name, r);
                 }
             }
@@ -744,40 +744,40 @@ int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
     if (de_ctx->sgh_mpm_context_proto_tcp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 1);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
     if (de_ctx->sgh_mpm_context_proto_udp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 1);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
     if (de_ctx->sgh_mpm_context_proto_other_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_other_packet, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
     if (de_ctx->sgh_mpm_context_stream != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 1);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
@@ -1618,7 +1618,7 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
     } else {
         if (ms->sgh_mpm_context == MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
             if (mpm_table[ms->mpm_ctx->mpm_type].Prepare != NULL) {
-                mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx);
+                mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx, de_ctx->mpm_cache_to_disk);
             }
         }
     }
diff --git a/src/detect-engine.c b/src/detect-engine.c
index c47653dca6a2..1c9474990cc8 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -2457,6 +2457,24 @@ static int DetectEngineReloadThreads(DetectEngineCtx *new_de_ctx)
     return -1;
 }
 
+static bool DetectEngineMpmCachingEnabled(void)
+{
+    const char *strval = NULL;
+    if (ConfGet("detect.sgh-mpm-caching", &strval) != 1)
+        return false;
+
+    int sgh_mpm_caching = 0;
+    (void)ConfGetBool("detect.sgh-mpm-caching", &sgh_mpm_caching);
+    return (bool)sgh_mpm_caching;
+}
+
+const char *DetectEngineMpmCachingGetPath(void)
+{
+    const char *strval = NULL;
+    ConfGet("detect.sgh-mpm-caching-path", &strval);
+    return strval;
+}
+
 static DetectEngineCtx *DetectEngineCtxInitReal(
         enum DetectEngineType type, const char *prefix, uint32_t tenant_id)
 {
@@ -2488,6 +2506,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal(
     de_ctx->failure_fatal = (failure_fatal == 1);
 
     de_ctx->mpm_matcher = PatternMatchDefaultMatcher();
+    de_ctx->mpm_cache_to_disk = DetectEngineMpmCachingEnabled();
     de_ctx->spm_matcher = SinglePatternMatchDefaultMatcher();
     SCLogConfig("pattern matchers: MPM: %s, SPM: %s",
         mpm_table[de_ctx->mpm_matcher].name,
@@ -3001,24 +3020,6 @@ static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
     return 0;
 }
 
-bool DetectEngineMpmCachingEnabled(void)
-{
-    const char *strval = NULL;
-    if (ConfGet("detect.sgh-mpm-caching", &strval) != 1)
-        return false;
-
-    int sgh_mpm_caching = 0;
-    (void)ConfGetBool("detect.sgh-mpm-caching", &sgh_mpm_caching);
-    return (bool)sgh_mpm_caching;
-}
-
-const char *DetectEngineMpmCachingGetPath(void)
-{
-    const char *strval = NULL;
-    ConfGet("detect.sgh-mpm-caching-path", &strval);
-    return strval;
-}
-
 void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
 {
     de_ctx->signum = 0;
diff --git a/src/detect-engine.h b/src/detect-engine.h
index 305f77910ee6..961e5976b627 100644
--- a/src/detect-engine.h
+++ b/src/detect-engine.h
@@ -99,7 +99,6 @@ void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, i
 
 TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **);
 TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
-bool DetectEngineMpmCachingEnabled(void);
 const char *DetectEngineMpmCachingGetPath(void);
 /* faster as a macro than a inline function on my box -- VJ */
 #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum)
diff --git a/src/detect.h b/src/detect.h
index fe755b7f0d14..1be1c0ad79b0 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -1045,6 +1045,9 @@ typedef struct DetectEngineCtx_ {
 
     /* number of signatures using filestore, limited as u16 */
     uint16_t filestore_cnt;
+
+    /* If enabled, MPM matchers can store compiled pattern databases to disk */
+    bool mpm_cache_to_disk;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */
diff --git a/src/util-mpm-ac-ks.c b/src/util-mpm-ac-ks.c
index b8443276b729..6eeff3e3a29c 100644
--- a/src/util-mpm-ac-ks.c
+++ b/src/util-mpm-ac-ks.c
@@ -90,7 +90,7 @@ int SCACTileAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t,
                          uint32_t, SigIntId, uint8_t);
 int SCACTileAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t,
                          uint32_t, SigIntId, uint8_t);
-int SCACTilePreparePatterns(MpmCtx *mpm_ctx);
+int SCACTilePreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk);
 uint32_t SCACTileSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         PrefilterRuleStore *pmq, const uint8_t *buf,
                         uint32_t buflen);
@@ -861,7 +861,7 @@ static void SCACTilePrepareSearch(MpmCtx *mpm_ctx)
  *
  * \param mpm_ctx Pointer to the mpm context.
  */
-int SCACTilePreparePatterns(MpmCtx *mpm_ctx)
+int SCACTilePreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk)
 {
     SCACTileSearchCtx *search_ctx = (SCACTileSearchCtx *)mpm_ctx->ctx;
 
@@ -1433,7 +1433,7 @@ static int SCACTileTest01(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
 
@@ -1465,7 +1465,7 @@ static int SCACTileTest02(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1500,7 +1500,7 @@ static int SCACTileTest03(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1532,7 +1532,7 @@ static int SCACTileTest04(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1564,7 +1564,7 @@ static int SCACTileTest05(void)
     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1594,7 +1594,7 @@ static int SCACTileTest06(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcd";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1636,7 +1636,7 @@ static int SCACTileTest07(void)
     PmqSetup(&pmq);
     /* total matches: 135: 6 unique */
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1663,7 +1663,7 @@ static int SCACTileTest08(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                                   (uint8_t *)"a", 1);
@@ -1693,7 +1693,7 @@ static int SCACTileTest09(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                                   (uint8_t *)"ab", 2);
@@ -1723,7 +1723,7 @@ static int SCACTileTest10(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "01234567890123456789012345678901234567890123456789"
                 "01234567890123456789012345678901234567890123456789"
@@ -1764,7 +1764,7 @@ static int SCACTileTest11(void)
         goto end;
     PmqSetup(&pmq);
 
-    if (SCACTilePreparePatterns(&mpm_ctx) == -1)
+    if (SCACTilePreparePatterns(&mpm_ctx, false) == -1)
         goto end;
 
     result = 1;
@@ -1805,7 +1805,7 @@ static int SCACTileTest12(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyz";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1837,7 +1837,7 @@ static int SCACTileTest13(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABCD";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1869,7 +1869,7 @@ static int SCACTileTest14(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1901,7 +1901,7 @@ static int SCACTileTest15(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1933,7 +1933,7 @@ static int SCACTileTest16(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABC";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1965,7 +1965,7 @@ static int SCACTileTest17(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzAB";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2002,7 +2002,7 @@ static int SCACTileTest18(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcde""fghij""klmno""pqrst""uvwxy""z";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2034,7 +2034,7 @@ static int SCACTileTest19(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2072,7 +2072,7 @@ static int SCACTileTest20(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AA";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2103,7 +2103,7 @@ static int SCACTileTest21(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                                   (uint8_t *)"AA", 2);
@@ -2135,7 +2135,7 @@ static int SCACTileTest22(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyz";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2166,7 +2166,7 @@ static int SCACTileTest23(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                                   (uint8_t *)"aa", 2);
@@ -2196,7 +2196,7 @@ static int SCACTileTest24(void)
     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                                   (uint8_t *)"aa", 2);
@@ -2227,7 +2227,7 @@ static int SCACTileTest25(void)
     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2258,7 +2258,7 @@ static int SCACTileTest26(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "works";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2289,7 +2289,7 @@ static int SCACTileTest27(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "tone";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2320,7 +2320,7 @@ static int SCACTileTest28(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACTilePreparePatterns(&mpm_ctx);
+    SCACTilePreparePatterns(&mpm_ctx, false);
 
     const char *buf = "tONE";
     uint32_t cnt = SCACTileSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c
index 38b061ef53b6..c8c01442ea73 100644
--- a/src/util-mpm-ac.c
+++ b/src/util-mpm-ac.c
@@ -68,7 +68,7 @@ int SCACAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t,
                      uint32_t, SigIntId, uint8_t);
 int SCACAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t,
                      uint32_t, SigIntId, uint8_t);
-int SCACPreparePatterns(MpmCtx *mpm_ctx);
+int SCACPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk);
 uint32_t SCACSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                     PrefilterRuleStore *pmq, const uint8_t *buf, uint32_t buflen);
 void SCACPrintInfo(MpmCtx *mpm_ctx);
@@ -702,7 +702,7 @@ static void SCACPrepareStateTable(MpmCtx *mpm_ctx)
  *
  * \param mpm_ctx Pointer to the mpm context.
  */
-int SCACPreparePatterns(MpmCtx *mpm_ctx)
+int SCACPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk)
 {
     SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx;
 
@@ -1131,7 +1131,7 @@ static int SCACTest01(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
 
@@ -1163,7 +1163,7 @@ static int SCACTest02(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1198,7 +1198,7 @@ static int SCACTest03(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1230,7 +1230,7 @@ static int SCACTest04(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1262,7 +1262,7 @@ static int SCACTest05(void)
     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1292,7 +1292,7 @@ static int SCACTest06(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcd";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1334,7 +1334,7 @@ static int SCACTest07(void)
     PmqSetup(&pmq);
     /* total matches: 135: unique matches: 6 */
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1361,7 +1361,7 @@ static int SCACTest08(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                                (uint8_t *)"a", 1);
@@ -1391,7 +1391,7 @@ static int SCACTest09(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                                (uint8_t *)"ab", 2);
@@ -1421,7 +1421,7 @@ static int SCACTest10(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "01234567890123456789012345678901234567890123456789"
                 "01234567890123456789012345678901234567890123456789"
@@ -1462,7 +1462,7 @@ static int SCACTest11(void)
         goto end;
     PmqSetup(&pmq);
 
-    if (SCACPreparePatterns(&mpm_ctx) == -1)
+    if (SCACPreparePatterns(&mpm_ctx, false) == -1)
         goto end;
 
     result = 1;
@@ -1503,7 +1503,7 @@ static int SCACTest12(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyz";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1535,7 +1535,7 @@ static int SCACTest13(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABCD";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1567,7 +1567,7 @@ static int SCACTest14(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1599,7 +1599,7 @@ static int SCACTest15(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1631,7 +1631,7 @@ static int SCACTest16(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABC";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1663,7 +1663,7 @@ static int SCACTest17(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzAB";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1700,7 +1700,7 @@ static int SCACTest18(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcde""fghij""klmno""pqrst""uvwxy""z";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1732,7 +1732,7 @@ static int SCACTest19(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1770,7 +1770,7 @@ static int SCACTest20(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AAAAA""AA";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1801,7 +1801,7 @@ static int SCACTest21(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                               (uint8_t *)"AA", 2);
@@ -1833,7 +1833,7 @@ static int SCACTest22(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyz";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1864,7 +1864,7 @@ static int SCACTest23(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                               (uint8_t *)"aa", 2);
@@ -1894,7 +1894,7 @@ static int SCACTest24(void)
     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
                               (uint8_t *)"aa", 2);
@@ -1925,7 +1925,7 @@ static int SCACTest25(void)
     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1956,7 +1956,7 @@ static int SCACTest26(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "works";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -1987,7 +1987,7 @@ static int SCACTest27(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "tone";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2017,7 +2017,7 @@ static int SCACTest28(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf = "tONE";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq,
@@ -2084,7 +2084,7 @@ static int SCACTest30(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"xyz", 3, 0, 0, 0, 0, MPM_PATTERN_FLAG_ENDSWITH);
     PmqSetup(&pmq);
 
-    SCACPreparePatterns(&mpm_ctx);
+    SCACPreparePatterns(&mpm_ctx, false);
 
     const char *buf1 = "abcdefghijklmnopqrstuvwxyz";
     uint32_t cnt = SCACSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf1, strlen(buf1));
diff --git a/src/util-mpm-hs-cache.c b/src/util-mpm-hs-cache.c
index 39111b2e7752..b46c2c3c5e30 100644
--- a/src/util-mpm-hs-cache.c
+++ b/src/util-mpm-hs-cache.c
@@ -237,7 +237,11 @@ void HSSaveCacheIterator(void *data, void *aux)
 {
     PatternDatabase *pd = (PatternDatabase *)data;
     PatternDatabaseCache *pd_stats = (PatternDatabaseCache *)aux;
-    pd_stats->hs_dbs_cnt++;
+    if (pd->no_cache)
+        return;
+
+    // count only cacheable DBs
+    pd_stats->hs_cacheable_dbs_cnt++;
     if (pd->cached) {
         pd_stats->hs_dbs_cache_loaded_cnt++;
         return;
diff --git a/src/util-mpm-hs-core.h b/src/util-mpm-hs-core.h
index 77d6082877db..3a4b901ce1f8 100644
--- a/src/util-mpm-hs-core.h
+++ b/src/util-mpm-hs-core.h
@@ -85,10 +85,12 @@ typedef struct PatternDatabase_ {
     uint32_t ref_cnt;
     /* Signals if the matcher has loaded/saved the pattern database to disk */
     bool cached;
+    /* Matcher will not cache this pattern DB */
+    bool no_cache;
 } PatternDatabase;
 
 typedef struct PatternDatabaseCache_ {
-    uint32_t hs_dbs_cnt;
+    uint32_t hs_cacheable_dbs_cnt;
     uint32_t hs_dbs_cache_loaded_cnt;
     uint32_t hs_dbs_cache_saved_cnt;
 } PatternDatabaseCache;
diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c
index 050ce2784e4b..393e5c6fd40d 100644
--- a/src/util-mpm-hs.c
+++ b/src/util-mpm-hs.c
@@ -59,7 +59,7 @@ int SCHSAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t,
                      uint32_t, SigIntId, uint8_t);
 int SCHSAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t,
                      uint32_t, SigIntId, uint8_t);
-int SCHSPreparePatterns(MpmCtx *mpm_ctx);
+int SCHSPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk);
 uint32_t SCHSSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                     PrefilterRuleStore *pmq, const uint8_t *buf, const uint32_t buflen);
 void SCHSPrintInfo(MpmCtx *mpm_ctx);
@@ -679,7 +679,7 @@ static int CompileDataExtensionsInit(hs_expr_ext_t **ext, const SCHSPattern *p)
  * \param SCHSCompileData* [in] Pointer to the compile data.
  * \retval 0 On success, negative value on failure.
  */
-static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd)
+static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd, bool disk_cache)
 {
     /* Check global hash table to see if we've seen this pattern database
      * before, and reuse the Hyperscan database if so. */
@@ -694,7 +694,7 @@ static int PatternDatabaseGetCached(PatternDatabase **pd, SCHSCompileData *cd)
         CompileDataFree(cd);
         *pd = pd_cached;
         return 0;
-    } else if (DetectEngineMpmCachingEnabled()) {
+    } else if (disk_cache) {
         pd_cached = *pd;
         uint64_t db_lookup_hash = HSHashDb(pd_cached);
         if (HSLoadCache(&pd_cached->hs_db, db_lookup_hash) == 0) {
@@ -759,7 +759,7 @@ static int PatternDatabaseCompile(PatternDatabase *pd, SCHSCompileData *cd)
  *
  * \param mpm_ctx Pointer to the mpm context.
  */
-int SCHSPreparePatterns(MpmCtx *mpm_ctx)
+int SCHSPreparePatterns(MpmCtx *mpm_ctx, bool cache_to_disk)
 {
     SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx;
 
@@ -774,6 +774,7 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx)
     }
 
     HSPatternArrayInit(ctx, pd);
+    pd->no_cache = cache_to_disk ? false : true;
     /* Serialise whole database compilation as a relatively easy way to ensure
      * dedupe is safe. */
     SCMutexLock(&g_db_table_mutex);
@@ -782,7 +783,7 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx)
         goto error;
     }
 
-    if (PatternDatabaseGetCached(&pd, cd) == 0 && pd != NULL) {
+    if (PatternDatabaseGetCached(&pd, cd, cache_to_disk) == 0 && pd != NULL) {
         ctx->pattern_db = pd;
         if (PatternDatabaseGetSize(pd, &ctx->hs_db_size) != 0) {
             SCMutexUnlock(&g_db_table_mutex);
@@ -835,9 +836,9 @@ static int SCHSCacheRuleset(void)
     SCMutexLock(&g_db_table_mutex);
     HashTableIterate(g_db_table, HSSaveCacheIterator, &pd_stats);
     SCMutexUnlock(&g_db_table_mutex);
-    SCLogInfo("%u rule groups cached (%u newly cached) of total %u groups",
+    SCLogInfo("%u rule groups cached (%u newly cached) of total %u cacheable groups",
             pd_stats.hs_dbs_cache_loaded_cnt + pd_stats.hs_dbs_cache_saved_cnt,
-            pd_stats.hs_dbs_cache_saved_cnt, pd_stats.hs_dbs_cnt);
+            pd_stats.hs_dbs_cache_saved_cnt, pd_stats.hs_cacheable_dbs_cnt);
     return 0;
 }
 
@@ -1207,7 +1208,7 @@ static int SCHSTest01(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
@@ -1241,7 +1242,7 @@ static int SCHSTest02(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
@@ -1278,7 +1279,7 @@ static int SCHSTest03(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
@@ -1312,7 +1313,7 @@ static int SCHSTest04(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
@@ -1346,7 +1347,7 @@ static int SCHSTest05(void)
     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
@@ -1378,7 +1379,7 @@ static int SCHSTest06(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcd";
@@ -1422,7 +1423,7 @@ static int SCHSTest07(void)
                     0, 0, 5, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
@@ -1455,7 +1456,7 @@ static int SCHSTest08(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     uint32_t cnt =
@@ -1487,7 +1488,7 @@ static int SCHSTest09(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     uint32_t cnt =
@@ -1519,7 +1520,7 @@ static int SCHSTest10(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "01234567890123456789012345678901234567890123456789"
@@ -1562,7 +1563,7 @@ static int SCHSTest11(void)
         goto end;
     PmqSetup(&pmq);
 
-    if (SCHSPreparePatterns(&mpm_ctx) == -1)
+    if (SCHSPreparePatterns(&mpm_ctx, false) == -1)
         goto end;
 
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
@@ -1606,7 +1607,7 @@ static int SCHSTest12(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyz";
@@ -1640,7 +1641,7 @@ static int SCHSTest13(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABCD";
@@ -1674,7 +1675,7 @@ static int SCHSTest14(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE";
@@ -1708,7 +1709,7 @@ static int SCHSTest15(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF";
@@ -1742,7 +1743,7 @@ static int SCHSTest16(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzABC";
@@ -1776,7 +1777,7 @@ static int SCHSTest17(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyzAB";
@@ -1815,7 +1816,7 @@ static int SCHSTest18(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcde"
@@ -1854,7 +1855,7 @@ static int SCHSTest19(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
@@ -1894,7 +1895,7 @@ static int SCHSTest20(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, sizeof(pat) - 1, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "AAAAA"
@@ -1933,7 +1934,7 @@ static int SCHSTest21(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     uint32_t cnt =
@@ -1967,7 +1968,7 @@ static int SCHSTest22(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "abcdefghijklmnopqrstuvwxyz";
@@ -2000,7 +2001,7 @@ static int SCHSTest23(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     uint32_t cnt =
@@ -2032,7 +2033,7 @@ static int SCHSTest24(void)
     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     uint32_t cnt =
@@ -2065,7 +2066,7 @@ static int SCHSTest25(void)
     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
@@ -2098,7 +2099,7 @@ static int SCHSTest26(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "works";
@@ -2131,7 +2132,7 @@ static int SCHSTest27(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "tone";
@@ -2164,7 +2165,7 @@ static int SCHSTest28(void)
     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0);
     PmqSetup(&pmq);
 
-    SCHSPreparePatterns(&mpm_ctx);
+    SCHSPreparePatterns(&mpm_ctx, false);
     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
 
     const char *buf = "tONE";
diff --git a/src/util-mpm.h b/src/util-mpm.h
index 5df4bc0c8de2..c5db7184a389 100644
--- a/src/util-mpm.h
+++ b/src/util-mpm.h
@@ -163,7 +163,7 @@ typedef struct MpmTableElmt_ {
      */
     int  (*AddPattern)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t);
     int  (*AddPatternNocase)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t);
-    int  (*Prepare)(struct MpmCtx_ *);
+    int (*Prepare)(struct MpmCtx_ *, bool);
     int (*CacheRuleset)(void);
     /** \retval cnt number of patterns that matches: once per pattern max. */
     uint32_t (*Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t);