util-mpm: prepare MPM codebase for ruleset caching

OISF · Jan 13, 2025 · 9b1c453 · 9b1c453
1 parent e684b4f
commit 9b1c453
Show file tree

Hide file tree

Showing 15 changed files with 158 additions and 107 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -37,6 +37,7 @@ install-conf:
 	install -d "$(DESTDIR)$(e_rundir)"
 	install -m 770 -d "$(DESTDIR)$(e_localstatedir)"
 	install -m 770 -d "$(DESTDIR)$(e_datadir)"
+	install -m 660 -d "$(DESTDIR)$(e_sghcachedir)"
 
 install-rules:
 if INSTALL_SURICATA_UPDATE

diff --git a/configure.ac b/configure.ac
@@ -2455,6 +2455,7 @@ if test "$WINDOWS_PATH" = "yes"; then
 
     e_sysconfdir="${e_winbase}\\\\"
     e_defaultruledir="$e_winbase\\\\rules\\\\"
+    e_sghcachedir="$e_winbase\\\\cache\\\\sgh\\\\"
     e_magic_file="$e_winbase\\\\magic.mgc"
     e_logdir="$e_winbase\\\\log"
     e_logfilesdir="$e_logdir\\\\files"
@@ -2476,6 +2477,7 @@ else
     EXPAND_VARIABLE(sysconfdir, e_sysconfdir, "/suricata/")
     EXPAND_VARIABLE(localstatedir, e_localstatedir, "/run/suricata")
     EXPAND_VARIABLE(datadir, e_datarulesdir, "/suricata/rules")
+    EXPAND_VARIABLE(localstatedir, e_sghcachedir, "/lib/suricata/cache/sgh")
     EXPAND_VARIABLE(localstatedir, e_datadir, "/lib/suricata/data")
     EXPAND_VARIABLE(localstatedir, e_defaultruledir, "/lib/suricata/rules")
 
@@ -2489,6 +2491,8 @@ AC_SUBST(e_logcertsdir)
 AC_SUBST(e_sysconfdir)
 AC_DEFINE_UNQUOTED([CONFIG_DIR],["$e_sysconfdir"],[Our CONFIG_DIR])
 AC_SUBST(e_localstatedir)
+AC_SUBST(e_sghcachedir)
+AC_DEFINE_UNQUOTED([SGH_CACHE_DIR],["$e_sghcachedir"],[Directory path for signature group head cache])
 AC_SUBST(e_datadir)
 AC_DEFINE_UNQUOTED([DATA_DIR],["$e_datadir"],[Our DATA_DIR])
 AC_SUBST(e_magic_file)

diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c
@@ -1297,7 +1297,7 @@ static int AppLayerProtoDetectPMPrepareMpm(AppLayerProtoDetectPMCtx *ctx)
     int ret = 0;
     MpmCtx *mpm_ctx = &ctx->mpm_ctx;
 
-    if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx) < 0)
+    if (mpm_table[mpm_ctx->mpm_type].Prepare(mpm_ctx, false) < 0)
         goto error;
 
     goto end;

diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c
@@ -1296,8 +1296,7 @@ static void FTPSetMpmState(void)
                        i /*  id */, i /* rule id */ , 0 /* no flags */);
     }
 
-    mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx);
-
+    mpm_table[FTP_MPM].Prepare(ftp_mpm_ctx, false);
 }
 
 static void FTPFreeMpmState(void)

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
@@ -1636,7 +1636,7 @@ static void SMTPSetMpmState(void)
                         i /* pattern id */, i /* rule id */ , 0 /* no flags */);
     }
 
-    mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx);
+    mpm_table[SMTP_MPM].Prepare(smtp_mpm_ctx, false);
 }
 
 static void SMTPFreeMpmState(void)

diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c
@@ -33,6 +33,7 @@
 #include "tm-threads.h"
 #include "queue.h"
 
+#include "detect-engine.h"
 #include "detect-engine-loader.h"
 #include "detect-engine-build.h"
 #include "detect-engine-analyzer.h"
@@ -402,6 +403,10 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, bool sig_file_exc
 
     ret = 0;
 
+    if (de_ctx->mpm_cache_to_disk && mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) {
+        mpm_table[de_ctx->mpm_matcher].CacheRuleset();
+    }
+
  end:
     gettimeofday(&de_ctx->last_reload, NULL);
     if (SCRunmodeGet() == RUNMODE_ENGINE_ANALYSIS) {

diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
@@ -295,7 +295,7 @@ int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
             MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
                 }
             }
         }
@@ -524,7 +524,7 @@ int DetectMpmPrepareFrameMpms(DetectEngineCtx *de_ctx)
             SCLogDebug("%s: %d mpm_Ctx %p", am->name, r, mpm_ctx);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
                     SCLogDebug("%s: %d", am->name, r);
                 }
             }
@@ -689,7 +689,7 @@ int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx)
             MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, 0);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
                     SCLogDebug("%s: %d", am->name, r);
                 }
             }
@@ -744,40 +744,40 @@ int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
     if (de_ctx->sgh_mpm_context_proto_tcp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 1);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
     if (de_ctx->sgh_mpm_context_proto_udp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 1);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
     if (de_ctx->sgh_mpm_context_proto_other_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_other_packet, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
     if (de_ctx->sgh_mpm_context_stream != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 0);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
         mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 1);
         if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx, de_ctx->mpm_cache_to_disk);
         }
     }
 
@@ -1621,7 +1621,7 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
     } else {
         if (ms->sgh_mpm_context == MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
             if (mpm_table[ms->mpm_ctx->mpm_type].Prepare != NULL) {
-                mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx);
+                mpm_table[ms->mpm_ctx->mpm_type].Prepare(ms->mpm_ctx, de_ctx->mpm_cache_to_disk);
             }
         }
     }

diff --git a/src/detect-engine.c b/src/detect-engine.c
@@ -2481,6 +2481,35 @@ static int DetectEngineReloadThreads(DetectEngineCtx *new_de_ctx)
     return -1;
 }
 
+static bool DetectEngineMpmCachingEnabled(void)
+{
+    const char *strval = NULL;
+    if (ConfGet("detect.sgh-mpm-caching", &strval) != 1)
+        return false;
+
+    int sgh_mpm_caching = 0;
+    (void)ConfGetBool("detect.sgh-mpm-caching", &sgh_mpm_caching);
+    return (bool)sgh_mpm_caching;
+}
+
+const char *DetectEngineMpmCachingGetPath(void)
+{
+    char yamlpath[] = "detect.sgh-mpm-caching-path";
+    const char *strval = NULL;
+    ConfGet(yamlpath, &strval);
+
+    if (strval != NULL) {
+        return strval;
+    }
+
+    static bool notified = false;
+    if (!notified) {
+        SCLogInfo("%s has no path specified, using %s", yamlpath, SGH_CACHE_DIR);
+        notified = true;
+    }
+    return SGH_CACHE_DIR;
+}
+
 static DetectEngineCtx *DetectEngineCtxInitReal(
         enum DetectEngineType type, const char *prefix, uint32_t tenant_id)
 {
@@ -2512,6 +2541,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal(
     de_ctx->failure_fatal = (failure_fatal == 1);
 
     de_ctx->mpm_matcher = PatternMatchDefaultMatcher();
+    de_ctx->mpm_cache_to_disk = DetectEngineMpmCachingEnabled();
     de_ctx->spm_matcher = SinglePatternMatchDefaultMatcher();
     SCLogConfig("pattern matchers: MPM: %s, SPM: %s",
         mpm_table[de_ctx->mpm_matcher].name,

diff --git a/src/detect-engine.h b/src/detect-engine.h
@@ -100,6 +100,7 @@ void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, i
 
 TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **);
 TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
+const char *DetectEngineMpmCachingGetPath(void);
 /* faster as a macro than a inline function on my box -- VJ */
 #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum)
 void DetectEngineResetMaxSigId(DetectEngineCtx *);

diff --git a/src/detect.h b/src/detect.h
@@ -1050,6 +1050,9 @@ typedef struct DetectEngineCtx_ {
 
     /* number of signatures using filestore, limited as u16 */
     uint16_t filestore_cnt;
+
+    /** If enabled, MPM matchers can store compiled pattern databases to disk */
+    bool mpm_cache_to_disk;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */