From bdf73495dec75715a13a26616e7a218fc2aad668 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 16 Oct 2024 14:55:17 +0000 Subject: [PATCH 01/15] chore(release): 3.32.0 # [3.32.0](https://github.com/splunk/splunk-connect-for-syslog/compare/v3.31.0...v3.32.0) (2024-10-16) ### Bug Fixes * fix GDB installation ([#2615](https://github.com/splunk/splunk-connect-for-syslog/issues/2615)) ([44d6433](https://github.com/splunk/splunk-connect-for-syslog/commit/44d6433c799f62a90ce82f7cf64df8d956c37e75)) * remove goss from healthcheck ([#2600](https://github.com/splunk/splunk-connect-for-syslog/issues/2600)) ([b6ac66b](https://github.com/splunk/splunk-connect-for-syslog/commit/b6ac66b05a822c804bf802d1e3588898fd120b10)) ### Features * Adds config checksums to Helm chart to trigger rollout on change ([#2551](https://github.com/splunk/splunk-connect-for-syslog/issues/2551)) ([3a18d00](https://github.com/splunk/splunk-connect-for-syslog/commit/3a18d006e9dc709556e18078bc2fb10e951ff25d)) --- charts/splunk-connect-for-syslog/Chart.yaml | 4 ++-- package/etc/VERSION | 2 +- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/splunk-connect-for-syslog/Chart.yaml b/charts/splunk-connect-for-syslog/Chart.yaml index 0c832a81cb..2a0337c186 100644 --- a/charts/splunk-connect-for-syslog/Chart.yaml +++ b/charts/splunk-connect-for-syslog/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: splunk-connect-for-syslog description: Deploy Splunk Connect for Syslog type: application -version: 3.31.0 -appVersion: "3.31.0" +version: 3.32.0 +appVersion: "3.32.0" diff --git a/package/etc/VERSION b/package/etc/VERSION index d9351e5882..cac6068cfc 100644 --- a/package/etc/VERSION +++ b/package/etc/VERSION @@ -1 +1 @@ -3.31.0 +3.32.0 diff --git a/pyproject.toml b/pyproject.toml index ed44759638..39ddfafda3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "splunk-connect-for-syslog" -version = "3.31.0" +version = "3.32.0" description = "" authors = ["rjha-splunk "] license = "Apache-2.0" From 96f6cc604252c2828246194bbe704332ffbba9ad Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:28:36 +0200 Subject: [PATCH 02/15] chore(deps): update dependency mkdocs-material to v9.5.42 (#2624) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index b057aa3a3c..def4dd7cec 100644 --- a/poetry.lock +++ b/poetry.lock @@ -680,13 +680,13 @@ mkdocs = ">=1.0.4" [[package]] name = "mkdocs-material" -version = "9.5.41" +version = "9.5.42" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.41-py3-none-any.whl", hash = "sha256:990bc138c33342b5b73e7545915ebc0136e501bfbd8e365735144f5120891d83"}, - {file = "mkdocs_material-9.5.41.tar.gz", hash = "sha256:30fa5d459b4b8130848ecd8e1c908878345d9d8268f7ddbc31eebe88d462d97b"}, + {file = "mkdocs_material-9.5.42-py3-none-any.whl", hash = "sha256:452a7c5d21284b373f36b981a2cbebfff59263feebeede1bc28652e9c5bbe316"}, + {file = "mkdocs_material-9.5.42.tar.gz", hash = "sha256:92779b5e9b5934540c574c11647131d217dc540dce72b05feeda088c8eb1b8f2"}, ] [package.dependencies] From f625806924c1e0259115425da06ad1308e297478 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:29:23 +0200 Subject: [PATCH 03/15] chore(deps): update splunk/addonfactory-test-matrix-action action to v2.1.9 (#2620) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/ci-lite.yaml | 2 +- .github/workflows/ci-main.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 96cc4e7b29..5ebcf2532f 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -86,7 +86,7 @@ jobs: type=ref,event=tag - name: matrix id: matrix - uses: splunk/addonfactory-test-matrix-action@v2.1.8 + uses: splunk/addonfactory-test-matrix-action@v2.1.9 security-fossa-scan: continue-on-error: true diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 9d02ba3fc7..0a44116cd7 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -86,7 +86,7 @@ jobs: type=ref,event=tag - name: matrix id: matrix - uses: splunk/addonfactory-test-matrix-action@v2.1.8 + uses: splunk/addonfactory-test-matrix-action@v2.1.9 security-fossa-scan: continue-on-error: true From c85df6841dc1256b30b21b8606b81898470fa08b Mon Sep 17 00:00:00 2001 From: wojtekzyla <108660584+wojtekzyla@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:31:52 +0200 Subject: [PATCH 04/15] fix: fix CISE_Alarm messages parsing (#2609) --- .../post-filter/app-postfilter-cisco_ise.conf | 3 +- .../conflib/syslog/app-syslog-cisco_ise.conf | 29 ++++- .../cisco/app-postfilter-cisco_ise.conf | 3 +- .../addons/cisco/app-syslog-cisco_ise.conf | 29 ++++- tests/test_cisco_ise.py | 101 +++++++++++++++++- 5 files changed, 154 insertions(+), 11 deletions(-) diff --git a/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf b/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf index d47f3ce445..640705d906 100644 --- a/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf +++ b/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf @@ -57,7 +57,8 @@ block parser app-postfilter-cisco_ise() { application app-postfilter-cisco_ise[sc4s-finalfilter] { filter { program('CISE_' type(string) flags(prefix)) - and "${.values.num}" != 1; + and "${.values.num}" != 1 + and not program('CISE_Alarm'); }; parser { app-postfilter-cisco_ise(); }; }; diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf b/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf index d4f2ff2079..ebdb9136bd 100644 --- a/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf +++ b/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf @@ -18,6 +18,27 @@ parser ise_event_time { block parser app-syslog-cisco_ise() { channel { + + if { + parser { + regexp-parser( + template("${MESSAGE}") + patterns("^(?\\d{2}) (?\\d{2}:\\d{2}:\\d{2}) (?[^ ]+) (?[^ ]+) (?.*)") + prefix(".parsed.") + ); + + date-parser-nofilter( + format('%b %d %H:%M:%S') + template("${PROGRAM} ${.parsed.real_day} ${.parsed.real_hour}") + ); + }; + rewrite { + set("${.parsed.real_host}" value("HOST")); + set("${.parsed.real_program}" value("PROGRAM")); + set("${.parsed.rest_of_message}" value("MESSAGE")); + }; + }; + parser { csv-parser( columns(serial, num, seq, message) @@ -44,13 +65,13 @@ block parser app-syslog-cisco_ise() { product('ise') ); }; - - - }; + }; }; + application app-syslog-cisco_ise[sc4s-syslog-pgm] { filter { - program('CISE_' type(string) flags(prefix)); + program('CISE_' type(string) flags(prefix)) + or message('CISE_' type(string) flags(substring)); }; parser { app-syslog-cisco_ise(); }; }; diff --git a/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf b/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf index d47f3ce445..640705d906 100644 --- a/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf +++ b/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf @@ -57,7 +57,8 @@ block parser app-postfilter-cisco_ise() { application app-postfilter-cisco_ise[sc4s-finalfilter] { filter { program('CISE_' type(string) flags(prefix)) - and "${.values.num}" != 1; + and "${.values.num}" != 1 + and not program('CISE_Alarm'); }; parser { app-postfilter-cisco_ise(); }; }; diff --git a/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf b/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf index d4f2ff2079..ebdb9136bd 100644 --- a/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf +++ b/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf @@ -18,6 +18,27 @@ parser ise_event_time { block parser app-syslog-cisco_ise() { channel { + + if { + parser { + regexp-parser( + template("${MESSAGE}") + patterns("^(?\\d{2}) (?\\d{2}:\\d{2}:\\d{2}) (?[^ ]+) (?[^ ]+) (?.*)") + prefix(".parsed.") + ); + + date-parser-nofilter( + format('%b %d %H:%M:%S') + template("${PROGRAM} ${.parsed.real_day} ${.parsed.real_hour}") + ); + }; + rewrite { + set("${.parsed.real_host}" value("HOST")); + set("${.parsed.real_program}" value("PROGRAM")); + set("${.parsed.rest_of_message}" value("MESSAGE")); + }; + }; + parser { csv-parser( columns(serial, num, seq, message) @@ -44,13 +65,13 @@ block parser app-syslog-cisco_ise() { product('ise') ); }; - - - }; + }; }; + application app-syslog-cisco_ise[sc4s-syslog-pgm] { filter { - program('CISE_' type(string) flags(prefix)); + program('CISE_' type(string) flags(prefix)) + or message('CISE_' type(string) flags(substring)); }; parser { app-syslog-cisco_ise(); }; }; diff --git a/tests/test_cisco_ise.py b/tests/test_cisco_ise.py index 8958efb963..2b4ee45819 100644 --- a/tests/test_cisco_ise.py +++ b/tests/test_cisco_ise.py @@ -207,7 +207,7 @@ def test_cisco_ise_cise_alarm_single( sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" "Server=10.0.0.5"' + 'search index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" "CISE_Alarm WARN: RADIUS Authentication Request dropped : Server=10.0.0.5;"' ) search = st.render(epoch=epoch, host=host) @@ -218,3 +218,102 @@ def test_cisco_ise_cise_alarm_single( record_property("message", message) assert result_count == 1 + +@pytest.mark.addons("cisco") +def test_cisco_ise_double_timestamp_and_hostname( + record_property, setup_splunk, setup_sc4s +): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, time, date, tzoffset, _, epoch = time_operations(dt) + + # Tune time functions for Cisco ISE + time = time[:-3] + tzoffset = tzoffset[0:3] + ":" + tzoffset[3:] + epoch = epoch[:-3] + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 1 4 2020-01-01 10:00:00.000000 +00:00 0000015291 70501 NOTICE System-Stats: ISE Counters, ConfigVersionId=1, OperationCounters=Counter=1_LocalEndPointReads:1]\n" + ) + message = mt.render( + mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" "CISE_System_Statistics: 0000001313 1 4 2020-01-01 10:00:00.000000"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + +@pytest.mark.addons("cisco") +def test_cisco_ise_double_timestamp_and_hostname_sequence_eq_0( + record_property, setup_splunk, setup_sc4s +): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, time, date, tzoffset, _, epoch = time_operations(dt) + + # Tune time functions for Cisco ISE + time = time[:-3] + tzoffset = tzoffset[0:3] + ":" + tzoffset[3:] + epoch = epoch[:-3] + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 0 {{ date }} {{ time }} {{ tzoffset }} 0000015291 70501 NOTICE System-Stats: part one,\n" + ) + + message = mt.render( + mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + # Generate new datetime for subsequent messages; not used in log path parser so actually could be anything + dt = datetime.datetime.now() + datetime.timedelta(seconds=1) + bsd = dt.strftime("%b %d %H:%M:%S") + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 1 part two,\n" + ) + message = mt.render( + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 2 part three,\n" + ) + message = mt.render( + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 3 part four,\n" + ) + message = mt.render( + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" one two three four' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 \ No newline at end of file From ede18827d1773d3400ad8ff3ffa905ce78798b65 Mon Sep 17 00:00:00 2001 From: mstopa-splunk <139441697+mstopa-splunk@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:10:12 +0200 Subject: [PATCH 05/15] fix: improve SC4S Dashboard performance (#2592) --- dashboard/dashboard.xml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/dashboard/dashboard.xml b/dashboard/dashboard.xml index 88382ff68f..d17d725e04 100644 --- a/dashboard/dashboard.xml +++ b/dashboard/dashboard.xml @@ -18,7 +18,7 @@ - index=* sc4s_container=$sc4s_instance$ + | tstats count where index=* sc4s_container=$sc4s_instance$ by index _time $time_range.earliest$ $time_range.latest$ @@ -43,8 +43,8 @@ - rt-15m - rt + -15m + now @@ -220,6 +220,7 @@ + @@ -310,7 +311,7 @@ Total volume of actual syslog traffic delivered by this SC4S instance to Splunk - | stats count + | stats sum(count) @@ -318,6 +319,7 @@ + @@ -336,7 +338,7 @@ Distributions of events by index - | stats count by index + | stats sum(count) as count by index @@ -366,6 +368,7 @@ + @@ -375,7 +378,7 @@ Trends of events by index - | chart sparkline(count) AS "Indexes Trend" count AS Total BY index + | stats sparkline(sum(count)) as "Indexes Trend" sum(count) as Total by index @@ -393,7 +396,7 @@ - index=* sc4s_container=$sc4s_instance$ | eval tags=split(sc4s_tags,"|") | mvexpand tags | search tags=".app.*" | timechart count by tags + | tstats count where index=* sc4s_container=$sc4s_instance$ by sc4s_tags _time | eval tags=split(sc4s_tags,"|") | mvexpand tags | search tags=".app.*" | timechart sum(count) by tags $time_range.earliest$ $time_range.latest$ @@ -439,7 +442,7 @@
- index=* sc4s_container=$sc4s_instance$ | eval tags=split(sc4s_tags,"|") | mvexpand tags | chart count by tags + | tstats count where index=* sc4s_container=$sc4s_instance$ by sc4s_tags _time | eval tags=split(sc4s_tags,"|") | mvexpand tags | stats sum(count) as eventCount by tags | sort - eventCount $time_range.earliest$ $time_range.latest$ @@ -449,4 +452,4 @@
- \ No newline at end of file + From d56e7f95faa34664adf515f679c393d955417bd8 Mon Sep 17 00:00:00 2001 From: cwadhwani-splunk Date: Tue, 29 Oct 2024 16:33:26 +0530 Subject: [PATCH 06/15] docs: Removed reference of Cisco eStreamer for Splunk app from ASA/FTD doc (#2629) * docs: Removed reference of Cisco eStreamer for Splunk app --- docs/sources/vendor/Cisco/cisco_asa.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sources/vendor/Cisco/cisco_asa.md b/docs/sources/vendor/Cisco/cisco_asa.md index adf03eab8c..8cafb37184 100644 --- a/docs/sources/vendor/Cisco/cisco_asa.md +++ b/docs/sources/vendor/Cisco/cisco_asa.md @@ -11,7 +11,6 @@ | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| | Splunk Add-on for ASA (No long supports FWSM and PIX) | | -| Cisco eStreamer for Splunk | | | Product Manual | | ## Sourcetypes From 4b198d48de53c7eb15bb7444473961d9958a21cb Mon Sep 17 00:00:00 2001 From: mstopa-splunk <139441697+mstopa-splunk@users.noreply.github.com> Date: Thu, 14 Nov 2024 13:30:38 +0100 Subject: [PATCH 07/15] fix: fix sqlitedict CVE (#2621) --- .github/workflows/ci-lite.yaml | 10 ++++++- .github/workflows/ci-main.yaml | 10 ++++++- .trivyignore | 2 ++ package/Dockerfile | 2 +- package/Dockerfile.lite | 2 +- package/etc/pylib/parser_source_cache.py | 14 +++++---- package/etc/pylib/parser_vps_cache.py | 10 ++++--- package/etc/pylib/sqlite_utils.py | 28 +++++++++++++++++ tests/test_name_cache.py | 38 +++++++++++++++++++++++- 9 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 .trivyignore create mode 100644 package/etc/pylib/sqlite_utils.py diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 5ebcf2532f..df174ada6a 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -161,6 +161,12 @@ jobs: - meta - build_action steps: + # To use .trivyignore file, you must check out the repository + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false - name: Run docker vulnerability scanner uses: aquasecurity/trivy-action@master with: @@ -168,7 +174,9 @@ jobs: format: 'table' exit-code: '1' severity: 'CRITICAL,HIGH,MEDIUM,LOW' - + trivyignores: '.trivyignore' + scanners: "vuln" + test-container: runs-on: ubuntu-latest needs: diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 0a44116cd7..d56ab76959 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -161,6 +161,12 @@ jobs: - meta - build_action steps: + # To use .trivyignore file, you must check out the repository + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false - name: Run docker vulnerability scanner uses: aquasecurity/trivy-action@master with: @@ -168,7 +174,9 @@ jobs: format: 'table' exit-code: '1' severity: 'CRITICAL,HIGH,MEDIUM,LOW' - + trivyignores: '.trivyignore' + scanners: "vuln" + test-container: runs-on: ubuntu-latest needs: diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 0000000000..0a7be8250b --- /dev/null +++ b/.trivyignore @@ -0,0 +1,2 @@ +# This has been safeguarded directly in the code +CVE-2024-35515 \ No newline at end of file diff --git a/package/Dockerfile b/package/Dockerfile index c91e3bc53a..b730d652d6 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -28,7 +28,7 @@ RUN apk add -U --upgrade --no-cache \ less \ net-tools \ netcat-openbsd \ - openssl \ + "openssl>=3.3.2-r1" \ procps \ py3-pip \ python3 \ diff --git a/package/Dockerfile.lite b/package/Dockerfile.lite index 24f30686d2..e54ceee047 100644 --- a/package/Dockerfile.lite +++ b/package/Dockerfile.lite @@ -28,7 +28,7 @@ RUN apk add -U --upgrade --no-cache \ less \ net-tools \ netcat-openbsd \ - openssl \ + "openssl>=3.3.2-r1" \ procps \ py3-pip \ python3 \ diff --git a/package/etc/pylib/parser_source_cache.py b/package/etc/pylib/parser_source_cache.py index dc573a3449..1ffa3c114e 100644 --- a/package/etc/pylib/parser_source_cache.py +++ b/package/etc/pylib/parser_source_cache.py @@ -2,7 +2,6 @@ import traceback import socket import struct -from sqlitedict import SqliteDict import time @@ -17,7 +16,6 @@ class LogParser: class LogDestination: pass - def ip2int(addr): ip4_to_int = lambda addr: struct.unpack("!I", socket.inet_aton(addr))[0] @@ -53,8 +51,10 @@ def int_to_ip6(num): class psc_parse(LogParser): def init(self, options): + from sqlite_utils import RestrictedSqliteDict + self.logger = syslogng.Logger() - self.db = SqliteDict(f"{hostdict}.sqlite") + self.db = RestrictedSqliteDict(f"{hostdict}.sqlite") return True def deinit(self): @@ -80,9 +80,11 @@ def parse(self, log_message): class psc_dest(LogDestination): def init(self, options): + from sqlite_utils import RestrictedSqliteDict + self.logger = syslogng.Logger() try: - self.db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) + self.db = RestrictedSqliteDict(f"{hostdict}.sqlite", autocommit=True) except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) @@ -123,7 +125,9 @@ def flush(self): if __name__ == "__main__": - db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) + from sqlite_utils import RestrictedSqliteDict + + db = RestrictedSqliteDict(f"{hostdict}.sqlite", autocommit=True) db[0] = "seed" db.commit() db.close() \ No newline at end of file diff --git a/package/etc/pylib/parser_vps_cache.py b/package/etc/pylib/parser_vps_cache.py index a95162862b..61575c1f8c 100644 --- a/package/etc/pylib/parser_vps_cache.py +++ b/package/etc/pylib/parser_vps_cache.py @@ -2,7 +2,6 @@ import traceback import socket import struct -from sqlitedict import SqliteDict import time @@ -17,14 +16,15 @@ class LogParser: class LogDestination: pass - hostdict = str("/var/lib/syslog-ng/vps") class vpsc_parse(LogParser): def init(self, options): + from sqlite_utils import RestrictedSqliteDict + self.logger = syslogng.Logger() - self.db = SqliteDict(f"{hostdict}.sqlite") + self.db = RestrictedSqliteDict(f"{hostdict}.sqlite") return True def deinit(self): @@ -50,9 +50,11 @@ def parse(self, log_message): class vpsc_dest(LogDestination): def init(self, options): + from sqlite_utils import RestrictedSqliteDict + self.logger = syslogng.Logger() try: - self.db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) + self.db = RestrictedSqliteDict(f"{hostdict}.sqlite", autocommit=True) except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) diff --git a/package/etc/pylib/sqlite_utils.py b/package/etc/pylib/sqlite_utils.py new file mode 100644 index 0000000000..fa16ace292 --- /dev/null +++ b/package/etc/pylib/sqlite_utils.py @@ -0,0 +1,28 @@ +import io +import pickle +from base64 import b64decode +from sqlitedict import SqliteDict + + +class RestrictedUnpickler(pickle.Unpickler): + def find_class(self, module, name): + """Override pickle.Unpickler.find_class() to prevent deserialization of class instances.""" + raise pickle.UnpicklingError("Class deserialization is disabled") + + +def restricted_loads(s): + """Helper function analogous to pickle.loads().""" + return RestrictedUnpickler(io.BytesIO(s)).load() + +def restricted_decode(obj): + """Overwrite sqlitedict.decode() to prevent code injection.""" + return restricted_loads(bytes(obj)) + +def restricted_decode_key(key): + """Overwrite sqlitedict.decode_key() to prevent code injection.""" + return restricted_loads(b64decode(key.encode("ascii"))) + + +class RestrictedSqliteDict(SqliteDict): + def __init__(self, *args, **kwargs): + super(RestrictedSqliteDict, self).__init__(*args, decode=restricted_decode, decode_key=restricted_decode_key, **kwargs) \ No newline at end of file diff --git a/tests/test_name_cache.py b/tests/test_name_cache.py index ce8a56f502..21ac8ad2c5 100644 --- a/tests/test_name_cache.py +++ b/tests/test_name_cache.py @@ -5,8 +5,10 @@ # https://opensource.org/licenses/BSD-2-Clause import datetime +import pickle import random import re +import tempfile import time from jinja2 import Environment @@ -16,6 +18,7 @@ from .sendmessage import sendsingle from .splunkutils import splunk_single from package.etc.pylib.parser_source_cache import ip2int, int2ip +from package.etc.pylib.sqlite_utils import RestrictedSqliteDict env = Environment() @@ -73,4 +76,37 @@ def test_ipv4_utils(): @pytest.mark.name_cache def test_ipv6_utils(): ip = generate_random_ipv6() - assert ip == int2ip(ip2int(ip)) \ No newline at end of file + assert ip == int2ip(ip2int(ip)) + +@pytest.mark.name_cache +def test_RestrictedSqliteDict_stores_and_retrieves_string(): + with tempfile.NamedTemporaryFile(delete=True) as temp_db_file: + cache = RestrictedSqliteDict(f"{temp_db_file.name}.db") + cache["key"] = "value" + cache.commit() + cache.close() + + cache = RestrictedSqliteDict(f"{temp_db_file.name}.db") + assert cache["key"] == "value" + cache.close() + +@pytest.mark.name_cache +def test_RestrictedSqliteDict_prevents_code_injection(): + class InjectionTestClass: + def __reduce__(self): + import os + return os.system, ('touch pwned.txt',) + + with tempfile.NamedTemporaryFile(delete=True) as temp_db_file: + # Initialize the RestrictedSqliteDict and insert an 'injected' object + cache = RestrictedSqliteDict(f"{temp_db_file.name}.db") + cache["key"] = InjectionTestClass() + cache.commit() + cache.close() + + # Re-open cache and attempt to deserialize 'injected' object + # Expecting UnpicklingError due to RestrictedSqliteDict restrictions + cache = RestrictedSqliteDict(f"{temp_db_file.name}.db") + with pytest.raises(pickle.UnpicklingError): + _ = cache["key"] + cache.close() \ No newline at end of file From ed79c9b9d60155969240b70e4ef83b04370c054c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 14 Nov 2024 13:39:09 +0100 Subject: [PATCH 08/15] chore(deps): update dependency mkdocs-material to v9.5.44 (#2633) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index def4dd7cec..bf251502c9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -680,13 +680,13 @@ mkdocs = ">=1.0.4" [[package]] name = "mkdocs-material" -version = "9.5.42" +version = "9.5.44" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.42-py3-none-any.whl", hash = "sha256:452a7c5d21284b373f36b981a2cbebfff59263feebeede1bc28652e9c5bbe316"}, - {file = "mkdocs_material-9.5.42.tar.gz", hash = "sha256:92779b5e9b5934540c574c11647131d217dc540dce72b05feeda088c8eb1b8f2"}, + {file = "mkdocs_material-9.5.44-py3-none-any.whl", hash = "sha256:47015f9c167d58a5ff5e682da37441fc4d66a1c79334bfc08d774763cacf69ca"}, + {file = "mkdocs_material-9.5.44.tar.gz", hash = "sha256:f3a6c968e524166b3f3ed1fb97d3ed3e0091183b0545cedf7156a2a6804c56c0"}, ] [package.dependencies] From e97cf5f4db0dd758c300da54b61b83af5ef0edc6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 14 Nov 2024 13:39:49 +0100 Subject: [PATCH 09/15] chore(deps): update dependency splunk-sdk to v2.1.0 (#2632) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index bf251502c9..8167cbce42 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1215,12 +1215,12 @@ files = [ [[package]] name = "splunk-sdk" -version = "2.0.2" +version = "2.1.0" description = "The Splunk Software Development Kit for Python." optional = false python-versions = "*" files = [ - {file = "splunk-sdk-2.0.2.tar.gz", hash = "sha256:d5ccf6e1b96e493b1399e071ef10f8337e0a39ac2b2acc73fdd375b87b4104cb"}, + {file = "splunk-sdk-2.1.0.tar.gz", hash = "sha256:63f9a259a7c84d0c3b0b32cae652365b03f0f926acdb894b51456005df74ae21"}, ] [package.dependencies] From 833cecf627ec564eec40c3b691ac0757eec42b65 Mon Sep 17 00:00:00 2001 From: nitronarcosis Date: Thu, 14 Nov 2024 04:43:51 -0800 Subject: [PATCH 10/15] docs: update avamar.md (#2625) --- docs/sources/vendor/Dell/avamar.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/vendor/Dell/avamar.md b/docs/sources/vendor/Dell/avamar.md index 0abaf3b97a..e2a5f661f0 100644 --- a/docs/sources/vendor/Dell/avamar.md +++ b/docs/sources/vendor/Dell/avamar.md @@ -22,4 +22,4 @@ | key | sourcetype | index | notes | |----------------|----------------|----------------|----------------| -| dell_avamar_cms| dell:avamar:msc| netops | none | \ No newline at end of file +| dell_avamar_msc| dell:avamar:msc| netops | none | From 6da306096bf9b951f3c97d59a1c65e108721a282 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 14 Nov 2024 14:21:01 +0000 Subject: [PATCH 11/15] chore(release): 3.32.1 ## [3.32.1](https://github.com/splunk/splunk-connect-for-syslog/compare/v3.32.0...v3.32.1) (2024-11-14) ### Bug Fixes * fix CISE_Alarm messages parsing ([#2609](https://github.com/splunk/splunk-connect-for-syslog/issues/2609)) ([c85df68](https://github.com/splunk/splunk-connect-for-syslog/commit/c85df6841dc1256b30b21b8606b81898470fa08b)) * fix sqlitedict CVE ([#2621](https://github.com/splunk/splunk-connect-for-syslog/issues/2621)) ([4b198d4](https://github.com/splunk/splunk-connect-for-syslog/commit/4b198d48de53c7eb15bb7444473961d9958a21cb)) * improve SC4S Dashboard performance ([#2592](https://github.com/splunk/splunk-connect-for-syslog/issues/2592)) ([ede1882](https://github.com/splunk/splunk-connect-for-syslog/commit/ede18827d1773d3400ad8ff3ffa905ce78798b65)) --- charts/splunk-connect-for-syslog/Chart.yaml | 4 ++-- package/etc/VERSION | 2 +- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/splunk-connect-for-syslog/Chart.yaml b/charts/splunk-connect-for-syslog/Chart.yaml index 2a0337c186..fa8ba5b485 100644 --- a/charts/splunk-connect-for-syslog/Chart.yaml +++ b/charts/splunk-connect-for-syslog/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: splunk-connect-for-syslog description: Deploy Splunk Connect for Syslog type: application -version: 3.32.0 -appVersion: "3.32.0" +version: 3.32.1 +appVersion: "3.32.1" diff --git a/package/etc/VERSION b/package/etc/VERSION index cac6068cfc..1367bf750d 100644 --- a/package/etc/VERSION +++ b/package/etc/VERSION @@ -1 +1 @@ -3.32.0 +3.32.1 diff --git a/pyproject.toml b/pyproject.toml index 39ddfafda3..795669f53a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "splunk-connect-for-syslog" -version = "3.32.0" +version = "3.32.1" description = "" authors = ["rjha-splunk "] license = "Apache-2.0" From 15ed2ce994173fa4b9256089965ba7678bc6f7a7 Mon Sep 17 00:00:00 2001 From: cwadhwani-splunk Date: Tue, 10 Dec 2024 18:14:56 +0530 Subject: [PATCH 12/15] feat: Release 3.33 (#2654) * fix: Non pri cef fix (#2641) * chore(deps): update dependency mkdocs-material to v9.5.42 (#2624) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * chore(deps): update splunk/addonfactory-test-matrix-action action to v2.1.9 (#2620) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix: fix CISE_Alarm messages parsing (#2609) * fix: improve SC4S Dashboard performance (#2592) * docs: Removed reference of Cisco eStreamer for Splunk app from ASA/FTD doc (#2629) * docs: Removed reference of Cisco eStreamer for Splunk app * fix: Updated the regex for non pri cef formated logs such that a 'digit' date will be accepted * Updated the regex with some more constraints --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: wojtekzyla <108660584+wojtekzyla@users.noreply.github.com> Co-authored-by: mstopa-splunk <139441697+mstopa-splunk@users.noreply.github.com> * chore(deps): update dependency mkdocs-material to v9.5.47 (#2644) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * feat: Netapp ontap audit ems support (#2639) * chore(deps): update dependency mkdocs-material to v9.5.42 (#2624) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * chore(deps): update splunk/addonfactory-test-matrix-action action to v2.1.9 (#2620) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix: fix CISE_Alarm messages parsing (#2609) * fix: improve SC4S Dashboard performance (#2592) * docs: Removed reference of Cisco eStreamer for Splunk app from ASA/FTD doc (#2629) * docs: Removed reference of Cisco eStreamer for Splunk app * feat: Added support for ems logs and fixed the existing classification * Updated the documentation and made some changes in the parser * Updating the test file such that all the test cases are passing * Added support in sc4s lite * docs: Added the migration precaution in the upgrade.md file * Removed an addionally created test file and merged my changes to the existing one * Updated the code to maintain the backward compatibility * Updated the test-container workflow to set the newly introduced environment variable's value to 'yes' and made some changes in the test vps parser. --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: wojtekzyla <108660584+wojtekzyla@users.noreply.github.com> Co-authored-by: mstopa-splunk <139441697+mstopa-splunk@users.noreply.github.com> * Added juniper log parsing * Migrated to restricted sqlite --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: wojtekzyla <108660584+wojtekzyla@users.noreply.github.com> Co-authored-by: mstopa-splunk <139441697+mstopa-splunk@users.noreply.github.com> Co-authored-by: sbylica-splunk Co-authored-by: Szymon Bylica <142112966+sbylica-splunk@users.noreply.github.com> --- .github/workflows/ci-lite.yaml | 1 + .github/workflows/ci-main.yaml | 1 + docs/sources/vendor/NetApp/ontap.md | 44 ++++++++++++++- docs/upgrade.md | 3 + .../netsource/app-netsource-netapp_ontap.conf | 55 +++++++++++++++++++ .../conf.d/conflib/raw/app-raw-bsd_nopri.conf | 2 +- ...app-syslog-juniper_junos_unstructured.conf | 1 + .../syslog/app-syslog-netapp_ontap.conf | 3 +- package/etc/pylib/parser_source_cache.py | 22 +++----- package/etc/pylib/parser_vps_cache.py | 12 ++-- package/etc/pylib/psc_dump.py | 5 +- package/etc/pylib/sqlite_utils.py | 28 ---------- .../app-vps-test-netapp_ontap.conf | 15 +++++ ...app-syslog-juniper_junos_unstructured.conf | 1 + .../netapp/app-netsource-netapp_ontap.conf | 55 +++++++++++++++++++ .../netapp/app-syslog-netapp_ontap.conf | 3 +- .../conf.d/conflib/raw/app-raw-bsd_nopri.conf | 2 +- poetry.lock | 31 ++++++----- pyproject.toml | 2 +- tests/test_juniper_junos_rfc3164.py | 54 ++++++++++++++---- tests/test_name_cache.py | 31 +++++++---- tests/test_netapp.py | 47 ++++++++++++++-- tests/test_trellix.py | 34 ++++++++++++ 23 files changed, 348 insertions(+), 104 deletions(-) create mode 100644 package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf delete mode 100644 package/etc/pylib/sqlite_utils.py create mode 100644 package/etc/test_parsers/app-vps-test-netapp_ontap.conf create mode 100644 package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index df174ada6a..e895969c0b 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -227,6 +227,7 @@ jobs: TEST_SC4S_ACTIVATE_EXAMPLES: "yes" SC4S_DEBUG_CONTAINER: "yes" SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" + SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" SC4S_USE_VPS_CACHE: "yes" steps: - name: Checkout diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index d56ab76959..3fb2f70ad8 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -227,6 +227,7 @@ jobs: TEST_SC4S_ACTIVATE_EXAMPLES: "yes" SC4S_DEBUG_CONTAINER: "yes" SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" + SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" SC4S_USE_VPS_CACHE: "yes" steps: diff --git a/docs/sources/vendor/NetApp/ontap.md b/docs/sources/vendor/NetApp/ontap.md index 2eacc9c6a7..0ead82cf82 100644 --- a/docs/sources/vendor/NetApp/ontap.md +++ b/docs/sources/vendor/NetApp/ontap.md @@ -3,7 +3,7 @@ ## Key facts * MSG Format based filter -* Legacy BSD Format default port 514 +* Netapp Ontap messages are not distinctive. So, either configure known Netapp Ontap hosts in SC4S, or open unique ports for Netapp Ontap devices ## Links @@ -16,11 +16,49 @@ | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| netapp:ems | None | +| ontap:ems | This sourcetype will be assinged only when the environment variable `SC4S_NETAPP_ONTAP_NEW_FORMAT` is not set or is set to 'no'. By default it is unset | +| netapp:ontap:audit | This sourcetype will be assinged only when the environment variable `SC4S_NETAPP_ONTAP_NEW_FORMAT` is set to 'yes' | +| netapp:ontap:ems | This sourcetype will be assinged only when the environment variable `SC4S_NETAPP_ONTAP_NEW_FORMAT` is set to 'yes' | ## Sourcetype and Index Configuration | key | sourcetype | index | notes | |----------------|----------------|----------------|----------------| -| netapp_ontap | netapp:ems | infraops | none | +| netapp_ontap | ontap:ems | infraops | none | +| netapp_ontap_audit | netapp:ontap:audit | infraops | none | +| netapp_ontap_ems | netapp:ontap:ems | infraops | none | +## Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_NETAPP_ONTAP_NEW_FORMAT | empty string | (empty/yes) Set to "yes" for the applying the latest changes. Make sure to configure your system to send the logs to a specific port or have a hostname-based configuration | + +## Parser Configuration +1. Through sc4s-vps +```c +#/opt/sc4s/local/config/app-parsers/app-vps-netapp_ontap.conf +#File name provided is a suggestion it must be globally unique + +application app-vps-test-netapp_ontap[sc4s-vps] { + filter { + host("netapp-ontap-" type(string) flags(prefix)) + or ( + message("netapp-ontap-" type(string) flags(prefix)) + and program("netapp-ontap-" type(string) flags(prefix)) + ) + }; + parser { + p_set_netsource_fields( + vendor('netapp') + product('ontap') + ); + }; +}; +``` + +2. or through unique port +``` +# /opt/sc4s/env_file +SC4S_LISTEN_NETAPP_ONTAP_UDP_PORT=5005 +``` \ No newline at end of file diff --git a/docs/upgrade.md b/docs/upgrade.md index e17d2c0e76..813c4f1824 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -18,6 +18,9 @@ For a step by step guide [see here](./v3_upgrade.md). You may need to migrate legacy log paths or version 1 app-parsers for version 2. To do this, open an issue and attach the original configuration and a compressed pcap of sample data for testing. We will evaluate whether to include the source in an upcoming release. +### Upgrade from <3.33.0 +In NetApp ONTAP, the ontap:ems sourcetype has been updated to netapp:ontap:audit, so old logs are now classified under netapp:ontap:audit. Additionally, a new netapp:ontap:ems sourcetype has been introduced. If you upgrade and want these new changes, ensure that you set `SC4S_NETAPP_ONTAP_NEW_FORMAT` environment variable to `yes` and configure your system to send the logs to a specific port or have a hostname-based configuration in place for proper log onboarding into Splunk. + ### Upgrade from <2.23.0 * In VMware vSphere, update the ESX and vCenter sourcetype for add-on compatibility. diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf b/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf new file mode 100644 index 0000000000..08cd806164 --- /dev/null +++ b/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf @@ -0,0 +1,55 @@ +block parser app-netsource-netapp_ontap() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + vendor("netapp") + product("ontap") + ); + }; + + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^[A-Za-z0-9\-\_\.]+: [0-9a-f]+\.[0-9a-f]+ [0-9a-f]+ [A-Z][a-z][a-z] (?[A-Z][a-z][a-z] \d\d \d\d\d\d \d\d:\d\d:\d\d [+-]?\d{1,2}:\d\d)') + ); + date-parser-nofilter( + format( + '%b %d %Y %H:%M:%S %z', + ) + template("${.tmp.timestamp}") + ); + }; + + rewrite { + set('$PROGRAM: $MESSAGE', value(MESSAGE)); + set('$PROGRAM', value(HOST)); + unset(value(PROGRAM)); + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:audit') + class('audit') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:ems') + class('ems') + ); + }; + }; + }; +}; + +application app-netsource-netapp_ontap[sc4s-network-source] { + filter { + match("netapp", value('.netsource.sc4s_vendor'), type(string)) + and match("ontap", value('.netsource.sc4s_product'), type(string)) + and "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes" + }; + parser { app-netsource-netapp_ontap(); }; +}; diff --git a/package/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf b/package/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf index fc0a6c679b..7f66031700 100644 --- a/package/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf +++ b/package/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf @@ -53,7 +53,7 @@ block parser app-raw-bsd_nopri() { }; application app-raw-bsd_nopri[sc4s-raw-syslog] { filter { - message('^\w\w\w \d\d \d\d:\d\d:\d\d '); + message('^\w{3} ([0 ][1-9]|[12]\d|3[01]) (0\d|1\d|2[0-3]):([0-5]\d):([0-5]\d) '); }; parser { app-raw-bsd_nopri(); }; }; diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf b/package/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf index a229e46e28..f1a71f5076 100644 --- a/package/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf +++ b/package/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf @@ -74,6 +74,7 @@ application app-syslog-juniper_junos_unstructured-pgm[sc4s-syslog-pgm] { or program('RT_FLOW' type(string) flags(prefix)) or program('RT_IDS' type(string) flags(prefix)) or program('RT_UTM' type(string) flags(prefix)) + or program('RT_SYSTEM' type(string) flags(prefix)) or program('Juniper' type(string) flags(prefix)) or program('rpd' type(string) flags(prefix)) or program('eswd' type(string) flags(prefix)) diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf b/package/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf index 84db3a994c..46b37a1b0b 100644 --- a/package/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf +++ b/package/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf @@ -37,7 +37,8 @@ block parser app-syslog-netapp_ontap() { }; application app-syslog-netapp_ontap[sc4s-syslog] { filter { - program('^[A-Za-z0-9\-\_\.]+$'); + program('^[A-Za-z0-9\-\_\.]+$') + and not "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes"; }; parser { app-syslog-netapp_ontap(); }; }; diff --git a/package/etc/pylib/parser_source_cache.py b/package/etc/pylib/parser_source_cache.py index 1ffa3c114e..1a8bcbca2a 100644 --- a/package/etc/pylib/parser_source_cache.py +++ b/package/etc/pylib/parser_source_cache.py @@ -2,6 +2,7 @@ import traceback import socket import struct +from sqlitedict import SqliteDict import time @@ -16,14 +17,15 @@ class LogParser: class LogDestination: pass + def ip2int(addr): ip4_to_int = lambda addr: struct.unpack("!I", socket.inet_aton(addr))[0] - + def ip6_to_int(addr): ip6 = socket.inet_pton(socket.AF_INET6, addr) a, b = struct.unpack(">QQ", ip6) return (a << 64) | b - + try: return ip4_to_int(addr) except OSError: @@ -39,7 +41,7 @@ def int_to_ip6(num): ip6 = struct.pack(">QQ", a, b) addr = socket.inet_ntop(socket.AF_INET6, ip6) return addr - + try: return int_to_ip4(addr) except struct.error: @@ -51,10 +53,8 @@ def int_to_ip6(num): class psc_parse(LogParser): def init(self, options): - from sqlite_utils import RestrictedSqliteDict - self.logger = syslogng.Logger() - self.db = RestrictedSqliteDict(f"{hostdict}.sqlite") + self.db = SqliteDict(f"{hostdict}.sqlite") return True def deinit(self): @@ -80,11 +80,9 @@ def parse(self, log_message): class psc_dest(LogDestination): def init(self, options): - from sqlite_utils import RestrictedSqliteDict - self.logger = syslogng.Logger() try: - self.db = RestrictedSqliteDict(f"{hostdict}.sqlite", autocommit=True) + self.db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) @@ -125,9 +123,7 @@ def flush(self): if __name__ == "__main__": - from sqlite_utils import RestrictedSqliteDict - - db = RestrictedSqliteDict(f"{hostdict}.sqlite", autocommit=True) + db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) db[0] = "seed" db.commit() - db.close() \ No newline at end of file + db.close() diff --git a/package/etc/pylib/parser_vps_cache.py b/package/etc/pylib/parser_vps_cache.py index 61575c1f8c..4c8cf21250 100644 --- a/package/etc/pylib/parser_vps_cache.py +++ b/package/etc/pylib/parser_vps_cache.py @@ -2,6 +2,7 @@ import traceback import socket import struct +from sqlitedict import SqliteDict import time @@ -16,15 +17,14 @@ class LogParser: class LogDestination: pass + hostdict = str("/var/lib/syslog-ng/vps") class vpsc_parse(LogParser): def init(self, options): - from sqlite_utils import RestrictedSqliteDict - self.logger = syslogng.Logger() - self.db = RestrictedSqliteDict(f"{hostdict}.sqlite") + self.db = SqliteDict(f"{hostdict}.sqlite") return True def deinit(self): @@ -50,11 +50,9 @@ def parse(self, log_message): class vpsc_dest(LogDestination): def init(self, options): - from sqlite_utils import RestrictedSqliteDict - self.logger = syslogng.Logger() try: - self.db = RestrictedSqliteDict(f"{hostdict}.sqlite", autocommit=True) + self.db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) @@ -100,4 +98,4 @@ def flush(self): if __name__ == "__main__": - pass \ No newline at end of file + pass diff --git a/package/etc/pylib/psc_dump.py b/package/etc/pylib/psc_dump.py index 927ce6be92..eccbe1e650 100644 --- a/package/etc/pylib/psc_dump.py +++ b/package/etc/pylib/psc_dump.py @@ -1,4 +1,3 @@ - import sys import traceback import socket @@ -9,5 +8,5 @@ hostdict = str("/var/lib/syslog-ng/cache/hostip") db = SqliteDict(f"{hostdict}.sqlite") -for k,v in db.items(): - print(f"key={k}={v}") \ No newline at end of file +for k, v in db.items(): + print(f"key={k}={v}") diff --git a/package/etc/pylib/sqlite_utils.py b/package/etc/pylib/sqlite_utils.py deleted file mode 100644 index fa16ace292..0000000000 --- a/package/etc/pylib/sqlite_utils.py +++ /dev/null @@ -1,28 +0,0 @@ -import io -import pickle -from base64 import b64decode -from sqlitedict import SqliteDict - - -class RestrictedUnpickler(pickle.Unpickler): - def find_class(self, module, name): - """Override pickle.Unpickler.find_class() to prevent deserialization of class instances.""" - raise pickle.UnpicklingError("Class deserialization is disabled") - - -def restricted_loads(s): - """Helper function analogous to pickle.loads().""" - return RestrictedUnpickler(io.BytesIO(s)).load() - -def restricted_decode(obj): - """Overwrite sqlitedict.decode() to prevent code injection.""" - return restricted_loads(bytes(obj)) - -def restricted_decode_key(key): - """Overwrite sqlitedict.decode_key() to prevent code injection.""" - return restricted_loads(b64decode(key.encode("ascii"))) - - -class RestrictedSqliteDict(SqliteDict): - def __init__(self, *args, **kwargs): - super(RestrictedSqliteDict, self).__init__(*args, decode=restricted_decode, decode_key=restricted_decode_key, **kwargs) \ No newline at end of file diff --git a/package/etc/test_parsers/app-vps-test-netapp_ontap.conf b/package/etc/test_parsers/app-vps-test-netapp_ontap.conf new file mode 100644 index 0000000000..24b14274e0 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-netapp_ontap.conf @@ -0,0 +1,15 @@ +application app-vps-test-netapp_ontap[sc4s-vps] { + filter { + host("netapp-ontap-" type(string) flags(prefix)) + or ( + message("netapp-ontap-" type(string) flags(prefix)) + and program("netapp-ontap-" type(string) flags(prefix)) + ) + }; + parser { + p_set_netsource_fields( + vendor('netapp') + product('ontap') + ); + }; +}; \ No newline at end of file diff --git a/package/lite/etc/addons/juniper/app-syslog-juniper_junos_unstructured.conf b/package/lite/etc/addons/juniper/app-syslog-juniper_junos_unstructured.conf index a229e46e28..f1a71f5076 100644 --- a/package/lite/etc/addons/juniper/app-syslog-juniper_junos_unstructured.conf +++ b/package/lite/etc/addons/juniper/app-syslog-juniper_junos_unstructured.conf @@ -74,6 +74,7 @@ application app-syslog-juniper_junos_unstructured-pgm[sc4s-syslog-pgm] { or program('RT_FLOW' type(string) flags(prefix)) or program('RT_IDS' type(string) flags(prefix)) or program('RT_UTM' type(string) flags(prefix)) + or program('RT_SYSTEM' type(string) flags(prefix)) or program('Juniper' type(string) flags(prefix)) or program('rpd' type(string) flags(prefix)) or program('eswd' type(string) flags(prefix)) diff --git a/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf b/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf new file mode 100644 index 0000000000..08cd806164 --- /dev/null +++ b/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf @@ -0,0 +1,55 @@ +block parser app-netsource-netapp_ontap() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + vendor("netapp") + product("ontap") + ); + }; + + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^[A-Za-z0-9\-\_\.]+: [0-9a-f]+\.[0-9a-f]+ [0-9a-f]+ [A-Z][a-z][a-z] (?[A-Z][a-z][a-z] \d\d \d\d\d\d \d\d:\d\d:\d\d [+-]?\d{1,2}:\d\d)') + ); + date-parser-nofilter( + format( + '%b %d %Y %H:%M:%S %z', + ) + template("${.tmp.timestamp}") + ); + }; + + rewrite { + set('$PROGRAM: $MESSAGE', value(MESSAGE)); + set('$PROGRAM', value(HOST)); + unset(value(PROGRAM)); + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:audit') + class('audit') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:ems') + class('ems') + ); + }; + }; + }; +}; + +application app-netsource-netapp_ontap[sc4s-network-source] { + filter { + match("netapp", value('.netsource.sc4s_vendor'), type(string)) + and match("ontap", value('.netsource.sc4s_product'), type(string)) + and "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes" + }; + parser { app-netsource-netapp_ontap(); }; +}; diff --git a/package/lite/etc/addons/netapp/app-syslog-netapp_ontap.conf b/package/lite/etc/addons/netapp/app-syslog-netapp_ontap.conf index 84db3a994c..46b37a1b0b 100644 --- a/package/lite/etc/addons/netapp/app-syslog-netapp_ontap.conf +++ b/package/lite/etc/addons/netapp/app-syslog-netapp_ontap.conf @@ -37,7 +37,8 @@ block parser app-syslog-netapp_ontap() { }; application app-syslog-netapp_ontap[sc4s-syslog] { filter { - program('^[A-Za-z0-9\-\_\.]+$'); + program('^[A-Za-z0-9\-\_\.]+$') + and not "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes"; }; parser { app-syslog-netapp_ontap(); }; }; diff --git a/package/lite/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf b/package/lite/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf index fc0a6c679b..7f66031700 100644 --- a/package/lite/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf +++ b/package/lite/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf @@ -53,7 +53,7 @@ block parser app-raw-bsd_nopri() { }; application app-raw-bsd_nopri[sc4s-raw-syslog] { filter { - message('^\w\w\w \d\d \d\d:\d\d:\d\d '); + message('^\w{3} ([0 ][1-9]|[12]\d|3[01]) (0\d|1\d|2[0-3]):([0-5]\d):([0-5]\d) '); }; parser { app-raw-bsd_nopri(); }; }; diff --git a/poetry.lock b/poetry.lock index 8167cbce42..c379f0fc3b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.9.0.dev0 and should not be changed by hand. [[package]] name = "arrow" @@ -680,13 +680,13 @@ mkdocs = ">=1.0.4" [[package]] name = "mkdocs-material" -version = "9.5.44" +version = "9.5.47" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.44-py3-none-any.whl", hash = "sha256:47015f9c167d58a5ff5e682da37441fc4d66a1c79334bfc08d774763cacf69ca"}, - {file = "mkdocs_material-9.5.44.tar.gz", hash = "sha256:f3a6c968e524166b3f3ed1fb97d3ed3e0091183b0545cedf7156a2a6804c56c0"}, + {file = "mkdocs_material-9.5.47-py3-none-any.whl", hash = "sha256:53fb9c9624e7865da6ec807d116cd7be24b3cb36ab31b1d1d1a9af58c56009a2"}, + {file = "mkdocs_material-9.5.47.tar.gz", hash = "sha256:fc3b7a8e00ad896660bd3a5cc12ca0cb28bdc2bcbe2a946b5714c23ac91b0ede"}, ] [package.dependencies] @@ -1175,6 +1175,17 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "restricted-sqlitedict" +version = "1.0.0" +description = "Fork of sqlitedict with restricted pickle loading." +optional = false +python-versions = "*" +files = [ + {file = "restricted_sqlitedict-1.0.0-py3-none-any.whl", hash = "sha256:31cdeac0f48ee16cdbfeb8e8c14e8c1a17ebb67e6f098f5724a46515c2d2d3c6"}, + {file = "restricted_sqlitedict-1.0.0.tar.gz", hash = "sha256:f4c319ca51a01b37b2350f418640a9d80061402d58a7df7fcb1e0147b5d9cf13"}, +] + [[package]] name = "setuptools" version = "73.0.1" @@ -1226,16 +1237,6 @@ files = [ [package.dependencies] deprecation = "*" -[[package]] -name = "sqlitedict" -version = "2.1.0" -description = "Persistent dict in Python, backed up by sqlite3 and pickle, multithread-safe." -optional = false -python-versions = "*" -files = [ - {file = "sqlitedict-2.1.0.tar.gz", hash = "sha256:03d9cfb96d602996f1d4c2db2856f1224b96a9c431bdd16e78032a72940f9e8c"}, -] - [[package]] name = "tomli" version = "2.0.1" @@ -1384,4 +1385,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "0caa7b7eb023b55566f6cc063d7b2a5daf4a3ff78deaabd9dadb2c625f3e12bf" +content-hash = "15a805f6379fa28a3b310fc492b3f9d468945660693413c9c0c7e54c86841475" diff --git a/pyproject.toml b/pyproject.toml index 795669f53a..0f2c9e54c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,11 @@ license = "Apache-2.0" [tool.poetry.dependencies] python = "^3.9" Jinja2 = "^3.1.3" -sqlitedict = "^2.0.0" requests = "^2.28.1" shortuuid = "^1.0.11" pyyaml = "6.0.2" setuptools = "^73.0.1" +restricted-sqlitedict = "^1.0.0" [tool.poetry.group.dev.dependencies] diff --git a/tests/test_juniper_junos_rfc3164.py b/tests/test_juniper_junos_rfc3164.py index 16b2642fb0..98ce4424fa 100644 --- a/tests/test_juniper_junos_rfc3164.py +++ b/tests/test_juniper_junos_rfc3164.py @@ -6,7 +6,7 @@ from jinja2 import Environment, select_autoescape from .sendmessage import sendsingle -from .splunkutils import splunk_single +from .splunkutils import splunk_single from .timeutils import time_operations import datetime import pytest @@ -15,9 +15,7 @@ # <23> Mar 18 17:56:52 RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION="URL Permitted" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY="Enhanced_Information_Technology" REASON="BY_PRE_DEFINED" PROFILE="UTM-Wireless-Profile" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A @pytest.mark.addons("juniper") -def test_juniper_utm_standard( - record_property, get_host_key, setup_splunk, setup_sc4s -): +def test_juniper_utm_standard(record_property, get_host_key, setup_splunk, setup_sc4s): host = get_host_key dt = datetime.datetime.now() @@ -50,7 +48,7 @@ def test_juniper_utm_standard( # <23> Nov 18 09:56:58 INTERNET-ROUTER RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192 @pytest.mark.addons("juniper") def test_juniper_firewall_standard( - record_property, get_host_key, setup_splunk, setup_sc4s + record_property, get_host_key, setup_splunk, setup_sc4s ): host = get_host_key @@ -82,9 +80,7 @@ def test_juniper_firewall_standard( @pytest.mark.addons("juniper") -def test_juniper_idp_standard( - record_property, get_host_key, setup_splunk, setup_sc4s -): +def test_juniper_idp_standard(record_property, get_host_key, setup_splunk, setup_sc4s): host = get_host_key dt = datetime.datetime.now() @@ -123,7 +119,7 @@ def test_juniper_idp_standard( @pytest.mark.addons("juniper") @pytest.mark.parametrize("event", testdata_junos_snmp) def test_juniper_junos_snmp( - record_property, get_host_key, setup_splunk, setup_sc4s, event + record_property, get_host_key, setup_splunk, setup_sc4s, event ): host = get_host_key @@ -160,7 +156,7 @@ def test_juniper_junos_snmp( @pytest.mark.addons("juniper") @pytest.mark.parametrize("event", testdata_junos_firewall_switch) def test_juniper_junos_switch( - record_property, get_host_key, setup_splunk, setup_sc4s, event + record_property, get_host_key, setup_splunk, setup_sc4s, event ): host = get_host_key @@ -198,7 +194,7 @@ def test_juniper_junos_switch( @pytest.mark.parametrize("event", testdata_junos_firewall_router) @pytest.mark.addons("juniper") def test_juniper_junos_router( - record_property, get_host_key, setup_splunk, setup_sc4s, event + record_property, get_host_key, setup_splunk, setup_sc4s, event ): host = get_host_key @@ -235,7 +231,7 @@ def test_juniper_junos_router( @pytest.mark.addons("juniper") @pytest.mark.parametrize("event", testdata_junos_switch_rpd) def test_juniper_junos_switch_rpd( - record_property, get_host_key, setup_splunk, setup_sc4s, event + record_property, get_host_key, setup_splunk, setup_sc4s, event ): host = get_host_key @@ -263,3 +259,37 @@ def test_juniper_junos_switch_rpd( record_property("message", message) assert result_count == 1 + + +# <161>Mar 18 17:56:52 host RT_SYSTEM: RTLOG_CONN_ERROR: Connection error tcp_10.181.123.45 Error code: major 3 minor 1 code 110, description:TCP timed out after SYN is sent out +@pytest.mark.addons("juniper") +def test_juniper_system_standard( + record_property, get_host_key, setup_splunk, setup_sc4s +): + host = get_host_key + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} RT_SYSTEM: RTLOG_CONN_ERROR: Connection error tcp_10.181.123.45 Error code: major 3 minor 1 code 110, description:TCP timed out after SYN is sent out " + ) + message = mt.render(mark="<161>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="juniper:legacy"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 diff --git a/tests/test_name_cache.py b/tests/test_name_cache.py index 21ac8ad2c5..fb3a55865f 100644 --- a/tests/test_name_cache.py +++ b/tests/test_name_cache.py @@ -16,9 +16,9 @@ from .timeutils import time_operations from .sendmessage import sendsingle -from .splunkutils import splunk_single +from .splunkutils import splunk_single from package.etc.pylib.parser_source_cache import ip2int, int2ip -from package.etc.pylib.sqlite_utils import RestrictedSqliteDict +from sqlitedict import SqliteDict env = Environment() @@ -48,7 +48,7 @@ def test_name_cache(get_host_key, setup_splunk, setup_sc4s): _ = send_message(template_no_host, setup_sc4s) _ = send_message(template_with_host, setup_sc4s, host=get_host_key) - time.sleep(1) # time to save the new cache entry + time.sleep(1) # time to save the new cache entry epoch = send_message(template_no_host, setup_sc4s) search = f'search _time="{epoch}" index=* host="{get_host_key}"' @@ -61,52 +61,59 @@ def generate_random_ipv4(): random_octet = lambda: format(random.randint(0, 255)) return ".".join([random_octet() for _ in range(4)]) + def generate_random_ipv6(): def generate_random_hex(): - random_hex = format(random.randint(0, 65535), '04x') - random_hex = re.sub('^0+', '', random_hex) # leading zeros can be skipped + random_hex = format(random.randint(0, 65535), "04x") + random_hex = re.sub("^0+", "", random_hex) # leading zeros can be skipped return random_hex + return ":".join([generate_random_hex() for _ in range(8)]) + @pytest.mark.name_cache def test_ipv4_utils(): ip = generate_random_ipv4() assert ip == int2ip(ip2int(ip)) + @pytest.mark.name_cache def test_ipv6_utils(): ip = generate_random_ipv6() assert ip == int2ip(ip2int(ip)) + @pytest.mark.name_cache def test_RestrictedSqliteDict_stores_and_retrieves_string(): with tempfile.NamedTemporaryFile(delete=True) as temp_db_file: - cache = RestrictedSqliteDict(f"{temp_db_file.name}.db") + cache = SqliteDict(f"{temp_db_file.name}.db") cache["key"] = "value" cache.commit() cache.close() - cache = RestrictedSqliteDict(f"{temp_db_file.name}.db") + cache = SqliteDict(f"{temp_db_file.name}.db") assert cache["key"] == "value" cache.close() + @pytest.mark.name_cache def test_RestrictedSqliteDict_prevents_code_injection(): class InjectionTestClass: def __reduce__(self): import os - return os.system, ('touch pwned.txt',) - + + return os.system, ("touch pwned.txt",) + with tempfile.NamedTemporaryFile(delete=True) as temp_db_file: # Initialize the RestrictedSqliteDict and insert an 'injected' object - cache = RestrictedSqliteDict(f"{temp_db_file.name}.db") + cache = SqliteDict(f"{temp_db_file.name}.db") cache["key"] = InjectionTestClass() cache.commit() cache.close() # Re-open cache and attempt to deserialize 'injected' object # Expecting UnpicklingError due to RestrictedSqliteDict restrictions - cache = RestrictedSqliteDict(f"{temp_db_file.name}.db") + cache = SqliteDict(f"{temp_db_file.name}.db") with pytest.raises(pickle.UnpicklingError): _ = cache["key"] - cache.close() \ No newline at end of file + cache.close() diff --git a/tests/test_netapp.py b/tests/test_netapp.py index 07bf82c533..6451deac06 100644 --- a/tests/test_netapp.py +++ b/tests/test_netapp.py @@ -17,16 +17,16 @@ testdata = [ - "{{ mark }}{{ bsd }} {{ host }}: {{ host }}: 0000001e.0794c163 055b6737 {{ device_time }} [kern_audit:info:2385] 8503ea0000ba6b71 :: nodea:ontapi :: 10.10.10.10:41464 :: nodea-esx:usera :: clone-create :: Error: Missing input: source-path; Missing input: volume", + "{{ mark }}{{ bsd }} {{ host }}: {{ host }}: 00000030.00c8f1e2 11e5347f {{ device_time }} [kern_audit3167] 8004b7000021e73b:4005f7000021e73d :: cluster:ssh :: 0.0.0.0:32879 :: cluster:admin :: qos statistics volume performance show -rows 20 -iter 1 :: Pending", ] - +# <14>Oct 3 11:36:46 host: host: 00000030.00c8f1e2 11e5347f Thu Oct 03 2024 11:36:44 -06:00 [kern_audit3167] 8004b7000021e73b:4005f7000021e73d :: cluster:ssh :: 0.0.0.0:32879 :: cluster:admin :: qos statistics volume performance show -rows 20 -iter 1 :: Pending @pytest.mark.addons("netapp") @pytest.mark.parametrize("event", testdata) -def test_netapp( +def test_netapp_ontap_audit( record_property, get_host_key, setup_splunk, setup_sc4s, event ): - host = get_host_key + host = "netapp-ontap-" + get_host_key dt = datetime.datetime.now(datetime.timezone.utc) _, bsd, _, _, _, _, epoch = time_operations(dt) @@ -37,12 +37,12 @@ def test_netapp( device_time = dt.strftime("%a %b %d %Y %H:%M:%S +00:00") mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", bsd=bsd, host=host, device_time=device_time) + message = mt.render(mark="<14>", bsd=bsd, host=host, device_time=device_time) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=infraops _time={{ epoch }} sourcetype="ontap:ems" (host="{{ host }}" OR "{{ host }}")' + 'search index=infraops _time={{ epoch }} sourcetype="netapp:ontap:audit"' ) search = st.render(epoch=epoch, host=host) @@ -53,3 +53,38 @@ def test_netapp( record_property("message", message) assert result_count == 1 + + +# Netapp Ontap EMS event in rfc5424 format +# <5>1 2024-10-03T07:54:02-06:00 host program - wafl.scan.done - Completed Volume Footprint Estimator Scan on volume vm_unix002_0d@vserver:27902083bf98-11e9-87fe-00a098b15eb6 +@pytest.mark.addons("netapp") +def test_netapp_ontap_ems_rfc5424( + record_property, get_host_key, setup_splunk, setup_sc4s +): + host = "netapp-ontap-" + get_host_key + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, _, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-3] + + mt = env.from_string( + '{{ mark }} {{ iso }} {{ host }} program - wafl.scan.done - Completed Volume Footprint Estimator Scan on volume vm_unix002_0d@vserver:27902083bf98-11e9-87fe-00a098b15eb6' + ) + message = mt.render(mark="<5>1", iso=iso, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=infraops sourcetype="netapp:ontap:ems"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 \ No newline at end of file diff --git a/tests/test_trellix.py b/tests/test_trellix.py index 4f0a3dc968..c461e9ea0d 100644 --- a/tests/test_trellix.py +++ b/tests/test_trellix.py @@ -67,3 +67,37 @@ def test_trellix_cms( record_property("message", message) assert result_count == 1 + + +# Nov 2 00:00:00 trellix-host-xxxx CEF:0|Trellix|Database Security|0.0.0|alert|DML Queries executed from the backend|3|externalId=53 rt=1030482923264 cs1=Finacle Alert_DC_54 cs1Label=DBMS dst=10.10.10.10 src=10.10.10.11 duser=TESTUSER suser=testuser shost=KBPXXXUD00xx6 dproc=sxxxplus@KBPXXXUD00xx6 (TNS V1-V3) act=DELETE cs2=DELETE FROM TESTUSER.XXXX WHERE XXXX.ALERT_REFERENCE_NO \= :B1 cs2Label=SqlStatement cs3=XXXX|ALERT_HISTORY_TBL cs3Label=AccessedObjects. +@pytest.mark.addons("trellix") +def test_trellix_cef(record_property, get_host_key, setup_splunk, setup_sc4s): + host = "trellix-host-" + get_host_key + + dt = datetime.datetime(2024, 11, 2, 0, 0) + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # hard coding the bsd to test single digit date (Nov 2 00:00:00) + bsd = "Nov 2 00:00:00" + # Tune time functions + epoch = epoch[:-3] + + mt = env.from_string( + "{{ bsd }} {{ host }} CEF:0|Trellix|Database Security|0.0.0|alert|DML Queries executed from the backend|3|externalId=53 rt={{ epoch }} cs1=Finacle Alert_DC_54 cs1Label=DBMS dst=10.10.10.10 src=10.10.10.11 duser=TESTUSER suser=testuser shost=KBPXXXUD00xx6 dproc=sxxxplus@KBPXXXUD00xx6 (TNS V1-V3) act=DELETE cs2=DELETE FROM TESTUSER.XXXX WHERE XXXX.ALERT_REFERENCE_NO \= :B1 cs2Label=SqlStatement cs3=XXXX|ALERT_HISTORY_TBL cs3Label=AccessedObjects." + ) + message = mt.render(bsd=bsd, host=host, epoch=epoch) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="cef"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 \ No newline at end of file From 586ba400488c7c1c9350c49e91dfb14fda1ba350 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 10 Dec 2024 13:00:25 +0000 Subject: [PATCH 13/15] chore(release): 3.33.0 # [3.33.0](https://github.com/splunk/splunk-connect-for-syslog/compare/v3.32.1...v3.33.0) (2024-12-10) ### Features * Release 3.33 ([#2654](https://github.com/splunk/splunk-connect-for-syslog/issues/2654)) ([15ed2ce](https://github.com/splunk/splunk-connect-for-syslog/commit/15ed2ce994173fa4b9256089965ba7678bc6f7a7)), closes [#2641](https://github.com/splunk/splunk-connect-for-syslog/issues/2641) [#2624](https://github.com/splunk/splunk-connect-for-syslog/issues/2624) [#2620](https://github.com/splunk/splunk-connect-for-syslog/issues/2620) [#2609](https://github.com/splunk/splunk-connect-for-syslog/issues/2609) [#2592](https://github.com/splunk/splunk-connect-for-syslog/issues/2592) [#2629](https://github.com/splunk/splunk-connect-for-syslog/issues/2629) --- charts/splunk-connect-for-syslog/Chart.yaml | 4 ++-- package/etc/VERSION | 2 +- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/splunk-connect-for-syslog/Chart.yaml b/charts/splunk-connect-for-syslog/Chart.yaml index fa8ba5b485..4a7092e62c 100644 --- a/charts/splunk-connect-for-syslog/Chart.yaml +++ b/charts/splunk-connect-for-syslog/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: splunk-connect-for-syslog description: Deploy Splunk Connect for Syslog type: application -version: 3.32.1 -appVersion: "3.32.1" +version: 3.33.0 +appVersion: "3.33.0" diff --git a/package/etc/VERSION b/package/etc/VERSION index 1367bf750d..949654dc3e 100644 --- a/package/etc/VERSION +++ b/package/etc/VERSION @@ -1 +1 @@ -3.32.1 +3.33.0 diff --git a/pyproject.toml b/pyproject.toml index 0f2c9e54c0..5bd9a1cc5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "splunk-connect-for-syslog" -version = "3.32.1" +version = "3.33.0" description = "" authors = ["rjha-splunk "] license = "Apache-2.0" From eba35d8483e74dc046e457de98ed26a24e026ac0 Mon Sep 17 00:00:00 2001 From: Ilya Kheifets <138466237+ikheifets-splunk@users.noreply.github.com> Date: Tue, 24 Dec 2024 14:50:24 +0100 Subject: [PATCH 14/15] fix: backward compatibility for healthcheck (#2657) Signed-off-by: Ilya Kheifets --- package/Dockerfile | 1 + package/Dockerfile.lite | 1 + package/sbin/healthcheck.sh | 2 ++ 3 files changed, 4 insertions(+) create mode 100755 package/sbin/healthcheck.sh diff --git a/package/Dockerfile b/package/Dockerfile index b730d652d6..045b6eddaa 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -73,6 +73,7 @@ COPY package/etc/test_parsers /etc/syslog-ng/test_parsers COPY package/etc/local_config /etc/syslog-ng/local_config COPY package/etc/local_config /etc/syslog-ng/local_config COPY package/sbin/entrypoint.sh / +COPY package/sbin/healthcheck.sh / COPY package/sbin/source_ports_validator.py / ENV SC4S_CONTAINER_OPTS=--no-caps diff --git a/package/Dockerfile.lite b/package/Dockerfile.lite index e54ceee047..559d70e80d 100644 --- a/package/Dockerfile.lite +++ b/package/Dockerfile.lite @@ -95,6 +95,7 @@ COPY package/lite/etc/config.yaml /etc/syslog-ng/config.yaml COPY package/lite/etc/addons /etc/syslog-ng/addons COPY package/sbin/entrypoint.sh / +COPY package/sbin/healthcheck.sh / COPY package/sbin/source_ports_validator.py / diff --git a/package/sbin/healthcheck.sh b/package/sbin/healthcheck.sh new file mode 100755 index 0000000000..e073713dee --- /dev/null +++ b/package/sbin/healthcheck.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +/usr/sbin/syslog-ng-ctl healthcheck --timeout 5 \ No newline at end of file From 77e234a14bdf3a070bc278e87ac7652ca186ee87 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 24 Dec 2024 14:05:22 +0000 Subject: [PATCH 15/15] chore(release): 3.33.1 ## [3.33.1](https://github.com/splunk/splunk-connect-for-syslog/compare/v3.33.0...v3.33.1) (2024-12-24) ### Bug Fixes * backward compatibility for healthcheck ([#2657](https://github.com/splunk/splunk-connect-for-syslog/issues/2657)) ([eba35d8](https://github.com/splunk/splunk-connect-for-syslog/commit/eba35d8483e74dc046e457de98ed26a24e026ac0)) --- charts/splunk-connect-for-syslog/Chart.yaml | 4 ++-- package/etc/VERSION | 2 +- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/splunk-connect-for-syslog/Chart.yaml b/charts/splunk-connect-for-syslog/Chart.yaml index 4a7092e62c..dfe33f6c43 100644 --- a/charts/splunk-connect-for-syslog/Chart.yaml +++ b/charts/splunk-connect-for-syslog/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: splunk-connect-for-syslog description: Deploy Splunk Connect for Syslog type: application -version: 3.33.0 -appVersion: "3.33.0" +version: 3.33.1 +appVersion: "3.33.1" diff --git a/package/etc/VERSION b/package/etc/VERSION index 949654dc3e..a91ceb0062 100644 --- a/package/etc/VERSION +++ b/package/etc/VERSION @@ -1 +1 @@ -3.33.0 +3.33.1 diff --git a/pyproject.toml b/pyproject.toml index 5bd9a1cc5e..6969080182 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "splunk-connect-for-syslog" -version = "3.33.0" +version = "3.33.1" description = "" authors = ["rjha-splunk "] license = "Apache-2.0"