diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 5ebcf2532f..e895969c0b 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -161,6 +161,12 @@ jobs: - meta - build_action steps: + # To use .trivyignore file, you must check out the repository + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false - name: Run docker vulnerability scanner uses: aquasecurity/trivy-action@master with: @@ -168,7 +174,9 @@ jobs: format: 'table' exit-code: '1' severity: 'CRITICAL,HIGH,MEDIUM,LOW' - + trivyignores: '.trivyignore' + scanners: "vuln" + test-container: runs-on: ubuntu-latest needs: @@ -219,6 +227,7 @@ jobs: TEST_SC4S_ACTIVATE_EXAMPLES: "yes" SC4S_DEBUG_CONTAINER: "yes" SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" + SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" SC4S_USE_VPS_CACHE: "yes" steps: - name: Checkout diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 0a44116cd7..3fb2f70ad8 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -161,6 +161,12 @@ jobs: - meta - build_action steps: + # To use .trivyignore file, you must check out the repository + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false - name: Run docker vulnerability scanner uses: aquasecurity/trivy-action@master with: @@ -168,7 +174,9 @@ jobs: format: 'table' exit-code: '1' severity: 'CRITICAL,HIGH,MEDIUM,LOW' - + trivyignores: '.trivyignore' + scanners: "vuln" + test-container: runs-on: ubuntu-latest needs: @@ -219,6 +227,7 @@ jobs: TEST_SC4S_ACTIVATE_EXAMPLES: "yes" SC4S_DEBUG_CONTAINER: "yes" SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" + SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" SC4S_USE_VPS_CACHE: "yes" steps: diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 0000000000..0a7be8250b --- /dev/null +++ b/.trivyignore @@ -0,0 +1,2 @@ +# This has been safeguarded directly in the code +CVE-2024-35515 \ No newline at end of file diff --git a/charts/splunk-connect-for-syslog/Chart.yaml b/charts/splunk-connect-for-syslog/Chart.yaml index 0c832a81cb..dfe33f6c43 100644 --- a/charts/splunk-connect-for-syslog/Chart.yaml +++ b/charts/splunk-connect-for-syslog/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: splunk-connect-for-syslog description: Deploy Splunk Connect for Syslog type: application -version: 3.31.0 -appVersion: "3.31.0" +version: 3.33.1 +appVersion: "3.33.1" diff --git a/docs/sources/vendor/Dell/avamar.md b/docs/sources/vendor/Dell/avamar.md index 0abaf3b97a..e2a5f661f0 100644 --- a/docs/sources/vendor/Dell/avamar.md +++ b/docs/sources/vendor/Dell/avamar.md @@ -22,4 +22,4 @@ | key | sourcetype | index | notes | |----------------|----------------|----------------|----------------| -| dell_avamar_cms| dell:avamar:msc| netops | none | \ No newline at end of file +| dell_avamar_msc| dell:avamar:msc| netops | none | diff --git a/docs/sources/vendor/NetApp/ontap.md b/docs/sources/vendor/NetApp/ontap.md index 2eacc9c6a7..0ead82cf82 100644 --- a/docs/sources/vendor/NetApp/ontap.md +++ b/docs/sources/vendor/NetApp/ontap.md @@ -3,7 +3,7 @@ ## Key facts * MSG Format based filter -* Legacy BSD Format default port 514 +* Netapp Ontap messages are not distinctive. So, either configure known Netapp Ontap hosts in SC4S, or open unique ports for Netapp Ontap devices ## Links @@ -16,11 +16,49 @@ | sourcetype | notes | |----------------|---------------------------------------------------------------------------------------------------------| -| netapp:ems | None | +| ontap:ems | This sourcetype will be assinged only when the environment variable `SC4S_NETAPP_ONTAP_NEW_FORMAT` is not set or is set to 'no'. By default it is unset | +| netapp:ontap:audit | This sourcetype will be assinged only when the environment variable `SC4S_NETAPP_ONTAP_NEW_FORMAT` is set to 'yes' | +| netapp:ontap:ems | This sourcetype will be assinged only when the environment variable `SC4S_NETAPP_ONTAP_NEW_FORMAT` is set to 'yes' | ## Sourcetype and Index Configuration | key | sourcetype | index | notes | |----------------|----------------|----------------|----------------| -| netapp_ontap | netapp:ems | infraops | none | +| netapp_ontap | ontap:ems | infraops | none | +| netapp_ontap_audit | netapp:ontap:audit | infraops | none | +| netapp_ontap_ems | netapp:ontap:ems | infraops | none | +## Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_NETAPP_ONTAP_NEW_FORMAT | empty string | (empty/yes) Set to "yes" for the applying the latest changes. Make sure to configure your system to send the logs to a specific port or have a hostname-based configuration | + +## Parser Configuration +1. Through sc4s-vps +```c +#/opt/sc4s/local/config/app-parsers/app-vps-netapp_ontap.conf +#File name provided is a suggestion it must be globally unique + +application app-vps-test-netapp_ontap[sc4s-vps] { + filter { + host("netapp-ontap-" type(string) flags(prefix)) + or ( + message("netapp-ontap-" type(string) flags(prefix)) + and program("netapp-ontap-" type(string) flags(prefix)) + ) + }; + parser { + p_set_netsource_fields( + vendor('netapp') + product('ontap') + ); + }; +}; +``` + +2. or through unique port +``` +# /opt/sc4s/env_file +SC4S_LISTEN_NETAPP_ONTAP_UDP_PORT=5005 +``` \ No newline at end of file diff --git a/docs/upgrade.md b/docs/upgrade.md index e17d2c0e76..813c4f1824 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -18,6 +18,9 @@ For a step by step guide [see here](./v3_upgrade.md). You may need to migrate legacy log paths or version 1 app-parsers for version 2. To do this, open an issue and attach the original configuration and a compressed pcap of sample data for testing. We will evaluate whether to include the source in an upcoming release. +### Upgrade from <3.33.0 +In NetApp ONTAP, the ontap:ems sourcetype has been updated to netapp:ontap:audit, so old logs are now classified under netapp:ontap:audit. Additionally, a new netapp:ontap:ems sourcetype has been introduced. If you upgrade and want these new changes, ensure that you set `SC4S_NETAPP_ONTAP_NEW_FORMAT` environment variable to `yes` and configure your system to send the logs to a specific port or have a hostname-based configuration in place for proper log onboarding into Splunk. + ### Upgrade from <2.23.0 * In VMware vSphere, update the ESX and vCenter sourcetype for add-on compatibility. diff --git a/package/Dockerfile b/package/Dockerfile index c91e3bc53a..045b6eddaa 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -28,7 +28,7 @@ RUN apk add -U --upgrade --no-cache \ less \ net-tools \ netcat-openbsd \ - openssl \ + "openssl>=3.3.2-r1" \ procps \ py3-pip \ python3 \ @@ -73,6 +73,7 @@ COPY package/etc/test_parsers /etc/syslog-ng/test_parsers COPY package/etc/local_config /etc/syslog-ng/local_config COPY package/etc/local_config /etc/syslog-ng/local_config COPY package/sbin/entrypoint.sh / +COPY package/sbin/healthcheck.sh / COPY package/sbin/source_ports_validator.py / ENV SC4S_CONTAINER_OPTS=--no-caps diff --git a/package/Dockerfile.lite b/package/Dockerfile.lite index 24f30686d2..559d70e80d 100644 --- a/package/Dockerfile.lite +++ b/package/Dockerfile.lite @@ -28,7 +28,7 @@ RUN apk add -U --upgrade --no-cache \ less \ net-tools \ netcat-openbsd \ - openssl \ + "openssl>=3.3.2-r1" \ procps \ py3-pip \ python3 \ @@ -95,6 +95,7 @@ COPY package/lite/etc/config.yaml /etc/syslog-ng/config.yaml COPY package/lite/etc/addons /etc/syslog-ng/addons COPY package/sbin/entrypoint.sh / +COPY package/sbin/healthcheck.sh / COPY package/sbin/source_ports_validator.py / diff --git a/package/etc/VERSION b/package/etc/VERSION index d9351e5882..a91ceb0062 100644 --- a/package/etc/VERSION +++ b/package/etc/VERSION @@ -1 +1 @@ -3.31.0 +3.33.1 diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf b/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf new file mode 100644 index 0000000000..08cd806164 --- /dev/null +++ b/package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf @@ -0,0 +1,55 @@ +block parser app-netsource-netapp_ontap() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + vendor("netapp") + product("ontap") + ); + }; + + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^[A-Za-z0-9\-\_\.]+: [0-9a-f]+\.[0-9a-f]+ [0-9a-f]+ [A-Z][a-z][a-z] (?[A-Z][a-z][a-z] \d\d \d\d\d\d \d\d:\d\d:\d\d [+-]?\d{1,2}:\d\d)') + ); + date-parser-nofilter( + format( + '%b %d %Y %H:%M:%S %z', + ) + template("${.tmp.timestamp}") + ); + }; + + rewrite { + set('$PROGRAM: $MESSAGE', value(MESSAGE)); + set('$PROGRAM', value(HOST)); + unset(value(PROGRAM)); + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:audit') + class('audit') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:ems') + class('ems') + ); + }; + }; + }; +}; + +application app-netsource-netapp_ontap[sc4s-network-source] { + filter { + match("netapp", value('.netsource.sc4s_vendor'), type(string)) + and match("ontap", value('.netsource.sc4s_product'), type(string)) + and "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes" + }; + parser { app-netsource-netapp_ontap(); }; +}; diff --git a/package/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf b/package/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf index fc0a6c679b..7f66031700 100644 --- a/package/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf +++ b/package/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf @@ -53,7 +53,7 @@ block parser app-raw-bsd_nopri() { }; application app-raw-bsd_nopri[sc4s-raw-syslog] { filter { - message('^\w\w\w \d\d \d\d:\d\d:\d\d '); + message('^\w{3} ([0 ][1-9]|[12]\d|3[01]) (0\d|1\d|2[0-3]):([0-5]\d):([0-5]\d) '); }; parser { app-raw-bsd_nopri(); }; }; diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf b/package/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf index a229e46e28..f1a71f5076 100644 --- a/package/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf +++ b/package/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf @@ -74,6 +74,7 @@ application app-syslog-juniper_junos_unstructured-pgm[sc4s-syslog-pgm] { or program('RT_FLOW' type(string) flags(prefix)) or program('RT_IDS' type(string) flags(prefix)) or program('RT_UTM' type(string) flags(prefix)) + or program('RT_SYSTEM' type(string) flags(prefix)) or program('Juniper' type(string) flags(prefix)) or program('rpd' type(string) flags(prefix)) or program('eswd' type(string) flags(prefix)) diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf b/package/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf index 84db3a994c..46b37a1b0b 100644 --- a/package/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf +++ b/package/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf @@ -37,7 +37,8 @@ block parser app-syslog-netapp_ontap() { }; application app-syslog-netapp_ontap[sc4s-syslog] { filter { - program('^[A-Za-z0-9\-\_\.]+$'); + program('^[A-Za-z0-9\-\_\.]+$') + and not "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes"; }; parser { app-syslog-netapp_ontap(); }; }; diff --git a/package/etc/pylib/parser_source_cache.py b/package/etc/pylib/parser_source_cache.py index dc573a3449..1a8bcbca2a 100644 --- a/package/etc/pylib/parser_source_cache.py +++ b/package/etc/pylib/parser_source_cache.py @@ -20,12 +20,12 @@ class LogDestination: def ip2int(addr): ip4_to_int = lambda addr: struct.unpack("!I", socket.inet_aton(addr))[0] - + def ip6_to_int(addr): ip6 = socket.inet_pton(socket.AF_INET6, addr) a, b = struct.unpack(">QQ", ip6) return (a << 64) | b - + try: return ip4_to_int(addr) except OSError: @@ -41,7 +41,7 @@ def int_to_ip6(num): ip6 = struct.pack(">QQ", a, b) addr = socket.inet_ntop(socket.AF_INET6, ip6) return addr - + try: return int_to_ip4(addr) except struct.error: @@ -126,4 +126,4 @@ def flush(self): db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) db[0] = "seed" db.commit() - db.close() \ No newline at end of file + db.close() diff --git a/package/etc/pylib/parser_vps_cache.py b/package/etc/pylib/parser_vps_cache.py index a95162862b..4c8cf21250 100644 --- a/package/etc/pylib/parser_vps_cache.py +++ b/package/etc/pylib/parser_vps_cache.py @@ -98,4 +98,4 @@ def flush(self): if __name__ == "__main__": - pass \ No newline at end of file + pass diff --git a/package/etc/pylib/psc_dump.py b/package/etc/pylib/psc_dump.py index 927ce6be92..eccbe1e650 100644 --- a/package/etc/pylib/psc_dump.py +++ b/package/etc/pylib/psc_dump.py @@ -1,4 +1,3 @@ - import sys import traceback import socket @@ -9,5 +8,5 @@ hostdict = str("/var/lib/syslog-ng/cache/hostip") db = SqliteDict(f"{hostdict}.sqlite") -for k,v in db.items(): - print(f"key={k}={v}") \ No newline at end of file +for k, v in db.items(): + print(f"key={k}={v}") diff --git a/package/etc/test_parsers/app-vps-test-netapp_ontap.conf b/package/etc/test_parsers/app-vps-test-netapp_ontap.conf new file mode 100644 index 0000000000..24b14274e0 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-netapp_ontap.conf @@ -0,0 +1,15 @@ +application app-vps-test-netapp_ontap[sc4s-vps] { + filter { + host("netapp-ontap-" type(string) flags(prefix)) + or ( + message("netapp-ontap-" type(string) flags(prefix)) + and program("netapp-ontap-" type(string) flags(prefix)) + ) + }; + parser { + p_set_netsource_fields( + vendor('netapp') + product('ontap') + ); + }; +}; \ No newline at end of file diff --git a/package/lite/etc/addons/juniper/app-syslog-juniper_junos_unstructured.conf b/package/lite/etc/addons/juniper/app-syslog-juniper_junos_unstructured.conf index a229e46e28..f1a71f5076 100644 --- a/package/lite/etc/addons/juniper/app-syslog-juniper_junos_unstructured.conf +++ b/package/lite/etc/addons/juniper/app-syslog-juniper_junos_unstructured.conf @@ -74,6 +74,7 @@ application app-syslog-juniper_junos_unstructured-pgm[sc4s-syslog-pgm] { or program('RT_FLOW' type(string) flags(prefix)) or program('RT_IDS' type(string) flags(prefix)) or program('RT_UTM' type(string) flags(prefix)) + or program('RT_SYSTEM' type(string) flags(prefix)) or program('Juniper' type(string) flags(prefix)) or program('rpd' type(string) flags(prefix)) or program('eswd' type(string) flags(prefix)) diff --git a/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf b/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf new file mode 100644 index 0000000000..08cd806164 --- /dev/null +++ b/package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf @@ -0,0 +1,55 @@ +block parser app-netsource-netapp_ontap() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + vendor("netapp") + product("ontap") + ); + }; + + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^[A-Za-z0-9\-\_\.]+: [0-9a-f]+\.[0-9a-f]+ [0-9a-f]+ [A-Z][a-z][a-z] (?[A-Z][a-z][a-z] \d\d \d\d\d\d \d\d:\d\d:\d\d [+-]?\d{1,2}:\d\d)') + ); + date-parser-nofilter( + format( + '%b %d %Y %H:%M:%S %z', + ) + template("${.tmp.timestamp}") + ); + }; + + rewrite { + set('$PROGRAM: $MESSAGE', value(MESSAGE)); + set('$PROGRAM', value(HOST)); + unset(value(PROGRAM)); + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:audit') + class('audit') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:ems') + class('ems') + ); + }; + }; + }; +}; + +application app-netsource-netapp_ontap[sc4s-network-source] { + filter { + match("netapp", value('.netsource.sc4s_vendor'), type(string)) + and match("ontap", value('.netsource.sc4s_product'), type(string)) + and "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes" + }; + parser { app-netsource-netapp_ontap(); }; +}; diff --git a/package/lite/etc/addons/netapp/app-syslog-netapp_ontap.conf b/package/lite/etc/addons/netapp/app-syslog-netapp_ontap.conf index 84db3a994c..46b37a1b0b 100644 --- a/package/lite/etc/addons/netapp/app-syslog-netapp_ontap.conf +++ b/package/lite/etc/addons/netapp/app-syslog-netapp_ontap.conf @@ -37,7 +37,8 @@ block parser app-syslog-netapp_ontap() { }; application app-syslog-netapp_ontap[sc4s-syslog] { filter { - program('^[A-Za-z0-9\-\_\.]+$'); + program('^[A-Za-z0-9\-\_\.]+$') + and not "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes"; }; parser { app-syslog-netapp_ontap(); }; }; diff --git a/package/lite/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf b/package/lite/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf index fc0a6c679b..7f66031700 100644 --- a/package/lite/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf +++ b/package/lite/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf @@ -53,7 +53,7 @@ block parser app-raw-bsd_nopri() { }; application app-raw-bsd_nopri[sc4s-raw-syslog] { filter { - message('^\w\w\w \d\d \d\d:\d\d:\d\d '); + message('^\w{3} ([0 ][1-9]|[12]\d|3[01]) (0\d|1\d|2[0-3]):([0-5]\d):([0-5]\d) '); }; parser { app-raw-bsd_nopri(); }; }; diff --git a/package/sbin/healthcheck.sh b/package/sbin/healthcheck.sh new file mode 100755 index 0000000000..e073713dee --- /dev/null +++ b/package/sbin/healthcheck.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +/usr/sbin/syslog-ng-ctl healthcheck --timeout 5 \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index def4dd7cec..c379f0fc3b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.9.0.dev0 and should not be changed by hand. [[package]] name = "arrow" @@ -680,13 +680,13 @@ mkdocs = ">=1.0.4" [[package]] name = "mkdocs-material" -version = "9.5.42" +version = "9.5.47" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.42-py3-none-any.whl", hash = "sha256:452a7c5d21284b373f36b981a2cbebfff59263feebeede1bc28652e9c5bbe316"}, - {file = "mkdocs_material-9.5.42.tar.gz", hash = "sha256:92779b5e9b5934540c574c11647131d217dc540dce72b05feeda088c8eb1b8f2"}, + {file = "mkdocs_material-9.5.47-py3-none-any.whl", hash = "sha256:53fb9c9624e7865da6ec807d116cd7be24b3cb36ab31b1d1d1a9af58c56009a2"}, + {file = "mkdocs_material-9.5.47.tar.gz", hash = "sha256:fc3b7a8e00ad896660bd3a5cc12ca0cb28bdc2bcbe2a946b5714c23ac91b0ede"}, ] [package.dependencies] @@ -1175,6 +1175,17 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "restricted-sqlitedict" +version = "1.0.0" +description = "Fork of sqlitedict with restricted pickle loading." +optional = false +python-versions = "*" +files = [ + {file = "restricted_sqlitedict-1.0.0-py3-none-any.whl", hash = "sha256:31cdeac0f48ee16cdbfeb8e8c14e8c1a17ebb67e6f098f5724a46515c2d2d3c6"}, + {file = "restricted_sqlitedict-1.0.0.tar.gz", hash = "sha256:f4c319ca51a01b37b2350f418640a9d80061402d58a7df7fcb1e0147b5d9cf13"}, +] + [[package]] name = "setuptools" version = "73.0.1" @@ -1215,27 +1226,17 @@ files = [ [[package]] name = "splunk-sdk" -version = "2.0.2" +version = "2.1.0" description = "The Splunk Software Development Kit for Python." optional = false python-versions = "*" files = [ - {file = "splunk-sdk-2.0.2.tar.gz", hash = "sha256:d5ccf6e1b96e493b1399e071ef10f8337e0a39ac2b2acc73fdd375b87b4104cb"}, + {file = "splunk-sdk-2.1.0.tar.gz", hash = "sha256:63f9a259a7c84d0c3b0b32cae652365b03f0f926acdb894b51456005df74ae21"}, ] [package.dependencies] deprecation = "*" -[[package]] -name = "sqlitedict" -version = "2.1.0" -description = "Persistent dict in Python, backed up by sqlite3 and pickle, multithread-safe." -optional = false -python-versions = "*" -files = [ - {file = "sqlitedict-2.1.0.tar.gz", hash = "sha256:03d9cfb96d602996f1d4c2db2856f1224b96a9c431bdd16e78032a72940f9e8c"}, -] - [[package]] name = "tomli" version = "2.0.1" @@ -1384,4 +1385,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "0caa7b7eb023b55566f6cc063d7b2a5daf4a3ff78deaabd9dadb2c625f3e12bf" +content-hash = "15a805f6379fa28a3b310fc492b3f9d468945660693413c9c0c7e54c86841475" diff --git a/pyproject.toml b/pyproject.toml index ed44759638..6969080182 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "splunk-connect-for-syslog" -version = "3.31.0" +version = "3.33.1" description = "" authors = ["rjha-splunk "] license = "Apache-2.0" @@ -8,11 +8,11 @@ license = "Apache-2.0" [tool.poetry.dependencies] python = "^3.9" Jinja2 = "^3.1.3" -sqlitedict = "^2.0.0" requests = "^2.28.1" shortuuid = "^1.0.11" pyyaml = "6.0.2" setuptools = "^73.0.1" +restricted-sqlitedict = "^1.0.0" [tool.poetry.group.dev.dependencies] diff --git a/tests/test_juniper_junos_rfc3164.py b/tests/test_juniper_junos_rfc3164.py index 16b2642fb0..98ce4424fa 100644 --- a/tests/test_juniper_junos_rfc3164.py +++ b/tests/test_juniper_junos_rfc3164.py @@ -6,7 +6,7 @@ from jinja2 import Environment, select_autoescape from .sendmessage import sendsingle -from .splunkutils import splunk_single +from .splunkutils import splunk_single from .timeutils import time_operations import datetime import pytest @@ -15,9 +15,7 @@ # <23> Mar 18 17:56:52 RT_UTM: WEBFILTER_URL_PERMITTED: WebFilter: ACTION="URL Permitted" 192.168.32.1(62054)->1.1.1.1(443) CATEGORY="Enhanced_Information_Technology" REASON="BY_PRE_DEFINED" PROFILE="UTM-Wireless-Profile" URL=ent-shasta-rrs.symantec.com OBJ=/ username N/A roles N/A @pytest.mark.addons("juniper") -def test_juniper_utm_standard( - record_property, get_host_key, setup_splunk, setup_sc4s -): +def test_juniper_utm_standard(record_property, get_host_key, setup_splunk, setup_sc4s): host = get_host_key dt = datetime.datetime.now() @@ -50,7 +48,7 @@ def test_juniper_utm_standard( # <23> Nov 18 09:56:58 INTERNET-ROUTER RT_FLOW: RT_FLOW_SESSION_CREATE: session created 192.168.1.102/58662->8.8.8.8/53 junos-dns-udp 68.144.1.1/55893->8.8.8.8/53 TRUST-INET-ACCESS None 17 OUTBOUND-INTERNET-ACCESS TRUST INTERNET 6316 N/A(N/A) vlan.192 @pytest.mark.addons("juniper") def test_juniper_firewall_standard( - record_property, get_host_key, setup_splunk, setup_sc4s + record_property, get_host_key, setup_splunk, setup_sc4s ): host = get_host_key @@ -82,9 +80,7 @@ def test_juniper_firewall_standard( @pytest.mark.addons("juniper") -def test_juniper_idp_standard( - record_property, get_host_key, setup_splunk, setup_sc4s -): +def test_juniper_idp_standard(record_property, get_host_key, setup_splunk, setup_sc4s): host = get_host_key dt = datetime.datetime.now() @@ -123,7 +119,7 @@ def test_juniper_idp_standard( @pytest.mark.addons("juniper") @pytest.mark.parametrize("event", testdata_junos_snmp) def test_juniper_junos_snmp( - record_property, get_host_key, setup_splunk, setup_sc4s, event + record_property, get_host_key, setup_splunk, setup_sc4s, event ): host = get_host_key @@ -160,7 +156,7 @@ def test_juniper_junos_snmp( @pytest.mark.addons("juniper") @pytest.mark.parametrize("event", testdata_junos_firewall_switch) def test_juniper_junos_switch( - record_property, get_host_key, setup_splunk, setup_sc4s, event + record_property, get_host_key, setup_splunk, setup_sc4s, event ): host = get_host_key @@ -198,7 +194,7 @@ def test_juniper_junos_switch( @pytest.mark.parametrize("event", testdata_junos_firewall_router) @pytest.mark.addons("juniper") def test_juniper_junos_router( - record_property, get_host_key, setup_splunk, setup_sc4s, event + record_property, get_host_key, setup_splunk, setup_sc4s, event ): host = get_host_key @@ -235,7 +231,7 @@ def test_juniper_junos_router( @pytest.mark.addons("juniper") @pytest.mark.parametrize("event", testdata_junos_switch_rpd) def test_juniper_junos_switch_rpd( - record_property, get_host_key, setup_splunk, setup_sc4s, event + record_property, get_host_key, setup_splunk, setup_sc4s, event ): host = get_host_key @@ -263,3 +259,37 @@ def test_juniper_junos_switch_rpd( record_property("message", message) assert result_count == 1 + + +# <161>Mar 18 17:56:52 host RT_SYSTEM: RTLOG_CONN_ERROR: Connection error tcp_10.181.123.45 Error code: major 3 minor 1 code 110, description:TCP timed out after SYN is sent out +@pytest.mark.addons("juniper") +def test_juniper_system_standard( + record_property, get_host_key, setup_splunk, setup_sc4s +): + host = get_host_key + + dt = datetime.datetime.now() + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string( + "{{ mark }} {{ bsd }} {{ host }} RT_SYSTEM: RTLOG_CONN_ERROR: Connection error tcp_10.181.123.45 Error code: major 3 minor 1 code 110, description:TCP timed out after SYN is sent out " + ) + message = mt.render(mark="<161>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="juniper:legacy"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 diff --git a/tests/test_name_cache.py b/tests/test_name_cache.py index ce8a56f502..fb3a55865f 100644 --- a/tests/test_name_cache.py +++ b/tests/test_name_cache.py @@ -5,8 +5,10 @@ # https://opensource.org/licenses/BSD-2-Clause import datetime +import pickle import random import re +import tempfile import time from jinja2 import Environment @@ -14,8 +16,9 @@ from .timeutils import time_operations from .sendmessage import sendsingle -from .splunkutils import splunk_single +from .splunkutils import splunk_single from package.etc.pylib.parser_source_cache import ip2int, int2ip +from sqlitedict import SqliteDict env = Environment() @@ -45,7 +48,7 @@ def test_name_cache(get_host_key, setup_splunk, setup_sc4s): _ = send_message(template_no_host, setup_sc4s) _ = send_message(template_with_host, setup_sc4s, host=get_host_key) - time.sleep(1) # time to save the new cache entry + time.sleep(1) # time to save the new cache entry epoch = send_message(template_no_host, setup_sc4s) search = f'search _time="{epoch}" index=* host="{get_host_key}"' @@ -58,19 +61,59 @@ def generate_random_ipv4(): random_octet = lambda: format(random.randint(0, 255)) return ".".join([random_octet() for _ in range(4)]) + def generate_random_ipv6(): def generate_random_hex(): - random_hex = format(random.randint(0, 65535), '04x') - random_hex = re.sub('^0+', '', random_hex) # leading zeros can be skipped + random_hex = format(random.randint(0, 65535), "04x") + random_hex = re.sub("^0+", "", random_hex) # leading zeros can be skipped return random_hex + return ":".join([generate_random_hex() for _ in range(8)]) + @pytest.mark.name_cache def test_ipv4_utils(): ip = generate_random_ipv4() assert ip == int2ip(ip2int(ip)) + @pytest.mark.name_cache def test_ipv6_utils(): ip = generate_random_ipv6() - assert ip == int2ip(ip2int(ip)) \ No newline at end of file + assert ip == int2ip(ip2int(ip)) + + +@pytest.mark.name_cache +def test_RestrictedSqliteDict_stores_and_retrieves_string(): + with tempfile.NamedTemporaryFile(delete=True) as temp_db_file: + cache = SqliteDict(f"{temp_db_file.name}.db") + cache["key"] = "value" + cache.commit() + cache.close() + + cache = SqliteDict(f"{temp_db_file.name}.db") + assert cache["key"] == "value" + cache.close() + + +@pytest.mark.name_cache +def test_RestrictedSqliteDict_prevents_code_injection(): + class InjectionTestClass: + def __reduce__(self): + import os + + return os.system, ("touch pwned.txt",) + + with tempfile.NamedTemporaryFile(delete=True) as temp_db_file: + # Initialize the RestrictedSqliteDict and insert an 'injected' object + cache = SqliteDict(f"{temp_db_file.name}.db") + cache["key"] = InjectionTestClass() + cache.commit() + cache.close() + + # Re-open cache and attempt to deserialize 'injected' object + # Expecting UnpicklingError due to RestrictedSqliteDict restrictions + cache = SqliteDict(f"{temp_db_file.name}.db") + with pytest.raises(pickle.UnpicklingError): + _ = cache["key"] + cache.close() diff --git a/tests/test_netapp.py b/tests/test_netapp.py index 07bf82c533..6451deac06 100644 --- a/tests/test_netapp.py +++ b/tests/test_netapp.py @@ -17,16 +17,16 @@ testdata = [ - "{{ mark }}{{ bsd }} {{ host }}: {{ host }}: 0000001e.0794c163 055b6737 {{ device_time }} [kern_audit:info:2385] 8503ea0000ba6b71 :: nodea:ontapi :: 10.10.10.10:41464 :: nodea-esx:usera :: clone-create :: Error: Missing input: source-path; Missing input: volume", + "{{ mark }}{{ bsd }} {{ host }}: {{ host }}: 00000030.00c8f1e2 11e5347f {{ device_time }} [kern_audit3167] 8004b7000021e73b:4005f7000021e73d :: cluster:ssh :: 0.0.0.0:32879 :: cluster:admin :: qos statistics volume performance show -rows 20 -iter 1 :: Pending", ] - +# <14>Oct 3 11:36:46 host: host: 00000030.00c8f1e2 11e5347f Thu Oct 03 2024 11:36:44 -06:00 [kern_audit3167] 8004b7000021e73b:4005f7000021e73d :: cluster:ssh :: 0.0.0.0:32879 :: cluster:admin :: qos statistics volume performance show -rows 20 -iter 1 :: Pending @pytest.mark.addons("netapp") @pytest.mark.parametrize("event", testdata) -def test_netapp( +def test_netapp_ontap_audit( record_property, get_host_key, setup_splunk, setup_sc4s, event ): - host = get_host_key + host = "netapp-ontap-" + get_host_key dt = datetime.datetime.now(datetime.timezone.utc) _, bsd, _, _, _, _, epoch = time_operations(dt) @@ -37,12 +37,12 @@ def test_netapp( device_time = dt.strftime("%a %b %d %Y %H:%M:%S +00:00") mt = env.from_string(event + "\n") - message = mt.render(mark="<166>", bsd=bsd, host=host, device_time=device_time) + message = mt.render(mark="<14>", bsd=bsd, host=host, device_time=device_time) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=infraops _time={{ epoch }} sourcetype="ontap:ems" (host="{{ host }}" OR "{{ host }}")' + 'search index=infraops _time={{ epoch }} sourcetype="netapp:ontap:audit"' ) search = st.render(epoch=epoch, host=host) @@ -53,3 +53,38 @@ def test_netapp( record_property("message", message) assert result_count == 1 + + +# Netapp Ontap EMS event in rfc5424 format +# <5>1 2024-10-03T07:54:02-06:00 host program - wafl.scan.done - Completed Volume Footprint Estimator Scan on volume vm_unix002_0d@vserver:27902083bf98-11e9-87fe-00a098b15eb6 +@pytest.mark.addons("netapp") +def test_netapp_ontap_ems_rfc5424( + record_property, get_host_key, setup_splunk, setup_sc4s +): + host = "netapp-ontap-" + get_host_key + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, _, _, _, _, _, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-3] + + mt = env.from_string( + '{{ mark }} {{ iso }} {{ host }} program - wafl.scan.done - Completed Volume Footprint Estimator Scan on volume vm_unix002_0d@vserver:27902083bf98-11e9-87fe-00a098b15eb6' + ) + message = mt.render(mark="<5>1", iso=iso, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=infraops sourcetype="netapp:ontap:ems"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 \ No newline at end of file diff --git a/tests/test_trellix.py b/tests/test_trellix.py index 4f0a3dc968..c461e9ea0d 100644 --- a/tests/test_trellix.py +++ b/tests/test_trellix.py @@ -67,3 +67,37 @@ def test_trellix_cms( record_property("message", message) assert result_count == 1 + + +# Nov 2 00:00:00 trellix-host-xxxx CEF:0|Trellix|Database Security|0.0.0|alert|DML Queries executed from the backend|3|externalId=53 rt=1030482923264 cs1=Finacle Alert_DC_54 cs1Label=DBMS dst=10.10.10.10 src=10.10.10.11 duser=TESTUSER suser=testuser shost=KBPXXXUD00xx6 dproc=sxxxplus@KBPXXXUD00xx6 (TNS V1-V3) act=DELETE cs2=DELETE FROM TESTUSER.XXXX WHERE XXXX.ALERT_REFERENCE_NO \= :B1 cs2Label=SqlStatement cs3=XXXX|ALERT_HISTORY_TBL cs3Label=AccessedObjects. +@pytest.mark.addons("trellix") +def test_trellix_cef(record_property, get_host_key, setup_splunk, setup_sc4s): + host = "trellix-host-" + get_host_key + + dt = datetime.datetime(2024, 11, 2, 0, 0) + _, bsd, _, _, _, _, epoch = time_operations(dt) + + # hard coding the bsd to test single digit date (Nov 2 00:00:00) + bsd = "Nov 2 00:00:00" + # Tune time functions + epoch = epoch[:-3] + + mt = env.from_string( + "{{ bsd }} {{ host }} CEF:0|Trellix|Database Security|0.0.0|alert|DML Queries executed from the backend|3|externalId=53 rt={{ epoch }} cs1=Finacle Alert_DC_54 cs1Label=DBMS dst=10.10.10.10 src=10.10.10.11 duser=TESTUSER suser=testuser shost=KBPXXXUD00xx6 dproc=sxxxplus@KBPXXXUD00xx6 (TNS V1-V3) act=DELETE cs2=DELETE FROM TESTUSER.XXXX WHERE XXXX.ALERT_REFERENCE_NO \= :B1 cs2Label=SqlStatement cs3=XXXX|ALERT_HISTORY_TBL cs3Label=AccessedObjects." + ) + message = mt.render(bsd=bsd, host=host, epoch=epoch) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=main host="{{ host }}" sourcetype="cef"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 \ No newline at end of file