diff --git a/CHANGELOG.md b/CHANGELOG.md index 52f767db..c95406c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## New features - [Helm] Add optional liveness and readiness probe - [#1604](https://github.com/jertel/elastalert2/pull/1604) - @aizerin +- Add `include_rule_params_in_matches` rule parameter to enable copying of specific rule params into match data - [#1605](https://github.com/jertel/elastalert2/pull/1605) - @jertel ## Other changes - [Docs] Add missing documentation of the `aggregation_alert_time_compared_with_timestamp_field` option. - [#1588](https://github.com/jertel/elastalert2/pull/1588) - @nicolasnovelli @@ -33,6 +34,7 @@ - Upgrade dependency stomp.py to 8.2.0 - [#1599](https://github.com/jertel/elastalert2/pull/1599) - @jertel - Upgrade dependency tencentcloud-sdk-python to 3.0.1295 - [#1599](https://github.com/jertel/elastalert2/pull/1599) - @jertel - Upgrade dependency twilio to 9.4.1 - [#1599](https://github.com/jertel/elastalert2/pull/1599) - @jertel +- [Spike] Fixes spike rule error when no data exists in the current time window - [#1605](https://github.com/jertel/elastalert2/pull/1605) - @jertel # 2.22.0 diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 16502b7f..b3a1004d 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -120,6 +120,10 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``include_fields`` (list of strs, no default) | | +--------------------------------------------------------------+ | +| ``include_rule_params_in_matches`` (list of strs, no default)| | ++--------------------------------------------------------------+ | +| ``include_rule_params_in_first_match_only`` (boolean, False) | | ++--------------------------------------------------------------+ | | ``filter`` (ES filter DSL, no default) | | +--------------------------------------------------------------+ | | ``max_query_size`` (int, default global max_query_size) | | @@ -625,6 +629,26 @@ include_fields only these fields and those from ``include`` are included. When ``_source_enabled`` is True, these are in addition to source. This is used for runtime fields, script fields, etc. This only works with Elasticsearch version 7.11 and newer. (Optional, list of strings, no default) +include_rule_params_in_matches +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``include_rule_params_in_matches``: This is an optional list of rule parameter names that will have their values copied from the rule into the match records prior to sending out alerts. This allows alerters to have access to specific data in the originating rule. The parameters will be keyed into the match with a ``rule_param_`` prefix. For example, if the ``name`` rule parameter is specified in this list, the match record will have access to the rule name via the ``rule_param_name`` field. Including parameters with complex types, such as maps (Dictionaries) or lists (Arrays) can cause problems if the alerter is unable to convert these into formats that it needs. For example, including the ``query_key`` list parameter in matches that use the http_post2 alerter can cause JSON serialization errors. + +.. note:: + + That this option can cause performance to degrade when a rule is triggered with many matching records since each match record will need to have the rule parameter data copied into it. See the ``include_rule_params_in_first_match_only`` boolean setting, which can mitigate this performance degradation. This performance degradation is more likely to occur with aggregated alerts. + +Example:: + + include_rule_params_in_matches: + - name + - some_custom_param + +include_rule_params_in_first_match_only +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``include_rule_params_in_first_match_only``: When using the ``include_rule_params_in_matches`` setting mentioned above, optionally set to this setting to ``True`` to only copy the rule parameters into the first match record. This is primarily useful for aggregation rules that match hundreds or thousands of records during each run, and where only the first match is used in the alerter. The effectiveness of this setting is dependent upon which alerter(s) are being used. For example, using this setting with ``True`` in a rule that uses the http_post2 alerter will not be useful, since that alerter simply iterates across all matches and POSTs them to the HTTP URL. This would cause only the first POST to have the additional rule parameter values. + top_count_keys ^^^^^^^^^^^^^^ diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 44003a72..f1d29100 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1333,6 +1333,15 @@ def alert(self, matches, rule, alert_time=None, retried=False): except Exception as e: self.handle_uncaught_exception(e, rule) + def include_rule_params_in_matches(self, matches, rule): + if len(rule.get('include_rule_params_in_matches',[])) > 0: + tmp_matches = matches + if rule.get('include_rule_params_in_first_match_only', False): + tmp_matches = [matches[0]] + for match in tmp_matches: + for param in rule.get('include_rule_params_in_matches'): + match['rule_param_' + param] = rule.get(param) + def send_alert(self, matches, rule, alert_time=None, retried=False): """ Send out an alert. @@ -1379,7 +1388,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): opsh_link_formatter = self.get_opensearch_discover_external_url_formatter(rule) matches[0]['opensearch_discover_url'] = opsh_link_formatter.format(opsh_link) - + self.include_rule_params_in_matches(matches, rule) # Enhancements were already run at match time if # run_enhancements_first is set or diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 47509f62..05cd7db7 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -523,7 +523,7 @@ def add_match(self, match, qk): def find_matches(self, ref, cur): """ Determines if an event spike or dip happening. """ # Apply threshold limits - if self.field_value is None: + if self.field_value is None and cur is not None: if (cur < self.rules.get('threshold_cur', 0) or ref < self.rules.get('threshold_ref', 0)): return False diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 09d707c2..de81c002 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -295,6 +295,8 @@ properties: include: {type: array, items: {type: string}} include_fields: {type: array, item: {type: string}} + include_rule_params_in_matches: {type: array, items: {type: string}} + include_rule_params_in_first_match_only: {type: boolean} top_count_keys: {type: array, items: {type: string}} top_count_number: {type: integer} raw_count_keys: {type: boolean} diff --git a/tests/base_test.py b/tests/base_test.py index d7830583..6a345b61 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1484,3 +1484,54 @@ def test_get_kibana_discover_external_url_formatter_smoke(ea): formatter = ea.get_kibana_discover_external_url_formatter(rule) assert type(formatter) is ShortKibanaExternalUrlFormatter assert formatter.security_tenant == 'global' + + +def test_include_rule_params_in_matches(ea): + rule = { + 'include_rule_params_in_matches': ['name', 'foo'], + 'foo': 2, + 'name': 'test-name' + } + matches = [ + { + 'bar': 1, + }, + { + 'bar': 10, + } + ] + + ea.include_rule_params_in_matches(matches, rule) + + assert matches[0]['rule_param_name'] == 'test-name' + assert matches[0]['rule_param_foo'] == 2 + assert matches[0]['bar'] == 1 + assert matches[1]['rule_param_name'] == 'test-name' + assert matches[1]['rule_param_foo'] == 2 + assert matches[1]['bar'] == 10 + + +def test_include_rule_params_in_first_match_only(ea): + rule = { + 'include_rule_params_in_matches': ['name', 'foo'], + 'include_rule_params_in_first_match_only': True, + 'foo': 2, + 'name': 'test-name' + } + matches = [ + { + 'bar': 1, + }, + { + 'bar': 10, + } + ] + + ea.include_rule_params_in_matches(matches, rule) + + assert matches[0]['rule_param_name'] == 'test-name' + assert matches[0]['rule_param_foo'] == 2 + assert matches[0]['bar'] == 1 + assert 'rule_param_name' not in matches[1] + assert 'rule_param_foo' not in matches[1] + assert matches[1]['bar'] == 10 diff --git a/tests/rules_test.py b/tests/rules_test.py index 7d83a224..4dbb796a 100644 --- a/tests/rules_test.py +++ b/tests/rules_test.py @@ -253,6 +253,19 @@ def test_spike_deep_key(): assert 'LOL' in rule.cur_windows +def test_spike_no_data(): + rules = {'threshold_ref': 10, + 'spike_height': 2, + 'timeframe': datetime.timedelta(seconds=10), + 'spike_type': 'both', + 'timestamp_field': '@timestamp', + 'query_key': 'foo.bar.baz', + 'field_value': None} + rule = SpikeRule(rules) + result = rule.find_matches(1, None) + assert not result + + def test_spike(): # Events are 1 per second events = hits(100, timestamp_field='ts')