Merge pull request #1605 from jertel/jertel/wip

Fix spike rule error with no data in cur time window; Add ability to copy rule params into match data

CHANGELOG.md

@@ -5,6 +5,7 @@
 
 ## New features
 - [Helm] Add optional liveness and readiness probe - [#1604](https://github.com/jertel/elastalert2/pull/1604) - @aizerin
+- Add `include_rule_params_in_matches` rule parameter to enable copying of specific rule params into match data - [#1605](https://github.com/jertel/elastalert2/pull/1605) - @jertel
 
 ## Other changes
 - [Docs] Add missing documentation of the `aggregation_alert_time_compared_with_timestamp_field` option. - [#1588](https://github.com/jertel/elastalert2/pull/1588) - @nicolasnovelli
@@ -33,6 +34,7 @@
 - Upgrade dependency stomp.py to 8.2.0 - [#1599](https://github.com/jertel/elastalert2/pull/1599) - @jertel
 - Upgrade dependency tencentcloud-sdk-python to 3.0.1295 - [#1599](https://github.com/jertel/elastalert2/pull/1599) - @jertel
 - Upgrade dependency twilio to 9.4.1 - [#1599](https://github.com/jertel/elastalert2/pull/1599) - @jertel
+- [Spike] Fixes spike rule error when no data exists in the current time window - [#1605](https://github.com/jertel/elastalert2/pull/1605) - @jertel
 
 # 2.22.0
 

docs/source/ruletypes.rst

@@ -120,6 +120,10 @@ Rule Configuration Cheat Sheet
 +--------------------------------------------------------------+           |
 | ``include_fields`` (list of strs, no default)                |           |
 +--------------------------------------------------------------+           |
+| ``include_rule_params_in_matches`` (list of strs, no default)|           |
++--------------------------------------------------------------+           |
+| ``include_rule_params_in_first_match_only`` (boolean, False) |           |
++--------------------------------------------------------------+           |
 | ``filter`` (ES filter DSL, no default)                       |           |
 +--------------------------------------------------------------+           |
 | ``max_query_size`` (int, default global max_query_size)      |           |
@@ -625,6 +629,26 @@ include_fields
 only these fields and those from ``include`` are included.  When ``_source_enabled`` is True, these are in addition to source.  This is used
 for runtime fields, script fields, etc.  This only works with Elasticsearch version 7.11 and newer.  (Optional, list of strings, no default)
 
+include_rule_params_in_matches
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``include_rule_params_in_matches``: This is an optional list of rule parameter names that will have their values copied from the rule into the match records prior to sending out alerts. This allows alerters to have access to specific data in the originating rule. The parameters will be keyed into the match with a ``rule_param_`` prefix. For example, if the ``name`` rule parameter is specified in this list, the match record will have access to the rule name via the ``rule_param_name`` field. Including parameters with complex types, such as maps (Dictionaries) or lists (Arrays) can cause problems if the alerter is unable to convert these into formats that it needs. For example, including the ``query_key`` list parameter in matches that use the http_post2 alerter can cause JSON serialization errors.
+
+.. note::
+
+    That this option can cause performance to degrade when a rule is triggered with many matching records since each match record will need to have the rule parameter data copied into it. See the ``include_rule_params_in_first_match_only`` boolean setting, which can mitigate this performance degradation. This performance degradation is more likely to occur with aggregated alerts.
+
+Example::
+
+    include_rule_params_in_matches:
+    - name
+    - some_custom_param
+
+include_rule_params_in_first_match_only
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``include_rule_params_in_first_match_only``: When using the ``include_rule_params_in_matches`` setting mentioned above, optionally set to this setting to ``True`` to only copy the rule parameters into the first match record. This is primarily useful for aggregation rules that match hundreds or thousands of records during each run, and where only the first match is used in the alerter. The effectiveness of this setting is dependent upon which alerter(s) are being used. For example, using this setting with ``True`` in a rule that uses the http_post2 alerter will not be useful, since that alerter simply iterates across all matches and POSTs them to the HTTP URL. This would cause only the first POST to have the additional rule parameter values.
+
 top_count_keys
 ^^^^^^^^^^^^^^
 

elastalert/elastalert.py

@@ -1333,6 +1333,15 @@ def alert(self, matches, rule, alert_time=None, retried=False):
         except Exception as e:
             self.handle_uncaught_exception(e, rule)
 
+    def include_rule_params_in_matches(self, matches, rule):
+        if len(rule.get('include_rule_params_in_matches',[])) > 0:
+            tmp_matches = matches
+            if rule.get('include_rule_params_in_first_match_only', False):
+                tmp_matches = [matches[0]]
+            for match in tmp_matches:
+                for param in rule.get('include_rule_params_in_matches'):
+                    match['rule_param_' + param] = rule.get(param)
+
     def send_alert(self, matches, rule, alert_time=None, retried=False):
         """ Send out an alert.
 
@@ -1379,7 +1388,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False):
                 opsh_link_formatter = self.get_opensearch_discover_external_url_formatter(rule)
                 matches[0]['opensearch_discover_url'] =  opsh_link_formatter.format(opsh_link)
 
-
+        self.include_rule_params_in_matches(matches, rule)
 
         # Enhancements were already run at match time if
         # run_enhancements_first is set or

elastalert/ruletypes.py

@@ -523,7 +523,7 @@ def add_match(self, match, qk):
     def find_matches(self, ref, cur):
         """ Determines if an event spike or dip happening. """
         # Apply threshold limits
-        if self.field_value is None:
+        if self.field_value is None and cur is not None:
             if (cur < self.rules.get('threshold_cur', 0) or
                     ref < self.rules.get('threshold_ref', 0)):
                 return False

elastalert/schema.yaml

@@ -295,6 +295,8 @@ properties:
 
   include: {type: array, items: {type: string}}
   include_fields: {type: array, item: {type: string}}
+  include_rule_params_in_matches: {type: array, items: {type: string}}
+  include_rule_params_in_first_match_only: {type: boolean}
   top_count_keys: {type: array, items: {type: string}}
   top_count_number: {type: integer}
   raw_count_keys: {type: boolean}

tests/base_test.py

@@ -1484,3 +1484,54 @@ def test_get_kibana_discover_external_url_formatter_smoke(ea):
     formatter = ea.get_kibana_discover_external_url_formatter(rule)
     assert type(formatter) is ShortKibanaExternalUrlFormatter
     assert formatter.security_tenant == 'global'
+
+
+def test_include_rule_params_in_matches(ea):
+    rule = {
+        'include_rule_params_in_matches': ['name', 'foo'],
+        'foo': 2,
+        'name': 'test-name'
+    }
+    matches = [
+        {
+            'bar': 1,
+        },
+        {
+            'bar': 10,
+        }
+    ]
+
+    ea.include_rule_params_in_matches(matches, rule)
+
+    assert matches[0]['rule_param_name'] == 'test-name'
+    assert matches[0]['rule_param_foo'] == 2
+    assert matches[0]['bar'] == 1
+    assert matches[1]['rule_param_name'] == 'test-name'
+    assert matches[1]['rule_param_foo'] == 2
+    assert matches[1]['bar'] == 10
+
+
+def test_include_rule_params_in_first_match_only(ea):
+    rule = {
+        'include_rule_params_in_matches': ['name', 'foo'],
+        'include_rule_params_in_first_match_only': True,
+        'foo': 2,
+        'name': 'test-name'
+    }
+    matches = [
+        {
+            'bar': 1,
+        },
+        {
+            'bar': 10,
+        }
+    ]
+
+    ea.include_rule_params_in_matches(matches, rule)
+
+    assert matches[0]['rule_param_name'] == 'test-name'
+    assert matches[0]['rule_param_foo'] == 2
+    assert matches[0]['bar'] == 1
+    assert 'rule_param_name' not in matches[1]
+    assert 'rule_param_foo' not in matches[1]
+    assert matches[1]['bar'] == 10

tests/rules_test.py

@@ -253,6 +253,19 @@ def test_spike_deep_key():
     assert 'LOL' in rule.cur_windows
 
 
+def test_spike_no_data():
+    rules = {'threshold_ref': 10,
+             'spike_height': 2,
+             'timeframe': datetime.timedelta(seconds=10),
+             'spike_type': 'both',
+             'timestamp_field': '@timestamp',
+             'query_key': 'foo.bar.baz',
+             'field_value': None}
+    rule = SpikeRule(rules)
+    result = rule.find_matches(1, None)
+    assert not result
+
+
 def test_spike():
     # Events are 1 per second
     events = hits(100, timestamp_field='ts')