feat: improved karafka plugin for insights (#625)

* [feat] improved karafka events and metrics

This implements a new Karafka subscriber that monitors more relevant
events. The subscriber will then emit events and metrics where
applicable.

This also includes new configuration parameters that let you enable and
disable Karafka specific Insights data. These are enabled by default.

```
karafka:
  insights:
    events: [boolean] # allow plugin to emit events from monitor
    metrics: [boolean] # allow plugin to emit metrics from monitor
```

* Pull error notifier out of listener

* Add ability to configure via env

* Add more to histogram payloads

* Expand tags into proper keys

* Fix

* Add aggregated topic stats

* Use underscores in metric names

* Fix bug

* Refactor

* More refactoring

* Also subscribe to producer errors

* Use value keyword for all metric types

* Simplify implementation and update specs

* Fix some bugs and better uniformity to metrics code

* Remove uncessary config

* default values to 1 when not specified

Author: roelbondoc

lib/honeybadger/config.rb

@@ -309,6 +309,20 @@ def load_plugin_insights?(name)
       !!self[:"#{name}.insights.enabled"]
     end
 
+    def load_plugin_insights_events?(name)
+      return false unless insights_enabled?
+      return false unless load_plugin_insights?(name)
+      return true if self[:"#{name}.insights.events"].nil?
+      !!self[:"#{name}.insights.events"]
+    end
+
+    def load_plugin_insights_metrics?(name)
+      return false unless insights_enabled?
+      return false unless load_plugin_insights?(name)
+      return true if self[:"#{name}.insights.metrics"].nil?
+      !!self[:"#{name}.insights.metrics"]
+    end
+
     def root_regexp
       return @root_regexp if @root_regexp
       return nil if @no_root

lib/honeybadger/config/defaults.rb

@@ -363,6 +363,21 @@ class Boolean; end
         default: 60,
         type: Integer
       },
+      :'karafka.insights.enabled' => {
+        description: 'Enable automatic data collection for Karafka.',
+        default: true,
+        type: Boolean
+      },
+      :'karafka.insights.events' => {
+        description: 'Enable automatic event capturing for Karafka.',
+        default: true,
+        type: Boolean
+      },
+      :'karafka.insights.metrics' => {
+        description: 'Enable automatic metric data collection for Karafka.',
+        default: true,
+        type: Boolean
+      },
       :'net_http.insights.enabled' => {
         description: 'Allow automatic instrumentation of Net::HTTP requests.',
         default: true,

lib/honeybadger/histogram.rb

@@ -9,6 +9,15 @@ def record(value)
       return unless value
 
       @samples += 1
+
+      @total ||= 0
+      @total = @total + value
+
+      @min = value if @min.nil? || @min > value
+      @max = value if @max.nil? || @max < value
+      @avg = @total.to_f / @samples
+      @latest = value
+
       @bin_counts ||= Hash.new(0)
       @bin_counts[find_bin(value)] += 1
     end
@@ -25,6 +34,10 @@ def bins
 
     def payloads
       [{
+        min: @min,
+        max: @max,
+        avg: @avg,
+        latest: @latest,
         bins: (bins + [INFINITY]).map { |bin| [bin.to_f, @bin_counts[bin]] }
       }]
     end

lib/honeybadger/instrumentation.rb

@@ -47,63 +47,91 @@ def monotonic_timer
     def time(name, *args)
       attributes = extract_attributes(args)
       callable = extract_callable(args)
-      duration = attributes.delete(:duration)
+      value = nil
 
       if callable
-        duration = monotonic_timer{ callable.call }[0]
+        value = monotonic_timer{ callable.call }[0]
       elsif block_given?
-        duration = monotonic_timer{ yield }[0]
+        value = monotonic_timer{ yield }[0]
+      else
+        value = attributes.delete(:duration) || attributes.delete(:value)
       end
 
-      raise 'No duration found' if duration.nil?
+      raise 'No value found' if value.nil?
 
       Honeybadger::Timer.register(registry, name, attributes).tap do |timer|
-        timer.record(duration)
+        timer.record(value)
       end
     end
 
     def histogram(name, *args)
       attributes = extract_attributes(args)
       callable = extract_callable(args)
-      duration = attributes.delete(:duration)
+      value = nil
 
       if callable
-        duration = monotonic_timer{ callable.call }[0]
+        value = monotonic_timer{ callable.call }[0]
       elsif block_given?
-        duration = monotonic_timer{ yield }[0]
+        value = monotonic_timer{ yield }[0]
+      else
+        value = attributes.delete(:duration) || attributes.delete(:value)
       end
 
-      raise 'No duration found' if duration.nil?
+      raise 'No value found' if value.nil?
 
       Honeybadger::Histogram.register(registry, name, attributes).tap do |histogram|
-        histogram.record(duration)
+        histogram.record(value)
       end
     end
 
     def increment_counter(name, *args)
       attributes = extract_attributes(args)
-      by = extract_callable(args)&.call || attributes.delete(:by) || 1
-      by = yield if block_given?
+      callable = extract_callable(args)
+      value = nil
+
+      if callable
+        value = callable.call
+      elsif block_given?
+        value = yield
+      else
+        value = attributes.delete(:by) || attributes.delete(:value) || 1
+      end
 
       Honeybadger::Counter.register(registry, name, attributes).tap do |counter|
-        counter.count(by)
+        counter.count(value)
       end
     end
 
     def decrement_counter(name, *args)
       attributes = extract_attributes(args)
-      by = extract_callable(args)&.call || attributes.delete(:by) || 1
-      by = yield if block_given?
+      callable = extract_callable(args)
+      value = nil
+
+      if callable
+        value = callable.call
+      elsif block_given?
+        value = yield
+      else
+        value = attributes.delete(:by) || attributes.delete(:value) || 1
+      end
 
       Honeybadger::Counter.register(registry, name, attributes).tap do |counter|
-        counter.count(by * -1)
+        counter.count(value * -1)
       end
     end
 
     def gauge(name, *args)
       attributes = extract_attributes(args)
-      value = extract_callable(args)&.call || attributes.delete(:value)
-      value = yield if block_given?
+      callable = extract_callable(args)
+      value = nil
+
+      if callable
+        value = callable.call
+      elsif block_given?
+        value = yield
+      else
+        value = attributes.delete(:value)
+      end
 
       Honeybadger::Gauge.register(registry, name, attributes).tap do |gauge|
         gauge.record(value)

lib/honeybadger/instrumentation_helper.rb

@@ -56,7 +56,7 @@ def time(name, *args)
         metric_instrumentation.time(name, attributes, ->{ callable.call })
       elsif block_given?
         metric_instrumentation.time(name, attributes, ->{ yield })
-      elsif attributes.keys.include?(:duration)
+      else
         metric_instrumentation.time(name, attributes)
       end
     end
@@ -68,38 +68,44 @@ def histogram(name, *args)
         metric_instrumentation.histogram(name, attributes, ->{ callable.call })
       elsif block_given?
         metric_instrumentation.histogram(name, attributes, ->{ yield })
-      elsif attributes.keys.include?(:duration)
+      else
         metric_instrumentation.histogram(name, attributes)
       end
     end
 
     def increment_counter(name, *args)
       attributes = extract_attributes(args)
-      by = extract_callable(args)&.call || attributes.delete(:by) || 1
-      if block_given?
+      callable = extract_callable(args)
+      if callable
+        metric_instrumentation.increment_counter(name, attributes, ->{ callable.call })
+      elsif block_given?
         metric_instrumentation.increment_counter(name, attributes, ->{ yield })
       else
-        metric_instrumentation.increment_counter(name, attributes.merge(by: by))
+        metric_instrumentation.increment_counter(name, attributes)
       end
     end
 
     def decrement_counter(name, *args)
       attributes = extract_attributes(args)
-      by = extract_callable(args)&.call || attributes.delete(:by) || 1
-      if block_given?
+      callable = extract_callable(args)
+      if callable
+        metric_instrumentation.decrement_counter(name, attributes, ->{ callable.call })
+      elsif block_given?
         metric_instrumentation.decrement_counter(name, attributes, ->{ yield })
       else
-        metric_instrumentation.decrement_counter(name, attributes.merge(by: by))
+        metric_instrumentation.decrement_counter(name, attributes)
       end
     end
 
     def gauge(name, *args)
       attributes = extract_attributes(args)
-      value = extract_callable(args)&.call || attributes.delete(:value)
-      if block_given?
+      callable = extract_callable(args)
+      if callable
+        metric_instrumentation.gauge(name, attributes, ->{ callable.call })
+      elsif block_given?
         metric_instrumentation.gauge(name, attributes, ->{ yield })
       else
-        metric_instrumentation.gauge(name, attributes.merge(value: value))
+        metric_instrumentation.gauge(name, attributes)
       end
     end