[Logstash] Adding Health Report to Logstash Integration (elastic#12677)

* third time is the charm

packages/logstash/changelog.yml

@@ -1,9 +1,9 @@
 # newer versions go on top
 - version: "2.5.0"
   changes:
-    - description: Allow the usage of deprecated log input and support for stack 9.0
+    - description: Adding data and dashboards from Logstash Health Report
       type: enhancement
-      link: https://github.com/elastic/integrations/pull/12503
+      link: https://github.com/elastic/integrations/pull/12464
 - version: "2.4.12"
   changes:
     - description: Update documentation for GA of agent based monitoring

packages/logstash/data_stream/health_report/agent/stream/stream.yml.hbs

@@ -0,0 +1,63 @@
+config_version: "1"
+interval: {{period}}
+resource.url: "{{url}}/_health_report"
+{{#if resource_ssl}}
+resource.ssl:
+  {{resource_ssl}}
+{{/if}}
+
+{{#if username}}
+auth.basic.user: {{escape_string username}}
+{{/if}}
+{{#if password}}
+auth.basic.password: {{escape_string password}}
+{{/if}}
+{{#if condition}}
+condition: {{ condition }}
+{{/if}}
+
+redact: 
+  fields: ~
+
+program: |
+  get(state.url)
+  .as(resp, bytes(resp.Body)
+  .decode_json().as(body,
+    [   
+        [{
+            "logstash.node": {
+            "name":body.name,
+            "address":body.http_address,
+            "uuid":body.id,
+            "version":body.version,
+            "status":body.status,
+            "symptom":body.symptom,
+            },
+        }],
+        body.indicators.pipelines.indicators.as(pipelines, // pipelines = body.indicators.pipelines.indicators
+              pipelines.map(pipeline_name, pipelines[pipeline_name].as(pipeline, { // pipeline = pipelines[pipeline_name]
+                    "logstash": {
+                        "node": {
+                            "name": body.name,
+                            "version": body.version,
+                            "address": body.http_address,
+                            "uuid": body.id
+                        },
+                        "pipeline": {
+                            "id":pipeline_name,
+                            "status":pipeline.status,
+                            "symptom":pipeline.symptom,
+                            "state":pipeline.details.status.state,
+                            "flow": pipeline.details.flow,
+                            "diagnosis": has(pipeline.diagnosis) ? pipeline.diagnosis[0] : {},
+                            "impacts": has(pipeline.impacts) ? pipeline.impacts[0] : {},
+                        }
+                    }
+                }
+            ))
+        )
+    ].as(entries, {
+        "events": entries.flatten()
+    })
+    )
+  )

packages/logstash/data_stream/health_report/fields/base-fields.yml

@@ -0,0 +1,9 @@
+- name: data_stream.type
+  external: ecs
+- name: data_stream.dataset
+  external: ecs
+- name: data_stream.namespace
+  external: ecs
+- name: service.hostname
+  type: keyword
+  description: Hostname of the service

packages/logstash/data_stream/health_report/fields/ecs.yml

@@ -0,0 +1,24 @@
+- name: '@timestamp'
+  external: ecs
+- name: service.id
+  external: ecs
+- name: service.type
+  external: ecs
+- name: service.version
+  external: ecs
+- name: service.address
+  external: ecs
+- name: service.name
+  external: ecs
+- name: process.pid
+  external: ecs
+- name: ecs.version
+  external: ecs
+- name: event.dataset
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: event.module
+  external: ecs
+- name: error.message
+  external: ecs

packages/logstash/data_stream/health_report/fields/fields.yml

@@ -0,0 +1,90 @@
+- name: logstash
+  type: group
+  fields:
+    - name: node
+      type: group
+      fields:
+        - name: name
+          type: keyword
+        - name: version
+          type: keyword
+        - name: address
+          type: keyword
+        - name: symptom
+          type: keyword
+          multi_fields:
+            - name: text
+              type: match_only_text
+        - name: uuid
+          type: keyword
+        - name: status
+          type: keyword
+    - name: pipeline
+      type: group
+      fields:
+        - name: id
+          type: keyword
+        - name: status
+          type: keyword
+        - name: state
+          type: keyword
+        - name: symptom
+          type: keyword
+          multi_fields:
+            - name: text
+              type: match_only_text
+        - name: diagnosis
+          type: group
+          fields:
+            - name: id
+              type: keyword
+            - name: cause
+              type: keyword
+              multi_fields:
+                - name: text
+                  type: match_only_text
+            - name: action
+              type: keyword
+              multi_fields:
+                - name: text
+                  type: match_only_text
+            - name: help_url
+              type: keyword
+              format: url
+              multi_fields:
+                - name: text
+                  type: match_only_text
+        - name: impacts
+          type: group
+          fields:
+            - name: id
+              type: keyword
+            - name: severity
+              type: short
+            - name: description
+              type: keyword
+              multi_fields:
+                - name: text
+                  type: match_only_text
+            - name: impact_areas
+              type: keyword
+        - name: flow
+          type: group
+          fields:
+            - name: worker_utilization
+              type: group
+              fields:
+                - name: current
+                  type: float
+                - name: last_1_hour
+                  type: float
+                - name: last_5_minutes
+                  type: float
+                - name: last_15_minutes
+                  type: float
+                - name: lifetime
+                  type: float
+                - name: last_1_minute
+                  type: float
+                - name: last_24_hours
+                  type: float

packages/logstash/data_stream/health_report/manifest.yml

@@ -0,0 +1,20 @@
+title: "Health Report"
+type: metrics
+dataset: logstash.health_report
+release: beta
+elasticsearch:
+  index_template:
+    mappings:
+      dynamic: false
+streams:
+  - input: cel
+    title: "Collect Health Report"
+    description: "Health Report for Logstash instance"
+    vars:
+      - name: period
+        type: text
+        title: Period
+        multi: false
+        required: true
+        show_user: true
+        default: 30s

packages/logstash/data_stream/health_report/sample_event.json

@@ -0,0 +1,41 @@
+{
+    "logstash": {
+        "pipeline": {
+            "symptom": "The pipeline is unhealthy; 1 area is impacted and 1 diagnosis is available",
+            "diagnosis": {
+            "help_url": "https://www.elastic.co/guide/en/logstash/8.17/health-report-pipeline-status.html#terminated",
+            "action": "view logs to determine the cause of abnormal pipeline shutdown",
+            "cause": "pipeline is not running, likely because it has encountered an error",
+            "id": "logstash:health:pipeline:status:diagnosis:terminated"
+            },
+            "state": "TERMINATED",
+            "impacts": {
+            "severity": 1,
+            "impact_areas": [
+                "pipeline_execution"
+            ],
+            "description": "the pipeline is not currently processing",
+            "id": "logstash:health:pipeline:status:impact:not_processing"
+            },
+            "id": "bad",
+            "flow": {
+                "worker_utilization": {
+                "current": 0.000001826,
+                "last_24_hours": 0.000001826,
+                "last_5_minutes": 0.000001826,
+                "last_1_hour": 0.000001826,
+                "last_15_minutes": 0.000001826,
+                "lifetime": 0.000001826,
+                "last_1_minute": 0.000001826
+                }
+            },
+            "status": "red"
+            },
+        "node": {
+            "address": "0.0.0.0:9600",
+            "name": "9cbebe0b66d9",
+            "version": "8.17.1",
+            "uuid": "1a3d4358-0d0a-4b82-8cf7-21f59ea592e8"
+        }
+    }
+}

packages/logstash/data_stream/node/manifest.yml

@@ -8,6 +8,7 @@ elasticsearch:
 streams:
   - input: logstash/metrics
     title: Logstash node metrics
+    enabled: false
     description: Collect Logstash node metrics
     vars:
       - name: period

packages/logstash/data_stream/node_cel/manifest.yml

@@ -7,7 +7,6 @@ elasticsearch:
       dynamic: false
 streams:
   - input: cel
-    enabled: false
     title: "Collect Node Metrics"
     description: "Collect Metrics related to Nodes running Logstash"
     template_path: cel.yml.hbs

packages/logstash/data_stream/node_stats/manifest.yml

@@ -8,6 +8,7 @@ elasticsearch:
 streams:
   - input: logstash/metrics
     title: Logstash node stats metrics
+    enabled: false
     description: Collect Logstash node stats metrics
     vars:
       - name: period

packages/logstash/data_stream/pipeline/manifest.yml

@@ -1,14 +1,12 @@
 type: metrics
 title: Logstash pipeline
-release: experimental
 elasticsearch:
   index_mode: "time_series"
   index_template:
     mappings:
       dynamic: false
 streams:
   - input: cel
-    enabled: false
     title: "Collect Pipeline Metrics"
     description: "Collect Metrics related to Logstash Pipeline usage"
     template_path: cel.yml.hbs

packages/logstash/data_stream/plugins/manifest.yml

@@ -1,14 +1,12 @@
 type: metrics
 title: Logstash plugins
-release: experimental
 elasticsearch:
   index_mode: "time_series"
   index_template:
     mappings:
       dynamic: false
 streams:
   - input: cel
-    enabled: false
     title: "Collect Plugin Metrics"
     description: "Collect metrics for Logstash plugin use. Note that large pipelines will increase the volume of plugin metrics, and a slower rate of collection may be appropriate"
     template_path: cel.yml.hbs