Monitoring and Alerting System

$BASEDIR/monitor/prometheus-package/alert.rules.yml

groups:
- name: example
  rules:
  - alert: HighErrorRate # Alert name
    expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5 # Expression that sets de condition for the alert to be fired
    for: 10m # causes Prometheus to wait for a certain duration between first encountering a new expression output vector element and counting an alert as firing for this element. In this case, Prometheus will check that the alert continues to be active during each evaluation for 10 minutes before firing the alert.
    labels: # Set of additional labels to be attached to the alert.
      severity: page
    annotations: # specifies a set of informational labels that can be used to store longer additional information such as alert descriptions or runbook links
      summary: "High request latency on {{ $labels.instance }}"
      description: "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)"

# INCORRECT: THIS ALERT MAY NOT BE NEVER FIRED
groups:
- name: example
  rules:
  - alert: ExampleAlertIncorrect
    expr: metric > 10
    for: 5m
    labels:
      severity: page
      value: "{{ $value }}" ######## DON'T INCLUDE $value HERE!
    annotations:
      summary: "Instance {{ $labels.instance }}'s metric is too high"

 # CORRECT ALERT
 - alert: ExampleAlert
    expr: metric > 10
    for: 5m
    labels:
      severity: page
    annotations:
      summary: "Instance {{ $labels.instance }}'s metric is {{ $value }}"Combining recording and alerting rules**

groups:
  - name: recording_rules
    interval: 5s
    rules:
      - record: node_exporter:node_filesystem_free:fs_used_percents
        expr: 100 - 100 * ( node_filesystem_free{mountpoint="/"} / node_filesystem_size{mountpoint="/"} )

  - name: alerting_rules
    rules:
      - alert: DiskSpace10%Free
        expr: node_exporter:node_filesystem_free:fs_used_percents >= 90
        # Note that previous expression evaluates the metric defined in the recording rule.
        labels:
          severity: moderate
        annotations:
          summary: "Instance {{ $labels.instance }} is low on disk space"
          description: "{{ $labels.instance }} has only {{ $value }}% free."

[{
  "status": "firing",
  "labels": {
    "alertname": "1234567890",
    "alertType": "alertType1",
    "severity": "warning",
    "component": "KVDS",
    "serviceIp": "172.17.0.2:9993"},
  "annotations": {
    "summary": "Alarm message"}
}]

$BASEDIR/monitor/sendAlert.sh

 ./sendAlert.sh <firing|resolved>  # Alert status
                <warning|critical> # Alert severity
                <AlertId>          # Alert unique ID
                <AlertType>        # Alert Type (indicates an anomalous situation)
                <Component>        # Lx Component that fires alarm
                <ServiceIP>        # hostname/IP:Port
                "<Message>"

$BASEDIR/monitor/alarm_console.sh [simple|extended|json]

Alertname   Starts At                Summary
1234567890  2019-05-29 09:15:36 UTC  Alarm message

./sendAlert.sh resolved warning 1234567890 alertType1 KVDS "172.17.0.2:9993" "Alarm message"

.$BASEDIR-BIN/templates/tm/log4j2.properties

appender.http.type=Http
appender.http.name=http
appender.http.layout.type=PatternLayout
appender.http.layout.pattern=%m
appender.http.url=http://172.17.0.2:9093/api/v1/alerts

...

#Alert Manager
logger.Alert Manager.name=Alert Manager
logger.Alert Manager.level=INFO
logger.Alert Manager.appenderRefs=Alert Manager
logger.Alert Manager.appenderRef.Alert Manager.ref=async

public static void sendAlert(Severity severity, String idAlarm, Status status, String message);

.$BASEDIR-dependencies/TM/common/src/test/java/com/leanxcale/alert/TestAlertSender.java