225 lines
6.6 KiB
Cheetah
225 lines
6.6 KiB
Cheetah
# Documentation : https://prometheus.io/docs/alerting/configuration/
|
|
|
|
global:
|
|
resolve_timeout: 5m
|
|
slack_api_url: SLACK_WEBHOOK_ID
|
|
# The smarthost and SMTP sender used for mail notifications.
|
|
# smtp_smarthost: 'localhost:25'
|
|
# smtp_from: 'alertmanager@domain.com'
|
|
|
|
# The root route on which each incoming alert enters.
|
|
route:
|
|
# The root route must not have any matchers as it is the entry point for
|
|
# all alerts. It needs to have a receiver configured so alerts that do not
|
|
# match any of the sub-routes are sent to someone.
|
|
receiver: 'slack-default'
|
|
|
|
# The labels by which incoming alerts are grouped together. For example,
|
|
# multiple alerts coming in for cluster=A and alertname=LatencyHigh would
|
|
# be batched into a single group.
|
|
# group_by: ['alertname', 'cluster']
|
|
group_by: ['instance']
|
|
|
|
# When a new group of alerts is created by an incoming alert, wait at
|
|
# least 'group_wait' to send the initial notification.
|
|
# This way ensures that you get multiple alerts for the same group that start
|
|
# firing shortly after another are batched together on the first
|
|
# notification.
|
|
group_wait: 1m
|
|
|
|
# When the first notification was sent, wait 'group_interval' to send a batch
|
|
# of new alerts that started firing for that group.
|
|
group_interval: 1m
|
|
|
|
# If an alert has successfully been sent, wait 'repeat_interval' to
|
|
# resend them.
|
|
repeat_interval: 1h
|
|
|
|
|
|
# All the above attributes are inherited by all child routes and can
|
|
# overwritten on each.
|
|
|
|
# The child route trees.
|
|
# routes:
|
|
# This routes performs a regular expression match on alert labels to
|
|
# catch alerts that are related to a list of services.
|
|
# - match_re:
|
|
# service: ^(foo1|foo2|baz)$
|
|
# receiver: team-X-mails
|
|
|
|
routes:
|
|
- match:
|
|
severity: "low"
|
|
instance: "Hourly GMV"
|
|
type: "lower than static threshold"
|
|
receiver: slack-gmv-static-low
|
|
|
|
- match:
|
|
severity: "high"
|
|
instance: "Hourly GMV"
|
|
type: "lower than static threshold"
|
|
receiver: slack-gmv-static-high
|
|
|
|
- match:
|
|
severity: "low"
|
|
instance: "Hourly GMV"
|
|
receiver: slack-gmv-low
|
|
|
|
- match:
|
|
severity: "high"
|
|
instance: "Hourly GMV"
|
|
receiver: slack-gmv-high
|
|
|
|
- match:
|
|
instance: "Hourly GMV test"
|
|
receiver: slack-gmv-low
|
|
|
|
- match:
|
|
severity: "low"
|
|
receiver: slack-generic-low
|
|
|
|
- match:
|
|
severity: "medium"
|
|
receiver: slack-generic-medium
|
|
|
|
- match:
|
|
severity: "high"
|
|
receiver: slack-generic-high
|
|
|
|
- match:
|
|
severity: "critical"
|
|
receiver: slack-generic-critical
|
|
|
|
|
|
# The service has a sub-route for critical alerts, any alerts
|
|
# that do not match, i.e. severity != critical, fall-back to the
|
|
# parent node and are sent to 'team-X-mails'
|
|
# routes:
|
|
# - match:
|
|
# severity: critical
|
|
# receiver: team-X-pager
|
|
|
|
|
|
# Inhibition rules allow to mute a set of alerts given that another alert is
|
|
# firing.
|
|
# We use this to mute any warning-level notifications if the same alert is
|
|
# already critical.
|
|
inhibit_rules:
|
|
|
|
- source_match:
|
|
severity: 'medium'
|
|
target_match:
|
|
severity: 'low'
|
|
# Apply inhibition if the alertname is the same.
|
|
equal: ['instance']
|
|
|
|
- source_match:
|
|
severity: 'high'
|
|
target_match:
|
|
severity: 'medium'
|
|
# Apply inhibition if the alertname is the same.
|
|
equal: ['instance']
|
|
|
|
- source_match:
|
|
severity: 'high'
|
|
target_match:
|
|
severity: 'low'
|
|
# Addition for GMV, which has no "medium" severity
|
|
equal: ['instance']
|
|
|
|
- source_match:
|
|
severity: 'critical'
|
|
target_match:
|
|
severity: 'high'
|
|
# Apply inhibition if the alertname is the same.
|
|
equal: ['instance']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
receivers:
|
|
- name: 'slack-gmv-static-low'
|
|
slack_configs:
|
|
- api_url: SLACK_WEBHOOK_ID
|
|
send_resolved: true
|
|
username: 'Chouette Vigilante [severity - low]'
|
|
icon_emoji: ':owl:'
|
|
title: ':mostly_sunny: {{ .Status|toUpper }} issue on [ {{ .CommonLabels.instance }} ]'
|
|
text: "{{ .CommonAnnotations.description }}"
|
|
|
|
- name: 'slack-gmv-static-high'
|
|
slack_configs:
|
|
- api_url: SLACK_WEBHOOK_ID
|
|
send_resolved: true
|
|
username: 'Chouette Vigilante [severity - high]'
|
|
icon_emoji: ':fallen_leaf:'
|
|
title: ':tornado: {{ .Status|toUpper }} issue on [ {{ .CommonLabels.instance }} ]'
|
|
text: "{{ .CommonAnnotations.description }}"
|
|
|
|
- name: 'slack-gmv-low'
|
|
slack_configs:
|
|
- api_url: SLACK_WEBHOOK_ID
|
|
send_resolved: true
|
|
username: 'Chouette Vigilante [severity - low]'
|
|
icon_emoji: ':owl:'
|
|
title: ':mostly_sunny: {{ .Status|toUpper }} issue on [ {{ .CommonLabels.instance }} ]'
|
|
text: "{{ .CommonAnnotations.description }}
|
|
\n
|
|
\n{{ .CommonAnnotations.query }}"
|
|
|
|
- name: 'slack-gmv-high'
|
|
slack_configs:
|
|
- api_url: SLACK_WEBHOOK_ID
|
|
send_resolved: true
|
|
username: 'Chouette Vigilante [severity - high]'
|
|
icon_emoji: ':fallen_leaf:'
|
|
title: ':tornado: {{ .Status|toUpper }} issue on [ {{ .CommonLabels.instance }} ]'
|
|
text: "{{ .CommonAnnotations.description }}
|
|
\n
|
|
\n{{ .CommonAnnotations.query }}"
|
|
|
|
- name: 'slack-generic-low'
|
|
slack_configs:
|
|
- api_url: SLACK_WEBHOOK_ID
|
|
send_resolved: true
|
|
username: 'Chouette Vigilante [severity - low]'
|
|
icon_emoji: ':owl:'
|
|
title: ':mostly_sunny: {{ .Status|toUpper }} issue on [ {{ .CommonLabels.instance }} ]'
|
|
text: "*{{ .CommonLabels.instance }}* had a *{{ .CommonLabels.type }}* issue for a few seconds."
|
|
|
|
- name: 'slack-generic-medium'
|
|
slack_configs:
|
|
- api_url: SLACK_WEBHOOK_ID
|
|
send_resolved: true
|
|
username: 'Chouette Vigilante [severity - medium]'
|
|
icon_emoji: ':owl:'
|
|
title: ':sun_behind_rain_cloud: {{ .Status|toUpper }} issue on [ {{ .CommonLabels.instance }} ]'
|
|
text: "*{{ .CommonLabels.instance }}* had a *{{ .CommonLabels.type }}* issue for more than 5 minutes."
|
|
|
|
- name: 'slack-generic-high'
|
|
slack_configs:
|
|
- api_url: SLACK_WEBHOOK_ID
|
|
send_resolved: true
|
|
username: 'Chouette Vigilante [severity - high]'
|
|
icon_emoji: ':fallen_leaf:'
|
|
title: ':tornado: {{ .Status|toUpper }} issue on {{ .CommonLabels.instance }}'
|
|
text: "*{{ .CommonLabels.instance }}* had a *{{ .CommonLabels.type }}* issue for an hour"
|
|
|
|
- name: 'slack-generic-critical'
|
|
slack_configs:
|
|
- api_url: SLACK_WEBHOOK_ID
|
|
send_resolved: true
|
|
username: 'Chouette Vigilante [severity - critical]'
|
|
icon_emoji: ':fire:'
|
|
title: ':boom: {{ .Status|toUpper }} issue on {{ .CommonLabels.instance }}'
|
|
text: "*{{ .CommonLabels.instance }}* had a *{{ .CommonLabels.type }}* issue for more than 12 hours"
|
|
|
|
- name: 'slack-default'
|
|
slack_configs:
|
|
- api_url: SLACK_WEBHOOK_ID
|
|
send_resolved: true
|
|
username: 'Chouette Vigilante [default]'
|
|
icon_emoji: ':owl:'
|