2020-02-11 21:21:17 +00:00
rule_files :
- mixin.yaml
evaluation_interval : 1m
tests :
- interval : 1m
input_series :
- series : 'up{job="etcd",instance="10.10.10.0"}'
values : '1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0'
- series : 'up{job="etcd",instance="10.10.10.1"}'
values : '1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0'
- series : 'up{job="etcd",instance="10.10.10.2"}'
values : '1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0'
alert_rule_test :
- eval_time : 3m
alertname : etcdInsufficientMembers
- eval_time : 5m
alertname : etcdInsufficientMembers
2020-08-19 18:44:52 +00:00
- eval_time : 12m
2020-02-11 21:21:17 +00:00
alertname : etcdMembersDown
2020-08-19 18:44:52 +00:00
- eval_time : 14m
2020-02-11 21:21:17 +00:00
alertname : etcdMembersDown
exp_alerts :
- exp_labels :
job : etcd
severity : critical
exp_annotations :
2020-08-19 18:44:52 +00:00
message : 'etcd cluster "etcd": members are down (3).'
2020-02-11 21:21:17 +00:00
- eval_time : 7m
alertname : etcdInsufficientMembers
- eval_time : 11m
alertname : etcdInsufficientMembers
exp_alerts :
- exp_labels :
job : etcd
severity : critical
exp_annotations :
message : 'etcd cluster "etcd": insufficient members (1).'
- eval_time : 15m
alertname : etcdInsufficientMembers
exp_alerts :
- exp_labels :
job : etcd
severity : critical
exp_annotations :
message : 'etcd cluster "etcd": insufficient members (0).'
- interval : 1m
input_series :
- series : 'up{job="etcd",instance="10.10.10.0"}'
2020-08-19 18:44:52 +00:00
values : '1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0'
2020-02-11 21:21:17 +00:00
- series : 'up{job="etcd",instance="10.10.10.1"}'
2020-08-19 18:44:52 +00:00
values : '1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0'
2020-02-11 21:21:17 +00:00
- series : 'up{job="etcd",instance="10.10.10.2"}'
2020-08-19 18:44:52 +00:00
values : '1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0'
2020-02-11 21:21:17 +00:00
alert_rule_test :
2020-08-19 18:44:52 +00:00
- eval_time : 14m
2020-02-11 21:21:17 +00:00
alertname : etcdMembersDown
exp_alerts :
- exp_labels :
job : etcd
severity : critical
exp_annotations :
2020-08-19 18:44:52 +00:00
message : 'etcd cluster "etcd": members are down (3).'
2020-02-11 21:21:17 +00:00
- interval : 1m
input_series :
- series : 'up{job="etcd",instance="10.10.10.0"}'
2020-08-19 18:44:52 +00:00
values : '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0'
2020-02-11 21:21:17 +00:00
- series : 'up{job="etcd",instance="10.10.10.1"}'
2020-08-19 18:44:52 +00:00
values : '1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0'
2020-02-11 21:21:17 +00:00
- series : 'etcd_network_peer_sent_failures_total{To="member-1",job="etcd",endpoint="test"}'
2020-08-19 18:44:52 +00:00
values : '0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18'
2020-02-11 21:21:17 +00:00
alert_rule_test :
2020-08-19 18:44:52 +00:00
- eval_time : 13m
2020-02-11 21:21:17 +00:00
alertname : etcdMembersDown
exp_alerts :
- exp_labels :
job : etcd
severity : critical
exp_annotations :
message : 'etcd cluster "etcd": members are down (1).'
- interval : 1m
input_series :
- series : 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.0"}'
values : '0 0 2 0 0 1 0 0 0 0 0 0 0 0 0 0'
- series : 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.1"}'
values : '0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0'
- series : 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.2"}'
values : '0 0 0 0 0 0 0 0'
alert_rule_test :
- eval_time : 10m
alertname : etcdHighNumberOfLeaderChanges
exp_alerts :
- exp_labels :
job : etcd
severity : warning
exp_annotations :
2020-07-28 19:39:59 +00:00
message : 'etcd cluster "etcd": 4 leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.'
2020-02-11 21:21:17 +00:00
- interval : 1m
input_series :
- series : 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.0"}'
values : '0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0'
- series : 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.1"}'
values : '0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0'
- series : 'etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.2"}'
values : '0 0 0 0 0 0 0 0'
alert_rule_test :
- eval_time : 10m
alertname : etcdHighNumberOfLeaderChanges
exp_alerts :