监控常用指标
- node_exporter 标准性能指标
# CPU使用率
100 - (avg(irate(node_cpu_seconds_total{mode='idle', instance="10.12.69.173:9100"}[1m])) *100)
# 内存使用率
100 - (((node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes)/node_memory_MemTotal_bytes) * 100)
# 磁盘使用率
100 - (node_filesystem_free_bytes{fstype=~"ext4|xfs",mountpoint="/",instance="10.12.69.173:9100"} / node_filesystem_size_bytes{fstype=~"ext4|xfs",mountpoint="/",instance="10.12.69.173:9100"} *100)
# 网络传输速率
node_network_receive_bytes_total{device="eth0",instance="10.12.69.173:9100",job="node_exporter"}
# CPU平均负载
node_load5{instance="10.12.69.173:9100",job="node_exporter"}
- mysql 监控性能指标
# mysql存活指标（0代表down，1代表up）
mysql_up
# 每秒查询次数指标
rate(mysql_global_status_slow_queries[5m])
# 连接数指标
rate(mysql_global_status_threads_connected[5m])
# 可用连接数
rate(mysql_global_variables_max_connections - mysql_global_status_threads_connected[5m])
# mysql主从复制I/O线程
mysql_slave_status_slave_sql_running
# mysql主从延迟
rate(mysql_slave_status_seconds_behind_master[5m])
- Pod性能指标
# Pod的cpu使用率
container_memory_usage_bytes{container_name!=""} / container_spec_memory_limit_bytes{container_name!=""} * 100 != +Inf
# Pod内存使用率
sum by (pod_name)( rate(container_cpu_usage_seconds_total{image!=""}[1m] ) ) * 100
创建规则
Node告警规则
[root@prometheus Prometheus]# mkdir rules
[root@prometheus Prometheus]# vim rules/node_rules.yml
groups:
- name: test
rules:
- alert: 内存使用率过高
expr: 100-(node_memory_Buffers_bytes+node_memory_Cached_bytes+node_memory_MemFree_bytes)/node_memory_MemTotal_bytes*100 > 90
for: 30s # 告警持续时间，超过这个时间才会发送给alertmanager
labels:
severity: warning
annotations:
summary: "Instance {{ $labels.instance }} 内存使用率过高"
description: "{{ $labels.instance }} of job {{$labels.job}}内存使用率超过80%,当前使用率[{{ $value }}]."
- alert: cpu使用率过高
expr: 100-avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by(instance)*100 > 90
for: 30s
labels:
severity: warning
annotations:
summary: "Instance {{ $labels.instance }} cpu使用率过高"
description: "{{ $labels.instance }} of job {{$labels.job}}cpu使用率超过80%,当前使用率[{{ $value }}]."
MySQL告警规则
- 编写指标
[root@prometheus ~]# vim /usr/local/Prometheus/prometheus/rules/mysql_rules.yml
groups:
- name: mysql_rules
rules:
- record: mysql:status
expr: mysql_up{instance=~".*9104"}
- record: mysql:uptime
expr: mysql_global_status_uptime{job="mysqld_exporter"}
- record: mysql:mysql_threads_connected
expr: mysql_global_status_threads_connected{job="mysqld_exporter"}
- record: mysql:mysql_threads_running
expr: mysql_global_status_threads_running{job="mysqld_exporter"}
- record: mysql:mysql_aborted_connects
expr: increase(mysql_global_status_aborted_connects{job="mysqld_exporter"}[2m])
- record: mysql:mysql_slow_queries
expr: increase(mysql_global_status_slow_queries{job="mysqld_exporter"}[2m])
- record: mysql:mysql_table_locks
expr: increase(mysql_global_status_table_locks_waited{job="mysqld_exporter"}[2m])
- record: mysql:mysql_qps
expr: rate(mysql_global_status_queries{job="mysqld_exporter"}[2m])
- 编写告警规则
[root@prometheus ~]# vim /usr/local/Prometheus/prometheus/rules/
groups:
- name: mysql_alerts
rules:
- alert: MySQL_Down_Alert
expr: mysql:status==0
for: 1m
labels:
metric_type: db_monitor
resource: db
severity: critical
annotations:
summary: 主机 {{ $labels.nodename }} 数据库 出现异常!
description: 主机 {{ $labels.nodename }} 上的 {{ $labels.job }} 可能存在异常，请检查!
- alert: MySQL_uptime_Alert
expr: mysql:uptime<1
for: 1m
labels:
metric_type: db_monitor
resource: db
severity: critical
annotations:
summary: 主机 {{ $labels.nodename }} 数据库 出现异常!
description: 主机 {{ $labels.nodename }} 数据库状态异常，请检查!
- alert: MySQL_threads_connected_Alert
expr: mysql_threads_connected > 100
for: 1m
labels:
metric_type: db_monitor
resource: db
severity: critical
annotations:
summary: 主机 {{ $labels.nodename }} 上的数据库指标 threads_connected 超出阈值!
description: 主机 {{ $labels.nodename }} 上的数据库指标 threads_connected 超出阈值，当前值为{{humanize $value}}，请检查!
- alert: MySQL_threads_running_Alert
expr: mysql:mysql_threads_running > 200
for: 1m
labels:
metric_type: db_monitor
resource: db
severity: critical
annotations:
summary: 主机 {{ $labels.nodename }} 上的数据库指标 threads_running 超出阈值!
description: 主机 {{ $labels.nodename }} 上的数据库指标 threads_connected 超出阈值，当前值为{{humanize $value}}，请检查!
- alert: MySQL_aborted_connects_Alert
expr: mysql:mysql_aborted_connects > 10
for: 1m
labels:
metric_type: db_monitor
resource: db
severity:
annotations:
summary: 主机 {{ $labels.nodename }} 上的数据库指标 aborted_connects 超出阈值
description: 主机 {{ $labels.nodename }} 上的数据库指标 aborted_connects 超出阈值，当前值为{{humanize $value}}，
- alert
expr: mysql:mysql_slow_queries > 1
for: 1m
labels:
metric_type: db_monitor
resource: db
severity: critical
annotations:
summary: 主机 {{ $labels.nodename }} 上的数据库指标 slow_queries 超出阈值!
description: 主机 {{ $labels.nodename }} 上的数据库指标 slow_queries 超出阈值，当前值为{{humanize $value}}，请检查!
- alert: MySQL_table_locks_Alert
expr: mysql:mysql_table_locks > 1
for: 1m
labels:
metric_type: db_monitor
resource: db
severity: critical
annotations:
ummary: 主机 {{ $labels.nodename }} 上的数据库指标 table_locks 超出阈值
description: 主机 {{ $labels.nodename }} 上的数据库指标 table_locks 超出阈值，当前值为{{humanize $value}}，请检查!
- alert: MySQL_qps_Alert
expr: mysql:mysql_qps > 500
for: 1m
labels:
metric_type: db_monitor
resource: db
severity: critical
annotations:
summary: 主机 {{ $labels.nodename }} 上的数据库指标 qps 超出阈值!
description: 主机 {{ $labels.nodename }} 上的数据库指标 qps 超出阈值，当前值为{{humanize $value}}，请检查!
Pod告警规则
groups:
- name: noah_pod.rules
rules:
- alert: PodMemUsage
expr: container_memory_usage_bytes{container_name!=""} / container_spec_memory_limit_bytes{container_name!=""} *100 != +Inf > 80
for: 2m
labels:
severity: warning
annotations:
summary: "{{$labels.name}}: Pod High Mem usage detected"
description: "{{$labels.name}}: Pod Mem is above 80% ,(current value is: {{ $value }})"
- alert: PodCpuUsage
expr: sum by (pod_name)( rate(container_cpu_usage_seconds_total{image!=""}[1m] ) ) * 100 > 80
for: 2m
labels:
severity: warning
annotations:
summary: "{{$labels.name}}: Pod High CPU usage detected"
description: "{{$labels.name}}: Pod CPU is above 80% ,(current value is: {{ $value }})"
