{"id":555,"date":"2020-09-01T15:51:34","date_gmt":"2020-09-01T07:51:34","guid":{"rendered":"https:\/\/blog.espnlol.com\/?p=555"},"modified":"2022-04-20T18:26:12","modified_gmt":"2022-04-20T10:26:12","slug":"prometheus%e7%9b%91%e6%8e%a7%e7%b3%bb%e7%bb%9f%e7%9a%84%e7%9a%84%e6%8a%a5%e8%ad%a6%e8%a7%84%e5%88%99","status":"publish","type":"post","link":"https:\/\/blog.espnlol.com\/?p=555","title":{"rendered":"prometheus\u76d1\u63a7\u7cfb\u7edf\u7684\u7684\u62a5\u8b66\u89c4\u5219"},"content":{"rendered":"\n

prometheus\u76d1\u63a7\u7cfb\u7edf\u7684\u7684\u62a5\u8b66\u89c4\u5219\u662f\u5728prometheus\u8fd9\u4e2a\u7ec4\u4ef6\u5b8c\u6210\u914d\u7f6e\u7684\u3002 prometheus\u652f\u63012\u79cd\u7c7b\u578b\u7684\u89c4\u5219\uff0c\u8bb0\u5f55\u89c4\u5219\u548c\u62a5\u8b66\u89c4\u5219\uff0c \u8bb0\u5f55\u89c4\u5219\u4e3b\u8981\u662f\u4e3a\u4e86\u7b80\u5199\u62a5\u8b66\u89c4\u5219\u548c\u63d0\u9ad8\u89c4\u5219\u590d\u7528\u7684\uff0c \u62a5\u8b66\u89c4\u5219\u624d\u662f\u771f\u6b63\u53bb\u5224\u5b9a\u662f\u5426\u9700\u8981\u62a5\u8b66\u7684\u89c4\u5219\u3002 \u62a5\u8b66\u89c4\u5219\u4e2d\u662f\u53ef\u4ee5\u4f7f\u7528\u8bb0\u5f55\u89c4\u5219\u7684\u3002<\/p>\n\n\n\n

\u63d0\u4f9b\u4e0b\u6211\u6574\u7406\u7684node-exporter\u7684\u8bb0\u5f55\u89c4\u5219\u548c\u62a5\u8b66\u89c4\u5219\u3002<\/p>\n\n\n\n

groups:\n  - name: node-exporter-record\n    rules:\n    - expr: up{job=~\"node-exporter\"}\n      record: node_exporter:up \n      labels: \n        desc: \"\u8282\u70b9\u662f\u5426\u5728\u7ebf, \u5728\u7ebf1,\u4e0d\u5728\u7ebf0\"\n        unit: \" \"\n        job: \"node-exporter\"\n    - expr: time() - node_boot_time_seconds{}\n      record: node_exporter:node_uptime\n      labels: \n        desc: \"\u8282\u70b9\u7684\u8fd0\u884c\u65f6\u95f4\"\n        unit: \"s\"\n        job: \"node-exporter\"\n##############################################################################################\n#                              cpu                                                           #\n    - expr: (1 - avg by (environment,instance) (irate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}[5m])))  * 100 \n      record: node_exporter:cpu:total:percent\n      labels: \n        desc: \"\u8282\u70b9\u7684cpu\u603b\u6d88\u8017\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n\n    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}[5m])))  * 100 \n      record: node_exporter:cpu:idle:percent\n      labels: \n        desc: \"\u8282\u70b9\u7684cpu idle\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n\n    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"iowait\"}[5m])))  * 100 \n      record: node_exporter:cpu:iowait:percent\n      labels: \n        desc: \"\u8282\u70b9\u7684cpu iowait\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n\n\n    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"system\"}[5m])))  * 100 \n      record: node_exporter:cpu:system:percent\n      labels: \n        desc: \"\u8282\u70b9\u7684cpu system\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n\n    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job=\"node-exporter\",mode=\"user\"}[5m])))  * 100 \n      record: node_exporter:cpu:user:percent\n      labels: \n        desc: \"\u8282\u70b9\u7684cpu user\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n\n    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job=\"node-exporter\",mode=~\"softirq|nice|irq|steal\"}[5m])))  * 100 \n      record: node_exporter:cpu:other:percent\n      labels: \n        desc: \"\u8282\u70b9\u7684cpu \u5176\u4ed6\u7684\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n##############################################################################################\n\n\n##############################################################################################\n#                                    memory                                                  #\n    - expr: node_memory_MemTotal_bytes{job=\"node-exporter\"}\n      record: node_exporter:memory:total\n      labels: \n        desc: \"\u8282\u70b9\u7684\u5185\u5b58\u603b\u91cf\"\n        unit: byte\n        job: \"node-exporter\"\n\n    - expr: node_memory_MemFree_bytes{job=\"node-exporter\"}\n      record: node_exporter:memory:free\n      labels: \n        desc: \"\u8282\u70b9\u7684\u5269\u4f59\u5185\u5b58\u91cf\"\n        unit: byte\n        job: \"node-exporter\"\n\n    - expr: node_memory_MemTotal_bytes{job=\"node-exporter\"} - node_memory_MemFree_bytes{job=\"node-exporter\"}\n      record: node_exporter:memory:used\n      labels: \n        desc: \"\u8282\u70b9\u7684\u5df2\u4f7f\u7528\u5185\u5b58\u91cf\"\n        unit: byte\n        job: \"node-exporter\"\n\n    - expr: node_memory_MemTotal_bytes{job=\"node-exporter\"} - node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n      record: node_exporter:memory:actualused\n      labels: \n        desc: \"\u8282\u70b9\u7528\u6237\u5b9e\u9645\u4f7f\u7528\u7684\u5185\u5b58\u91cf\"\n        unit: byte\n        job: \"node-exporter\"\n\n    - expr: (1-(node_memory_MemAvailable_bytes{job=\"node-exporter\"} \/ (node_memory_MemTotal_bytes{job=\"node-exporter\"})))* 100\n      record: node_exporter:memory:used:percent\n      labels: \n        desc: \"\u8282\u70b9\u7684\u5185\u5b58\u4f7f\u7528\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n\n    - expr: ((node_memory_MemAvailable_bytes{job=\"node-exporter\"} \/ (node_memory_MemTotal_bytes{job=\"node-exporter\"})))* 100\n      record: node_exporter:memory:free:percent\n      labels: \n        desc: \"\u8282\u70b9\u7684\u5185\u5b58\u5269\u4f59\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n##############################################################################################\n#                                   load                                                     #\n    - expr: sum by (instance) (node_load1{job=\"node-exporter\"})\n      record: node_exporter:load:load1\n      labels: \n        desc: \"\u7cfb\u7edf1\u5206\u949f\u8d1f\u8f7d\"\n        unit: \" \"\n        job: \"node-exporter\"\n\n    - expr: sum by (instance) (node_load5{job=\"node-exporter\"})\n      record: node_exporter:load:load5\n      labels: \n        desc: \"\u7cfb\u7edf5\u5206\u949f\u8d1f\u8f7d\"\n        unit: \" \"\n        job: \"node-exporter\"\n\n    - expr: sum by (instance) (node_load15{job=\"node-exporter\"})\n      record: node_exporter:load:load15\n      labels: \n        desc: \"\u7cfb\u7edf15\u5206\u949f\u8d1f\u8f7d\"\n        unit: \" \"\n        job: \"node-exporter\"\n   \n##############################################################################################\n#                                 disk                                                       #\n    - expr: node_filesystem_size_bytes{job=\"node-exporter\" ,fstype=~\"ext4|xfs\"}\n      record: node_exporter:disk:usage:total\n      labels: \n        desc: \"\u8282\u70b9\u7684\u78c1\u76d8\u603b\u91cf\"\n        unit: byte\n        job: \"node-exporter\"\n\n    - expr: node_filesystem_avail_bytes{job=\"node-exporter\",fstype=~\"ext4|xfs\"}\n      record: node_exporter:disk:usage:free\n      labels: \n        desc: \"\u8282\u70b9\u7684\u78c1\u76d8\u5269\u4f59\u7a7a\u95f4\"\n        unit: byte\n        job: \"node-exporter\"\n\n    - expr: node_filesystem_size_bytes{job=\"node-exporter\",fstype=~\"ext4|xfs\"} - node_filesystem_avail_bytes{job=\"node-exporter\",fstype=~\"ext4|xfs\"}\n      record: node_exporter:disk:usage:used\n      labels: \n        desc: \"\u8282\u70b9\u7684\u78c1\u76d8\u4f7f\u7528\u7684\u7a7a\u95f4\"\n        unit: byte\n        job: \"node-exporter\"\n\n    - expr:  (1 - node_filesystem_avail_bytes{job=\"node-exporter\",fstype=~\"ext4|xfs\"} \/ node_filesystem_size_bytes{job=\"node-exporter\",fstype=~\"ext4|xfs\"}) * 100 \n      record: node_exporter:disk:used:percent    \n      labels: \n        desc: \"\u8282\u70b9\u7684\u78c1\u76d8\u7684\u4f7f\u7528\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n\n    - expr: irate(node_disk_reads_completed_total{job=\"node-exporter\"}[1m])\n      record: node_exporter:disk:read:count:rate\n      labels: \n        desc: \"\u8282\u70b9\u7684\u78c1\u76d8\u8bfb\u53d6\u901f\u7387\"\n        unit: \"\u6b21\/\u79d2\"\n        job: \"node-exporter\"\n\n    - expr: irate(node_disk_writes_completed_total{job=\"node-exporter\"}[1m])\n      record: node_exporter:disk:write:count:rate\n      labels: \n        desc: \"\u8282\u70b9\u7684\u78c1\u76d8\u5199\u5165\u901f\u7387\"\n        unit: \"\u6b21\/\u79d2\"\n        job: \"node-exporter\"\n\n    - expr: (irate(node_disk_written_bytes_total{job=\"node-exporter\"}[1m]))\/1024\/1024\n      record: node_exporter:disk:read:mb:rate\n      labels: \n        desc: \"\u8282\u70b9\u7684\u8bbe\u5907\u8bfb\u53d6MB\u901f\u7387\"\n        unit: \"MB\/s\"\n        job: \"node-exporter\"\n\n    - expr: (irate(node_disk_read_bytes_total{job=\"node-exporter\"}[1m]))\/1024\/1024\n      record: node_exporter:disk:write:mb:rate\n      labels: \n        desc: \"\u8282\u70b9\u7684\u8bbe\u5907\u5199\u5165MB\u901f\u7387\"\n        unit: \"MB\/s\"\n        job: \"node-exporter\"\n\n##############################################################################################\n#                                filesystem                                                  #\n    - expr:   (1 -node_filesystem_files_free{job=\"node-exporter\",fstype=~\"ext4|xfs\"} \/ node_filesystem_files{job=\"node-exporter\",fstype=~\"ext4|xfs\"}) * 100 \n      record: node_exporter:filesystem:used:percent    \n      labels: \n        desc: \"\u8282\u70b9\u7684inode\u7684\u5269\u4f59\u53ef\u7528\u7684\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n#############################################################################################\n#                                filefd                                                     #\n    - expr: node_filefd_allocated{job=\"node-exporter\"}\n      record: node_exporter:filefd_allocated:count\n      labels: \n        desc: \"\u8282\u70b9\u7684\u6587\u4ef6\u63cf\u8ff0\u7b26\u6253\u5f00\u4e2a\u6570\"\n        unit: \"%\"\n        job: \"node-exporter\"\n \n    - expr: node_filefd_allocated{job=\"node-exporter\"}\/node_filefd_maximum{job=\"node-exporter\"} * 100 \n      record: node_exporter:filefd_allocated:percent\n      labels: \n        desc: \"\u8282\u70b9\u7684\u6587\u4ef6\u63cf\u8ff0\u7b26\u6253\u5f00\u767e\u5206\u6bd4\"\n        unit: \"%\"\n        job: \"node-exporter\"\n\n#############################################################################################\n#                                network                                                    #\n    - expr: avg by (environment,instance,device) (irate(node_network_receive_bytes_total{device=~\"eth0|eth1|ens33|ens37\"}[1m]))\n      record: node_exporter:network:netin:bit:rate\n      labels: \n        desc: \"\u8282\u70b9\u7f51\u5361eth0\u6bcf\u79d2\u63a5\u6536\u7684\u6bd4\u7279\u6570\"\n        unit: \"bit\/s\"\n        job: \"node-exporter\"\n\n    - expr: avg by (environment,instance,device) (irate(node_network_transmit_bytes_total{device=~\"eth0|eth1|ens33|ens37\"}[1m]))\n      record: node_exporter:network:netout:bit:rate\n      labels: \n        desc: \"\u8282\u70b9\u7f51\u5361eth0\u6bcf\u79d2\u53d1\u9001\u7684\u6bd4\u7279\u6570\"\n        unit: \"bit\/s\"\n        job: \"node-exporter\"\n\n    - expr: avg by (environment,instance,device) (irate(node_network_receive_packets_total{device=~\"eth0|eth1|ens33|ens37\"}[1m]))\n      record: node_exporter:network:netin:packet:rate\n      labels: \n        desc: \"\u8282\u70b9\u7f51\u5361\u6bcf\u79d2\u63a5\u6536\u7684\u6570\u636e\u5305\u4e2a\u6570\"\n        unit: \"\u4e2a\/\u79d2\"\n        job: \"node-exporter\"\n\n    - expr: avg by (environment,instance,device) (irate(node_network_transmit_packets_total{device=~\"eth0|eth1|ens33|ens37\"}[1m]))\n      record: node_exporter:network:netout:packet:rate\n      labels: \n        desc: \"\u8282\u70b9\u7f51\u5361\u53d1\u9001\u7684\u6570\u636e\u5305\u4e2a\u6570\"\n        unit: \"\u4e2a\/\u79d2\"\n        job: \"node-exporter\"\n\n    - expr: avg by (environment,instance,device) (irate(node_network_receive_errs_total{device=~\"eth0|eth1|ens33|ens37\"}[1m]))\n      record: node_exporter:network:netin:error:rate\n      labels: \n        desc: \"\u8282\u70b9\u8bbe\u5907\u9a71\u52a8\u5668\u68c0\u6d4b\u5230\u7684\u63a5\u6536\u9519\u8bef\u5305\u7684\u6570\u91cf\"\n        unit: \"\u4e2a\/\u79d2\"\n        job: \"node-exporter\"\n\n    - expr: avg by (environment,instance,device) (irate(node_network_transmit_errs_total{device=~\"eth0|eth1|ens33|ens37\"}[1m]))\n      record: node_exporter:network:netout:error:rate\n      labels: \n        desc: \"\u8282\u70b9\u8bbe\u5907\u9a71\u52a8\u5668\u68c0\u6d4b\u5230\u7684\u53d1\u9001\u9519\u8bef\u5305\u7684\u6570\u91cf\"\n        unit: \"\u4e2a\/\u79d2\"\n        job: \"node-exporter\"\n      \n    - expr: node_tcp_connection_states{job=\"node-exporter\", state=\"established\"}\n      record: node_exporter:network:tcp:established:count\n      labels: \n        desc: \"\u8282\u70b9\u5f53\u524destablished\u7684\u4e2a\u6570\"\n        unit: \"\u4e2a\"\n        job: \"node-exporter\"\n\n    - expr: node_tcp_connection_states{job=\"node-exporter\", state=\"time_wait\"}\n      record: node_exporter:network:tcp:timewait:count\n      labels: \n        desc: \"\u8282\u70b9timewait\u7684\u8fde\u63a5\u6570\"\n        unit: \"\u4e2a\"\n        job: \"node-exporter\"\n\n    - expr: sum by (environment,instance) (node_tcp_connection_states{job=\"node-exporter\"})\n      record: node_exporter:network:tcp:total:count\n      labels: \n        desc: \"\u8282\u70b9tcp\u8fde\u63a5\u603b\u6570\"\n        unit: \"\u4e2a\"\n        job: \"node-exporter\"\n   \n#############################################################################################\n#                                process                                                    #\n    - expr: node_processes_state{state=\"Z\"}\n      record: node_exporter:process:zoom:total:count\n      labels: \n        desc: \"\u8282\u70b9\u5f53\u524d\u72b6\u6001\u4e3azoom\u7684\u4e2a\u6570\"\n        unit: \"\u4e2a\"\n        job: \"node-exporter\"\n#############################################################################################\n#                                other                                                    #\n    - expr: abs(node_timex_offset_seconds{job=\"node-exporter\"})\n      record: node_exporter:time:offset\n      labels: \n        desc: \"\u8282\u70b9\u7684\u65f6\u95f4\u504f\u5dee\"\n        unit: \"s\"\n        job: \"node-exporter\"\n\n#############################################################################################\n   \n    - expr: count by (instance) ( count by (instance,cpu) (node_cpu_seconds_total{ mode='system'}) ) \n      record: node_exporter:cpu:count\n#<\/pre>\n\n\n\n
node-exporter-alert-rules.yml\ngroups:\n  - name: node-exporter-alert\n    rules:\n    - alert: node-exporter-down\n      expr: node_exporter:up == 0 \n      for: 1m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} \u5b95\u673a\u4e86\"  \n        description: \"instance: {{ $labels.instance }} \\n- job: {{ $labels.job }} \u5173\u673a\u4e86\uff0c \u65f6\u95f4\u5df2\u7ecf1\u5206\u949f\u4e86\u3002\" \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n\n\n    - alert: node-exporter-cpu-high \n      expr:  node_exporter:cpu:total:percent > 80\n      for: 3m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} cpu \u4f7f\u7528\u7387\u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-cpu-iowait-high \n      expr:  node_exporter:cpu:iowait:percent >= 12\n      for: 3m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} cpu iowait \u4f7f\u7528\u7387\u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-load-load1-high \n      expr:  (node_exporter:load:load1) > (node_exporter:cpu:count) * 1.2\n      for: 3m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} load1 \u4f7f\u7528\u7387\u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-memory-high\n      expr:  node_exporter:memory:used:percent > 85\n      for: 3m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} memory \u4f7f\u7528\u7387\u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-disk-high\n      expr:  node_exporter:disk:used:percent > 88\n      for: 10m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} disk \u4f7f\u7528\u7387\u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-disk-read:count-high\n      expr:  node_exporter:disk:read:count:rate > 3000\n      for: 2m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} iops read \u4f7f\u7528\u7387\u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-disk-write-count-high\n      expr:  node_exporter:disk:write:count:rate > 3000\n      for: 2m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} iops write \u4f7f\u7528\u7387\u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n\n\n\n    - alert: node-exporter-disk-read-mb-high\n      expr:  node_exporter:disk:read:mb:rate > 60 \n      for: 2m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} \u8bfb\u53d6\u5b57\u8282\u6570 \u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        instance: \"{{ $labels.instance }}\"\n        value: \"{{ $value }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-disk-write-mb-high\n      expr:  node_exporter:disk:write:mb:rate > 60\n      for: 2m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} \u5199\u5165\u5b57\u8282\u6570 \u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-filefd-allocated-percent-high \n      expr:  node_exporter:filefd_allocated:percent > 80\n      for: 10m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} \u6253\u5f00\u6587\u4ef6\u63cf\u8ff0\u7b26 \u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-network-netin-error-rate-high\n      expr:  node_exporter:network:netin:error:rate > 4\n      for: 1m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} \u5305\u8fdb\u5165\u7684\u9519\u8bef\u901f\u7387 \u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n    - alert: node-exporter-network-netin-packet-rate-high\n      expr:  node_exporter:network:netin:packet:rate > 35000\n      for: 1m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} \u5305\u8fdb\u5165\u901f\u7387 \u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-network-netout-packet-rate-high\n      expr:  node_exporter:network:netout:packet:rate > 35000\n      for: 1m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} \u5305\u6d41\u51fa\u901f\u7387 \u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-network-tcp-total-count-high\n      expr:  node_exporter:network:tcp:total:count > 40000\n      for: 1m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} tcp\u8fde\u63a5\u6570\u91cf \u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-process-zoom-total-count-high \n      expr:  node_exporter:process:zoom:total:count > 10\n      for: 10m\n      labels: \n        severity: info\n      annotations: \n        summary: \"instance: {{ $labels.instance }} \u50f5\u6b7b\u8fdb\u7a0b\u6570\u91cf \u9ad8\u4e8e {{ $value }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"\n\n    - alert: node-exporter-time-offset-high\n      expr:  node_exporter:time:offset > 0.03\n      for: 2m\n      labels: \n        severity: info\n      annotations:\n        summary: \"instance: {{ $labels.instance }} {{ $labels.desc }}  {{ $value }} {{ $labels.unit }}\"  \n        description: \"\"    \n        value: \"{{ $value }}\"\n        instance: \"{{ $labels.instance }}\"\n        grafana: \"http:\/\/xxxxxxxx.com\/d\/node-exporter\/node-exporter?orgId=1&var-instance={{ $labels.instance }} \"\n        console: \"https:\/\/ecs.console.aliyun.com\/#\/server\/{{ $labels.instanceid }}\/detail?regionId=cn-beijing\"\n        cloudmonitor: \"https:\/\/cloudmonitor.console.aliyun.com\/#\/hostDetail\/chart\/instanceId={{ $labels.instanceid }}&system=&region=cn-beijing&aliyunhost=true\"\n        id: \"{{ $labels.instanceid }}\"\n        type: \"aliyun_meta_ecs_info\"<\/pre>\n","protected":false},"excerpt":{"rendered":"
prometheus\u76d1\u63a7\u7cfb\u7edf\u7684\u7684\u62a5\u8b66\u89c4\u5219\u662f\u5728prometheus\u8fd9\u4e2a\u7ec4\u4ef6\u5b8c\u6210\u914d\u7f6e … \u7ee7\u7eed\u9605\u8bfb →<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[26],"tags":[],"class_list":["post-555","post","type-post","status-publish","format-standard","hentry","category-k8s"],"_links":{"self":[{"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=\/wp\/v2\/posts\/555","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=555"}],"version-history":[{"count":1,"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=\/wp\/v2\/posts\/555\/revisions"}],"predecessor-version":[{"id":556,"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=\/wp\/v2\/posts\/555\/revisions\/556"}],"wp:attachment":[{"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=555"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=555"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.espnlol.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=555"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}