通过consul注册自动加入prometheus监控

wget https://releases.hashicorp.com/consul/1.10.3/consul_1.10.3_linux_amd64.zip #下载
mv consul_1.10.3_linux_amd64.zip /usr/local/
unzip consul_1.10.3_linux_amd64.zip
mkdir -p /data/consul /opt/logs/consul /etc/consul.d

nohup ./consul agent -dev -client=192.168.48.139 -data-dir=/data/consul -bind=192.168.48.139 -log-file=/opt/logs/consul/consul.log -log-rotate-max-files=3 & 

设置system管理的话编辑文件 cat /etc/systemd/system/consul.service

[Unit]
Description=Consul Server
Documentation=https://www.consul.io/
Wants=network-online.target
After=network-online.target

[Service]
User=root
Group=root
ExecStart=/usr/bin/consul agent -dev -client=服务端ip -data-dir=/data/consul -bind=服务端ip \
          -log-file=/opt/logs/consul/consul.log -log-rotate-max-files=3 -config-dir=/etc/consul.d
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl start consul

启动命令，如果自己主机多ip，指定一个ip启动，使用0.0.0.0会报错

该服务会默认占用8300,8301,8302,5500,8502,8600端口，注意不要冲突

启动后访问自己的8500端口，进入web页面

设置批量发现node_exporter 从服务端主动加入其他节点

在prometheus的配置文件jobs内加入

  - job_name: 'consul_K8S1_node' #在prometheus中显示为consul_K8S_PNG2_node组
    consul_sd_configs:
      - server: 'ip:8500' #consul服务端的ip
        services:
          - 'K8S1_node'  #要监控的具体服务名称，自定义
    relabel_configs:
      - source_labels: [__meta_consul_service]
        action: keep #匹配正则
        regex: 'K8S1_node' #匹配上面对应的service
      - source_labels: [__address__]
        target_label: instance
      - target_label: instance_name #添加自定义标签，加入prometheus后携带，自定义键值对
        replacement: 'K8S1节点'

  - job_name: 'consul_K8S2_node' #多个组分开写
    consul_sd_configs:
      - server: 'ip:8500'
        services:
          - 'K8S2_node'
    relabel_configs:
      - source_labels: [__meta_consul_service]
        action: keep
        regex: 'K8S2_node'
      - source_labels: [__address__]
        target_label: instance
      - target_label: instance_name
        replacement: 'K8S-PNG1节点'

然后在需要加入的节点部署node_exporter并启动

在服务端添加客户端的脚本

#!/bin/bash

# Consul Server 地址
CONSUL_SERVER="http://172.11.11.11:8500"

# K8S2 服务 IP 列表
K8S2_IPS=(
  "172.30.35.196"
  "172.30.35.197"
)

# K8S1 服务 IP 列表
K8S1_IPS=(
  "172.30.35.196"
)

# 注册 K8S2 节点
for IP in "${K8S2_IPS[@]}"; do
  SERVICE_ID="node-exporter-k8s2-$IP"
  curl --request PUT --data "{
    \"ID\": \"$SERVICE_ID\",
    \"Name\": \"K8S2_node\",
    \"Address\": \"$IP\",
    \"Port\": 8089,
    \"Tags\": [\"monitoring\"],
    \"Check\": {
      \"HTTP\": \"http://$IP:'node端口'/metrics\",
      \"Interval\": \"10s\"
    }
  }" "$CONSUL_SERVER/v1/agent/service/register"
  echo "Registered K8S2 node: $IP"
done

# 注册 K8S1 节点
for IP in "${K8S1_IPS[@]}"; do
  SERVICE_ID="node-exporter-k8s1-$IP"
  curl --request PUT --data "{
    \"ID\": \"$SERVICE_ID\",
    \"Name\": \"K8S1_node\",
    \"Address\": \"$IP\",
    \"Port\": 8089,
    \"Tags\": [\"monitoring\"],
    \"Check\": {
      \"HTTP\": \"http://$IP:'node端口'/metrics\",
      \"Interval\": \"10s\"
    }
  }" "$CONSUL_SERVER/v1/agent/service/register"
  echo "Registered K8S1 node: $IP"
done

或者手动执行添加

curl --request PUT --data '{ \
  "ID": "node-exporter-1", \
  "Name": "node-exporter", \
  "Address": "客户端ip", \
  "Port": 'node端口', \
  "Tags": ["monitoring"], \
  "Check": { \
    "HTTP": "http://客户端ip:node端口/metrics", \
    "Interval": "10s" \
  } \
}' http://服务端id:8500/v1/agent/service/register

注册成功是绿色

查看prometheus页面

已加入成功

删除脚本

#!/bin/bash

# Consul Server 地址
CONSUL_SERVER="http://172.31.14.142:8500"

# PNG2 服务 IP 列表
PNG2_IPS=(
)

# PNG1 服务 IP 列表
PNG1_IPS=(
  "172.30.35.196"
  "172.30.35.76"
)

# 注销 K8S2 节点
for IP in "${K8S2_IPS[@]}"; do
  SERVICE_ID="node-exporter-k8s2-$IP"
  curl --request PUT "$CONSUL_SERVER/v1/agent/service/deregister/$SERVICE_ID"
  echo "Deregistered K8S2 node: $SERVICE_ID"
done

# 注销 K8S1 节点
for IP in "${K8S1_IPS[@]}"; do
  SERVICE_ID="node-exporter-k8s1-$IP"
  curl --request PUT "$CONSUL_SERVER/v1/agent/service/deregister/$SERVICE_ID"
  echo "Deregistered K8S1 node: $SERVICE_ID"
done