1、部署prometheus server
官网下载地址：https://prometheus.io/download/
[root@ubuntu2004 ~]#tar xf prometheus-2.40.2.linux-amd64.tar.gz -C /usr/local/
[root@ubuntu2004 ~]#cd /usr/local/
[root@ubuntu2004 local]#mv prometheus-2.40.2.linux-amd64 prometheus-2.40.2
[root@ubuntu2004 local]#ln -s /usr/local/prometheus-2.40.2 /usr/local/prometheus
[root@ubuntu2004 local]#cd prometheus
[root@ubuntu2004 prometheus]#ln -s /usr/local/prometheus/prometheus /usr/local/bin/
[root@ubuntu2004 prometheus]#cp prometheus.yml{,.bak}
#基于systemd管理启动、停止
[root@ubuntu2004 prometheus]#vim /usr/lib/systemd/system/prometheus.service
[Unit]
Description=Monitoring system and time series database
Documentation=https://prometheus.io/docs/introduction/overview/
[Service]
Restart=always
#User=prometheus
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml
ExecReload=/bin/kill -HUP $MAINPID
TimeoutStopSec=20s
SendSIGKILL=no
LimitNOFILE=8192
[Install]
WantedBy=multi-user.target
[root@ubuntu2004 prometheus]#systemctl daemon-reload
[root@ubuntu2004 prometheus]#systemctl stop prometheus.service
[root@ubuntu2004 prometheus]#systemctl start prometheus.service
[root@ubuntu2004 prometheus]#ss -ntlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=772,fd=3))
LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=739,fd=13))
LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=772,fd=4))
LISTEN 0 4096 *:9090 *:* users:(("prometheus",pid=6640,fd=3))
2、部署node exporter
#prometheus节点：
[root@ubuntu2004 ~]#tar xf node_exporter-1.4.0.linux-amd64.tar.gz
[root@ubuntu2004 ~]#cd node_exporter-1.4.0.linux-amd64/
[root@ubuntu2004 node_exporter-1.4.0.linux-amd64]#mv node_exporter /usr/local/bin/
[root@ubuntu2004 node_exporter-1.4.0.linux-amd64]#vim /usr/lib/systemd/system/exporter.service
[Unit]
Description=node_exporter
Documentation=https://prometheus.io/
After=network.target
[Service]
Type=simple
#User=prometheus
ExecStart=/usr/local/bin/node_exporter --collector.ntp --collector.mountstats --collector.systemd --collector.ethtool --collector.tcpstat
ExecReload=/bin/kill -HUP $MAINPID
TimeoutStopSec=20s
Restart=always
[Install]
WantedBy=multi-user.target
[root@ubuntu2004 node_exporter-1.4.0.linux-amd64]#systemctl daemon-reload
[root@ubuntu2004 node_exporter-1.4.0.linux-amd64]#systemctl start exporter.service
[root@ubuntu2004 ~]#ss -ntlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=772,fd=3))
LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=739,fd=13))
LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=772,fd=4))
LISTEN 0 4096 *:9090 *:* users:(("prometheus",pid=6640,fd=3))
LISTEN 0 4096 *:9100 *:* users:(("node_exporter",pid=51385,fd=7))
#其他被监控节点：
[root@ubuntu2004 ~]#tar xf node_exporter-1.4.0.linux-amd64.tar.gz
[root@ubuntu2004 ~]#mv node_exporter-1.4.0.linux-amd64/node_exporter /usr/local/bin/
[root@ubuntu2004 ~]#vim /usr/lib/systemd/system/exporter.service
[Unit]
Description=node_exporter
Documentation=https://prometheus.io/
After=network.target
[Service]
Type=simple
ExecStart=/usr/local/bin/node_exporter
ExecReload=/bin/kill -HUP $MAINPID
TimeoutStopSec=20s
Restart=always
[Install]
WantedBy=multi-user.target
[root@ubuntu2004 ~]#systemctl daemon-reload
[root@ubuntu2004 ~]#systemctl start exporter.service
[root@ubuntu2004 ~]#ss -ntlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=739,fd=13))
LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=772,fd=3))
LISTEN 0 4096 *:9100 *:* users:(("node_exporter",pid=46958,fd=3))
LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=772,fd=4))
3、修改prometheus配置监控各节点
[root@ubuntu2004 prometheus]#pwd
/usr/local/prometheus
[root@ubuntu2004 prometheus]#vim prometheus.yml
......
- job_name: "node_exporter"
metrics_path: '/metrics'
scheme: 'http'
static_configs:
- targets:
- "10.0.0.101:9100"
- "10.0.0.102:9100"
- "10.0.0.103:9100"
[root@ubuntu2004 prometheus]#systemctl restart prometheus.service
4、基于文件自动发现节点
[root@ubuntu2004 prometheus]#mkdir targets
[root@ubuntu2004 prometheus]#vim targets/node-linux.yaml
- targets:
- 10.0.0.101:9100
- 10.0.0.102:9100
- 10.0.0.103:9100
labels:
os: ubuntu
#更改配置为：
[root@ubuntu2004 prometheus]#vim prometheus.yml
......
- job_name: "node_exporter"
metrics_path: '/metrics'
scheme: 'http'
file_sd_configs:
- files:
- targets/node*.yaml
refresh_interval: 2m
[root@ubuntu2004 prometheus]#systemctl restart prometheus.service
[root@ubuntu2004 prometheus]#systemctl status prometheus.service
5、基于consul自动发现监控节点
[root@ubuntu2004 ~]#curl -LO https://releases.hashicorp.com/consul/1.14.1/consul_1.14.1_linux_amd64.zip
[root@ubuntu2004 ~]#unzip consul_1.14.1_linux_amd64.zip
[root@ubuntu2004 ~]#mv consul /usr/local/bin
[root@ubuntu2004 ~]#mkdir -pv /consul/data
[root@ubuntu2004 ~]#mkdir /etc/consul
#以开发者模式运行：
[root@ubuntu2004 ~]#consul agent -dev -ui -data-dir=/consul/data/ --config-dir=/etc/consul -client=0.0.0.0
[root@ubuntu2004 ~]#vim /etc/consul/nodes.json
{
"services": [
{
"id": "node_exporter-node01",
"name": "103.wang.org",
"address": "10.0.0.103",
"port": 9100,
"tags": ["nodes"],
"checks": [{
"http": "http://10.0.0.103:9100/metrics",
"interval": "5s"
}]
},
{
"id": "node_exporter-node02",
"name": "102.wang.org",
"address": "10.0.0.102",
"port": 9100,
"tags": ["nodes"],
"checks": [{
"http": "http://10.0.0.102:9100/metrics",
"interval": "5s"
}]
},
{
"id": "node_exporter-node03",
"name": "prometheus-server",
"address": "10.0.0.101",
"port": 9100,
"tags": ["nodes"],
"checks": [{
"http": "http://10.0.0.101:9100/metrics",
"interval": "5s"
}]
}
]
}
#修改配置文件：
[root@ubuntu2004 prometheus]#vim prometheus.yml
.......
- job_name: "node_exporter"
metrics_path: '/metrics'
scheme: 'http'
consul_sd_configs:
- server: "10.0.0.101:8500"
tags:
- "nodes"
refresh_interval: 2m
[root@ubuntu2004 prometheus]#systemctl reload prometheus.service
6、基于docker-compose部署prometheus
温馨提示：所有节点提前部署好docker和docker-compose，如需脚本，请参照：https://blog.51cto.com/dayu/5878166
避免冲突，请提前关闭宿主机部署的prometheus和node-exporter
6-1、部署prometheus
[root@ubuntu2004 01-prometheus-basics-example]#tree
.
├── docker-compose.yml
├── prometheus
│?? ├── prometheus.yml
│?? └── targets
│?? ├── nodes-linux.yaml
│?? └── prometheus-servers.yaml
└── README.md
[root@ubuntu2004 01-prometheus-basics-example]#cat docker-compose.yml
version: '3.6'
volumes:
prometheus_data: {}
networks:
monitoring:
driver: bridge
services:
prometheus:
image: prom/prometheus:v2.40.2
volumes:
- ./prometheus/:/etc/prometheus/
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
- '--web.console.templates=/usr/share/prometheus/consoles'
- '--web.enable-lifecycle'
networks:
- monitoring
ports:
- 9090:9090
restart: always
node-exporter:
image: prom/node-exporter:v1.4.0
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
- '--path.rootfs=/rootfs'
ports:
- 9100:9100
networks:
- monitoring
restart: always
[root@ubuntu2004 01-prometheus-basics-example]#cd prometheus/
[root@ubuntu2004 prometheus]#ls
prometheus.yml targets
[root@ubuntu2004 prometheus]#cat prometheus.yml
# my global config
# Author: MageEdu <mage@magedu.com>
# Repo: http://gitlab.magedu.com/MageEdu/prometheus-configs/
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
# Alertmanager configuration
alerting:
alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
- job_name: 'prometheus'
# metrics_path defaults to '/metrics'
# scheme defaults to 'http'.
static_configs:
- targets:
- localhost:9090
# All nodes
- job_name: 'nodes'
file_sd_configs:
- files:
- targets/nodes-*.yaml
refresh_interval: 2m
[root@ubuntu2004 prometheus]#cd targets/
[root@ubuntu2004 targets]#ls
nodes-linux.yaml prometheus-servers.yaml
[root@ubuntu2004 targets]#cat prometheus-servers.yaml
- targets:
- localhost:9090
labels:
app: prometheus
job: prometheus
[root@ubuntu2004 targets]#cat nodes-linux.yaml
- targets:
- 10.0.0.101:9100
- 10.0.0.102:9100
- 10.0.0.103:9100
labels:
app: node-exporter
[root@ubuntu2004 targets]#cd ../..
[root@ubuntu2004 01-prometheus-basics-example]#ls
docker-compose.yml prometheus README.md
[root@ubuntu2004 01-prometheus-basics-example]#docker-compose up -d
Creating 01-prometheus-basics-example_node-exporter_1 ... done
Creating 01-prometheus-basics-example_prometheus_1 ... done
[root@ubuntu2004 01-prometheus-basics-example]#docker-compose ps
Name Command State Ports
--------------------------------------------------------------------------------------------------------------------------------
01-prometheus-basics-example_node-exporter_1 /bin/node_exporter --path. ... Up 0.0.0.0:9100->9100/tcp,:::9100->9100/tcp
01-prometheus-basics-example_prometheus_1 /bin/prometheus --config.f ... Up 0.0.0.0:9090->9090/tcp,:::9090->9090/tcp
[root@ubuntu2004 00-node-exporter]#ss -ntlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=772,fd=3))
LISTEN 0 4096 0.0.0.0:9090 0.0.0.0:* users:(("docker-proxy",pid=65747,fd=4))
LISTEN 0 4096 0.0.0.0:9100 0.0.0.0:* users:(("docker-proxy",pid=65721,fd=4))
LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=739,fd=13))
LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=772,fd=4))
LISTEN 0 4096 [::]:9090 [::]:* users:(("docker-proxy",pid=65754,fd=4))
LISTEN 0 4096 [::]:9100 [::]:* users:(("docker-proxy",pid=65730,fd=4))
6-2、部署node-exporter
[root@ubuntu2004 ~]#cat docker-compose.yml
version: '3.3'
volumes:
prometheus_data: {}
networks:
monitoring:
driver: bridge
services:
node-exporter:
image: prom/node-exporter:v1.4.0
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
- '--path.rootfs=/rootfs'
ports:
- 9100:9100
networks:
- monitoring
restart: always
[root@ubuntu2004 ~]#docker-compose up -d
[root@ubuntu2004 ~]#ss -ntlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 4096 0.0.0.0:9100 0.0.0.0:* users:(("docker-proxy",pid=61112,fd=4))
LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=740,fd=13))
LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=773,fd=3))
LISTEN 0 4096 [::]:9100 [::]:* users:(("docker-proxy",pid=61118,fd=4))
LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=773,fd=4))
