logstash收集nginx访问日志

时间:2023-12-18 22:49:08

logstash收集nginx访问日志

安装nginx

#直接yum安装:
[root@elk-node1 ~]# yum install nginx -y
官方文档:http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format
#修改配置文件的日志格式:
vim /etc/nginx/nginx.conf
#在http模块中添加
log_format json '{"@timestamp":"$time_iso8601",'
'"@version":"1",'
'"client":"$remote_addr",'
'"url":"$uri",'
'"status":"$status",'
'"domain":"$host",'
'"host":"$server_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"referer": "$http_referer",'
'"ua": "$http_user_agent"'
'}';
#在server模块中添加
access_log /var/log/nginx/access_json.log json;
#修改后的nginx.conf文件
[root@elk-node1 ~]# grep -Ev "#|^&" /etc/nginx/nginx.conf user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid; include /usr/share/nginx/modules/*.conf; events {
worker_connections 1024;
} http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
log_format json '{"@timestamp":"$time_iso8601",'
'"@version":"1",'
'"client":"$remote_addr",'
'"url":"$uri",'
'"status":"$status",'
'"domain":"$host",'
'"host":"$server_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"referer": "$http_referer",'
'"ua": "$http_user_agent"'
'}';
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048; include /etc/nginx/mime.types;
default_type application/octet-stream; include /etc/nginx/conf.d/*.conf; server {
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
root /usr/share/nginx/html; include /etc/nginx/default.d/*.conf;
access_log /var/log/nginx/access_json.log json; location / {
} error_page 404 /404.html;
location = /40x.html {
} error_page 500 502 503 504 /50x.html;
location = /50x.html {
}
} }
#启动:
[root@controller ~]# systemctl start nginx
root@elk-node1 ~]# ss -lntp|grep 80
LISTEN 0 511 *:80 *:* users:(("nginx",pid=8045,fd=6),("nginx",pid=8044,fd=6),("nginx",pid=8043,fd=6))
LISTEN 0 511 :::80 :::* users:(("nginx",pid=8045,fd=7),("nginx",pid=8044,fd=7),("nginx",pid=8043,fd=7))

浏览器访问:http://192.168.247.135/
logstash收集nginx访问日志

查看nginx日志

logstash收集nginx访问日志

编写logstash

#添加nginx日志格式到之前logstash的elk-log.yml
[root@elk-node1 ~]# cat /etc/logstash/conf.d/elk_log.conf
input {
file {
path => "/var/log/messages"
type => "system"
start_position => "beginning"
}
file {
path => "/var/log/elasticsearch/hejianlai.log"
type => "es-error"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
file {
path => "/var/log/nginx/access_json.log"
codec => json
start_position => "beginning"
type => "nginx-log"
}
}
output { if [type] == "system"{
elasticsearch {
hosts => ["192.168.247.135:9200"]
index => "systemlog-%{+YYYY.MM.dd}"
}
} if [type] == "es-error"{
elasticsearch {
hosts => ["192.168.247.135:9200"]
index => "es-error-%{+YYYY.MM.dd}"
}
}
if [type] == "nginx-log"{
elasticsearch {
hosts => ["192.168.247.135:9200"]
index => "nginx-log-%{+YYYY.MM.dd}"
}
}
}

#添加--configtest参数检查配置语法是否有误!!!
[root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf --configtest
Configuration OK
#把之前后台运行的进程kill掉重启:
[root@elk-node1 ~]# ps aux|grep elk
root       3248  0.8  6.0 3632960 234924 pts/2  Sl   11:25   1:10 /usr/local/java/jdk1.8.0_171/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xss2048k -Djffi.boot.library.path=/opt/logstash/vendor/jruby/lib/jni -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/logstash/heapdump.hprof -Xbootclasspath/a:/opt/logstash/vendor/jruby/lib/jruby.jar -classpath ::/usr/local/java/jdk1.8.0_171/lib:/usr/local/java/jdk1.8.0_171/jre/lib -Djruby.home=/opt/logstash/vendor/jruby -Djruby.lib=/opt/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /opt/logstash/lib/bootstrap/environment.rb logstash/runner.rb agent -f /etc/logstash/conf.d/elk_log.conf
root       8135  0.0  0.0 112704   976 pts/0    S+   13:38   0:00 grep --color=auto elk
[root@elk-node1 ~]# kill -9 3248
You have new mail in /var/spool/mail/root
[root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf &
[1] 8178

kibana添加nginx日志

首先在es插件中我们能看到nginx-log的索引

logstash收集nginx访问日志

设置kibana

logstash收集nginx访问日志

logstash收集nginx访问日志