Logstash 收集nginx访问日志

以下 ELK 版本为7.10。
修改nginx日志格式为json格式：

[root@web01 ~]# vim /etc/nginx/nginx.conf
......
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    log_format main_json '{"@timestamp":"$time_iso8601",'
            '"host":"$server_addr",'
            '"clientip":"$remote_addr",'
            '"size":$body_bytes_sent,'
            '"responsetime":$request_time,'
            '"upstreamtime":"$upstream_response_time",'
            '"upstreamhost":"$upstream_addr",'
            '"http_host":"$host",'
            '"url":"$uri",'
            '"domain":"$host",'
            '"ua":"$http_user_agent",'
            '"xff":"$http_x_forwarded_for",'
            '"referer":"$http_referer",'
            '"status":"$status"}';

    access_log  /var/log/nginx/access_json.log  main_json;
    access_log  /var/log/nginx/access.log  main;
......

测试查看日志：

[root@web01 ~]# systemctl reload nginx
[root@web01 ~]# curl 10.0.0.8
[root@web01 ~]# tail -n1 /var/log/nginx/access_json.log 
{"@timestamp":"2020-12-11T17:04:48+08:00","host":"10.0.0.8","clientip":"10.0.0.8","size":5,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"10.0.0.8","url":"/index.html","domain":"10.0.0.8","ua":"curl/7.29.0","xff":"-","referer":"-","status":"200"}
[root@web01 ~]# tail -n1 /var/log/nginx/access.log 
10.0.0.8 - - [11/Dec/2020:17:04:48 +0800] "GET / HTTP/1.1" 200 5 "-" "curl/7.29.0" "-"

logstash收集：（索引名字也可以引用设置的type字段如：%{type}-%{+YYYY.MM.dd}"）

[root@elkstack03 conf.d]# vim nginx_es.yml
input {
  file {
    path => "/var/log/nginx/access_json.log"
    start_position => "end"
    type => "nginx_access"
    codec => json
  }
}
output {
  elasticsearch {
    hosts => "10.0.0.5:9200"
    index => "nginx_access-%{+YYYY.MM.dd}"
  }
}

[root@web01 logstash]# logstash -f nginx_es.yml -t
[root@web01 logstash]# logstash -f nginx_es.yml &

使用Kibana查看Elasticsearch索引是否创建成功：