filebeat 自定义nginx日志格式并写入ES:

(1)修改nginx的配置文件

vim /etc/nginx/nginx.conf

...
   log_format baimei_nginx_json '{"@timestamp":"$time_iso8601",'
                              '"host":"$server_addr",'
                              '"clientip":"$remote_addr",'
                              '"SendBytes":$body_bytes_sent,'
                              '"responsetime":$request_time,'
                              '"upstreamtime":"$upstream_response_time",'
                              '"upstreamhost":"$upstream_addr",'
                              '"http_host":"$host",'
                              '"uri":"$uri",'
                              '"domain":"$host",'
                              '"xff":"$http_x_forwarded_for",'
                              '"referer":"$http_referer",'
                              '"tcp_xff":"$proxy_protocol_addr",'
                              '"http_user_agent":"$http_user_agent",'
                              '"status":"$status"}';

具体参考： https://www.baimeidashu.com/8057.html

2.检查nginx的语法

[root@elk101 ~]# nginx -t
[root@elk101 ~]# systemctl reload nginx

3.访问nginx(访问后，修改公网IP地址即可，clientip字段)
http://10.0.0.101/

4.编写配置文件

cat config/01-nginx-to-es.yaml

filebeat.inputs:
- type: log
  paths: ["/var/log/nginx/access.log"]
  processors:
  - decode_json_fields:
      fields: ["message"]
      target: ""

#output.console:
#  pretty: true

output.elasticsearch:
  hosts: 
  - "http://10.0.0.101:19200"
  - "http://10.0.0.102:19200"
  - "http://10.0.0.103:19200"
  index: "baimei-filebeat-custom-nginx-access-%{+yyyy.MM.dd}"    

setup.ilm.enabled: false
setup.template.name: "baimei-filebeat-custom-nginx"
setup.template.pattern: "baimei-filebeat-custom-nginx*"
setup.template.overwrite: true
setup.template.settings:
  index.number_of_shards: 3
  index.number_of_replicas: 0

5.启动filebeat实例

[root@elk101 ~]# filebeat -e -c config/01-nginx-to-es.yaml