logstash 解析nginx error日志

input {
#    beats {
#        host => "0.0.0.0"
#        port => 5400
#    }

stdin { }
}

filter {
 grok {
   patterns_dir => "/etc/logstash/patterns"
   #match => [ "message" , "%{NGINXACCESS}"]
   match => [ "message" , "%{DATA:timestr} \[%{DATA:error_level}\] (?<nginx_message>(.|\r|\n)*)(?:, client: %{IPORHOST:clientip})(?:, server: %{IPORHOST:nginx_server})(?:, request: \"%{DATA:nginx_request}\")?(?:, upstream: \"%{DATA:nginx_upstream}\")?(?:, host: \"%{DATA:nginx_host}\")?(?:, referrer: \"%{DATA:nginx_referrer}\")?"]
 }

      if [http_x_forwarded_for] == "-" or [http_x_forwarded_for] == "null" {
         mutate {
            update => { "http_x_forwarded_for" => "" }
         }
      }

      if [referer] == "-" or [referer] == "null" {
         mutate {
            update => { "referer" => "" }
         }
      }

    geoip {
      source => "clientip"
    }

    useragent {
      source => "agent"
      target => "agent_fields"
    }

  date {
    match => [ "timestr", "yyyy/MM/dd HH:mm:ss" ]
    timezone => "Asia/Shanghai"
    #target => "newtimestr"
    #locale => "en"
  }

        ruby {
                code => "event.set('index_day', event.get('@timestamp').time.localtime.strftime('%Y%m%d'))"
        }


}

output {
 elasticsearch {
   hosts => ["127.0.0.1:9200"]
   #index => "tek-%{+YYYY.MM.dd}"
   index => "tek-%{index_day}"
   document_type => "nginx_logs"
   template_name => "ta"
 }
 stdout { codec => rubydebug }
}

防止一天的日志分在了两个index中

        ruby {
                code => "event.set('index_day', event.get('@timestamp').time.localtime.strftime('%Y%m%d'))"
        }

发表评论

电子邮件地址不会被公开。 必填项已用*标注