130 líneas
4.4 KiB
Plaintext
130 líneas
4.4 KiB
Plaintext
input {
|
|
beats {
|
|
port => "5044"
|
|
}
|
|
}
|
|
|
|
filter {
|
|
## beat and LSF compatibility
|
|
## https://discuss.elastic.co/t/problem-with-transfer-filebeat-6-1-3-logstash-6-1-3-elasticsearch-6-1-3/136264/6
|
|
## https://discuss.elastic.co/t/logstash-errors-after-upgrading-to-filebeat-6-3-0/135984/6
|
|
if [beat][hostname] {
|
|
if [source] {
|
|
if ![file] {
|
|
mutate {
|
|
add_field => {
|
|
"file" => "%{source}"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
mutate {
|
|
remove_field => [ "[host]" ]
|
|
}
|
|
mutate {
|
|
add_field => {
|
|
"host" => "%{[beat][hostname]}"
|
|
}
|
|
}
|
|
}
|
|
|
|
# ## apache2 module
|
|
# ## filebeat apache module https://www.elastic.co/guide/en/logstash/current/logstash-config-for-filebeat-modules.html
|
|
if [fileset][module] == "apache2" {
|
|
if [fileset][name] == "access" {
|
|
grok {
|
|
match => { "message" => ["%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?",
|
|
"%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \\[%{HTTPDATE:[apache2][access][time]}\\] \"-\" %{NUMBER:[apache2][access][response_code]} -" ] }
|
|
remove_field => "message"
|
|
}
|
|
mutate {
|
|
add_field => { "read_timestamp" => "%{@timestamp}" }
|
|
}
|
|
date {
|
|
match => [ "[apache2][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
|
|
remove_field => "[apache2][access][time]"
|
|
}
|
|
useragent {
|
|
source => "[apache2][access][agent]"
|
|
target => "[apache2][access][user_agent]"
|
|
remove_field => "[apache2][access][agent]"
|
|
}
|
|
geoip {
|
|
source => "[apache2][access][remote_ip]"
|
|
target => "[apache2][access][geoip]"
|
|
}
|
|
}
|
|
else if [fileset][name] == "error" {
|
|
grok {
|
|
match => { "message" => ["\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{LOGLEVEL:[apache2][error][level]}\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message]}",
|
|
"\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}\] \[pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message1]}" ] }
|
|
pattern_definitions => {
|
|
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
|
|
}
|
|
remove_field => "message"
|
|
}
|
|
mutate {
|
|
rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
|
|
}
|
|
date {
|
|
match => [ "[apache2][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
|
|
remove_field => "[apache2][error][timestamp]"
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
## syslog, there might be a module for this as well
|
|
if [type] == "syslog" {
|
|
### "$RepeatedMsgReduction off" /etc/rsyslog.conf
|
|
#if [message] =~ /last message repeated [0-9]+ times/ {
|
|
# drop { }
|
|
#}
|
|
## enable high precision timestamps
|
|
# comment out $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
|
|
grok {
|
|
match => { "message" => "(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:syslog_timestamp}) %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
|
|
add_field => {
|
|
"syslog_received_at" => "%{@timestamp}"
|
|
"syslog_received_from" => "%{host}"
|
|
}
|
|
}
|
|
syslog_pri {
|
|
|
|
}
|
|
date {
|
|
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
|
|
timezone => "Europe/Madrid"
|
|
}
|
|
mutate {
|
|
replace => { "syslog_timestamp" => "%{@timestamp}" }
|
|
}
|
|
# for check grok data type conversion bug???
|
|
mutate {
|
|
convert => {
|
|
"syslog_pid" => "integer"
|
|
}
|
|
}
|
|
}
|
|
|
|
## old apache filter
|
|
# if [type] == "apache" {
|
|
# grok {
|
|
# match => { "message" => "%{COMBINEDAPACHELOG}" }
|
|
# }
|
|
# date {
|
|
# match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
|
|
# timezone => "America/New York"
|
|
# }
|
|
# }
|
|
}
|
|
|
|
output {
|
|
elasticsearch {
|
|
hosts => ["elasticsearch:9200"]
|
|
index => "apache"
|
|
document_type => "log"
|
|
}
|
|
# stdout { codec => rubydebug }
|
|
}
|