plusieurs astuces elk/logstash

Tant qu’à fouiller dans des logs, autant s’amuser à tout mettre dans ELK ;

Je partage ici quelques astuces :

tracer le server-status d’apache httpd :

http_poller
kv

 

input {
http_poller {
urls => {
"custom_httpd_t0" => { url => "http://127.0.0.1:9680/server-status?auto"}
"custom_httpd_t1" => { url => "http://someserver:9681/server-status?auto"}
}
tags => apache_stats
type => xxxxxstatus
codec => plain
metadata_target => http_poller_metadata
interval => 5
}
 
if [http_poller_metadata] {
# Properly set the '@host' field based on the poller's metadat
mutate {
add_field => {
"@host" => "%{http_poller_metadata[name]}"
}
}
}
# Processed polled apache data
if "apache_stats" in [tags] {
# Apache stats uses inconsistent key names. Make sure all fields are camel cased, no spaces
mutate {
gsub => ["message", "^Total ", "Total"]
}
# Parse the keys/values in the apache stats, they're separated by ": '
kv {
source => message
target => apache_stats
field_split => "\n"
value_split => ":\ "
trim => " "
}
# We can make educated guesses that strings with mixes of numbers and dots
# are numbers, cast them for better behavior in Elasticsearch/Kibana
ruby {
code => "h=event['apache_stats']; h.each {|k,v| h[k] = v.to_f if v =~ /\A-?[0-9\.]+\Z/}"
}
mutate {
convert => {"BusyWorkers" => "integer"}
convert => {"IdleWorkers" => "integer"}
}
}

 

et aussi l’analyse des logs d’apache et weblogic

bon je ne montre que weblo, pour les astuces ignore_older, multiline  :

 

file {
type => "xxxxxfw"
path => "C:/xxxxx/logs-fw/xxxxx*"
sincedb_path => "C:/xxxxx/xxxxxfw.sincedb"
start_position => "beginning"
ignore_older => 8640000
codec => multiline {
pattern => "^2"
max_lines => 2000
negate => true
what => previous
}
tags => ["wlslog"]
}
et le filtre où je récupère dans le message le temps passé (1234 ms) ne pas oublier break_on_match
if [type] == "xxxxxfw" {
#2016-09-21 19:31:23,899 : WARN : com.xxxxxxxxxxxxxxValueStack:49 : warn() : Error setting expression 'dateFin' with value '[Ljava.lang.String;@29c00ac3'
#ognl.MethodFailedException: ConsulterSynthesexxxxx [java.lang.NoSuchMethod blablabla
# at xxxxxxxxntime.callAppropriateMethod(xxxxRuntime.java:125)
 grok {
 break_on_match => false
 match => [ "message" , "(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}%{SPACE}%{TIME})%{SPACE}:%{SPACE}%{WORD:verb}%{SPACE}:%{SPACE}%{GREEDYDATA:alertmessage}" ]
 match => [ "alertmessage" , "\(%{NUMBER:timespent} ms\)" ]
 }
 mutate {
 #remove_field => [ "message" ]
 convert => {"timespent" => "integer"}
 }
 date { match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss,SSS" ] }
 }

ah si, aussi pour apache je découpe les cookies en cookies

(bon ça ne m’a pas servi, mais peut être un jour…)

kv {
source => cookie
target => cookies
field_split => ";"
value_split => "="
trim => " "
}

 

++