一、场景：此文以一个简单的json字符串文件为例，描述如何用logstash解析嵌套的json，并删除其中的某些字段
我们在linux中test.json的内容如下：
{"timestamp":"2018-08-02T14:42:50.084467+0800","flow_id":364959073190719,"in_iface":"eth1","event_type":"alert","src_ip":"10.0.0.4","src_port":80,"dest_ip":"10.0.0.5","dest_port":16781,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2101201,"rev":10,"signature":"GPL WEB_SERVER 403 Forbidden","category":"Attempted Information Leak","severity":2},"http":{"hostname":"bapi.yahoo.com","url":"\/v1tns\/searchorderlist?_time=1533192163978","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","xff":"39.106.108.38","http_content_type":"text\/html","http_method":"POST","protocol":"HTTP\/1.0","status":403,"length":568},"app_proto":"http","flow":{"pkts_toserver":5,"pkts_toclient":5,"bytes_toserver":1547,"bytes_toclient":1076,"start":"2018-08-02T14:42:50.082751+0800"}}
为了方便查看，formate后，为如下格式
{
"timestamp":"2018-08-02T14:42:50.084467+0800",
"flow_id":364959073190719,
"in_iface":"eth1",
"event_type":"alert",
"src_ip":"10.0.0.4",
"src_port":80,
"dest_ip":"10.0.0.5",
"dest_port":16781,
"proto":"TCP",
"tx_id":0,
"alert":{
"action":"allowed",
"gid":1,
"signature_id":2101201,
"rev":10,
"signature":"GPL WEB_SERVER 403 Forbidden",
"category":"Attempted Information Leak",
"severity":2
},
"http":{
"hostname":"bapi.yahoo.com",
"url":"\/v1tns\/searchorderlist?_time=1533192163978",
"http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36",
"xff":"39.106.108.38",
"http_content_type":"text\/html",
"http_method":"POST",
"protocol":"HTTP\/1.0",
"status":403,
"length":568
},
"app_proto":"http",
"flow":{
"pkts_toserver":5,
"pkts_toclient":5,
"bytes_toserver":1547,
"bytes_toclient":1076,
"start":"2018-08-02T14:42:50.082751+0800"
}
}
二、目的： 我们需要解析这个json，并且删除json中"src_ip"字段和"http下的hostname"这个字段
我的配置文件如下：
input {
file {
path => "/usr/share/logstash/private.cond/nestjson.json"
codec => "json"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
json {
source => "message"
}
mutate {
remove_field => ["src_ip","[http][hostname]"]
}
}
output {
stdout {
codec => rubydebug
}
}
注意第14行删除
注意第14行删除字段和嵌套字段的写法
运行logstash我们得到如下输出：
{
"alert" => {
"gid" => 1,
"rev" => 10,
"severity" => 2,
"signature" => "GPL WEB_SERVER 403 Forbidden",
"action" => "allowed",
"signature_id" => 2101201,
"category" => "Attempted Information Leak"
},
"http" => {
"protocol" => "HTTP/1.0",
"http_content_type" => "text/html",
"http_user_agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"http_method" => "POST",
"length" => 568,
"url" => "/v1tns/searchorderlist?_time=1533192163978",
"xff" => "39.106.108.38",
"status" => 403
},
"path" => "/usr/share/logstash/private.cond/test.json",
"event_type" => "alert",
"src_port" => 80,
"dest_port" => 16781,
"dest_ip" => "10.0.0.5",
"proto" => "TCP",
"flow_id" => 364959073190719,
"tx_id" => 0,
"@version" => "1",
"in_iface" => "eth1",
"timestamp" => "2018-08-02T14:42:50.084467+0800",
"flow" => {
"pkts_toserver" => 5,
"pkts_toclient" => 5,
"bytes_toserver" => 1547,
"start" => "2018-08-02T14:42:50.082751+0800",
"bytes_toclient" => 1076
},
"host" => "elk",
"app_proto" => "http",
"@timestamp" => 2018-08-02T10:14:14.372Z
}
我们可以看到src_ip和http下的hostname已经被成功删除