logstash 字段引用

字段引用:10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (linux; U; Android 4.4.4; zh-cn; MX4 PRo Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103[elk@Vsftp logstash]$ cat logstash.conf input {   stdin{}   }filter {    grok {        match =>[              "message","%{ipORHOST:clientip} /[%{HTTPDATE:time}/] /"%{Word:verb} %{URIPATHPARAM:request}/?.* HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"(?<http_referer>/S+)/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",              "message" , "%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"(?<http_referer>/S+)/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} (?<http_url>/S+)/s+HTTP/%{NUMBER:httpversion}/"/s+/-/s+%{NUMBER:http_status_code}/s+%{NUMBER:bytes}/s+/"/-/"/s+/"(?<http_user_agent>(/S+))/"/s+(%{BASE16FLOAT:request_time})/s+(%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"                     ]    }}output {        stdout {                        codec => rubydebug                } }[elk@Vsftp logstash]$ logstash -f logstash.conf Settings: Default pipeline workers: 4Pipeline main started10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103{                 "message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] /"GET / HTTP/1.1/" - 200 23388 /"/" /"Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30/" 0.001 101.226.125.103",                "@version" => "1",              "@timestamp" => "2017-02-08T01:39:50.650Z",                    "host" => "Vsftp",                "clientip" => "10.168.255.134",                    "time" => "09/Oct/2016:15:28:52 +0800",                    "verb" => "GET",                 "request" => "/",             "httpversion" => "1.1",        "http_status_code" => "200",                   "bytes" => "23388",         "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",            "request_time" => "0.001",    "http_x_forwarded_for" => "101.226.125.103"}[elk@Vsftp logstash]$ cat logstash.conf input {   stdin{}   }filter {    grok {        match =>[              "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request}/?.* HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"(?<http_referer>/S+)/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",              "message" , "%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"(?<http_referer>/S+)/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} (?<http_url>/S+)/s+HTTP/%{NUMBER:httpversion}/"/s+/-/s+%{NUMBER:http_status_code}/s+%{NUMBER:bytes}/s+/"/-/"/s+/"(?<http_user_agent>(/S+))/"/s+(%{BASE16FLOAT:request_time})/s+(%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"                     ]    }geoip {                        source => "http_x_forwarded_for"                        target => "geoip"                        database => "/usr/local/logstash-2.3.4/etc/GeoLiteCity.dat"                        add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]                        add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]                }}output {        stdout {                        codec => rubydebug                } }[elk@Vsftp logstash]$ logstash -f logstash.conf Settings: Default pipeline workers: 4Pipeline main started10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103{                 "message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] /"GET / HTTP/1.1/" - 200 23388 /"/" /"Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30/" 0.001 101.226.125.103",                "@version" => "1",              "@timestamp" => "2017-02-08T01:42:33.645Z",                    "host" => "Vsftp",                "clientip" => "10.168.255.134",                    "time" => "09/Oct/2016:15:28:52 +0800",                    "verb" => "GET",                 "request" => "/",             "httpversion" => "1.1",        "http_status_code" => "200",                   "bytes" => "23388",         "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",            "request_time" => "0.001",    "http_x_forwarded_for" => "101.226.125.103",                   "geoip" => {                      "ip" => "101.226.125.103",           "country_code2" => "CN",           "country_code3" => "CHN",            "country_name" => "China",          "continent_code" => "AS",             "region_name" => "23",               "city_name" => "Shanghai",                "latitude" => 31.045600000000007,               "longitude" => 121.3997,                "timezone" => "Asia/Shanghai",        "real_region_name" => "Shanghai",                "location" => [            [0] 121.3997,            [1] 31.045600000000007        ],             "coordinates" => [            [0] 121.3997,            [1] 31.045600000000007        ]    }}字段引用字段引用是Logstash::Event 对象的属性,我们之前提过事件就像一个哈希一样,所以你可以想象字段就像一个键值对如果你想在Logstash 配置中使用字段的值,只需把字段的名字写在中括号[]里就行了,这就叫字段引用[elk@Vsftp logstash]$ cat logstash.conf input {   stdin{}   }filter {    grok {        match =>[              "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request}/?.* HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"(?<http_referer>/S+)/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",              "message" , "%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"(?<http_referer>/S+)/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} (?<http_url>/S+)/s+HTTP/%{NUMBER:httpversion}/"/s+/-/s+%{NUMBER:http_status_code}/s+%{NUMBER:bytes}/s+/"/-/"/s+/"(?<http_user_agent>(/S+))/"/s+(%{BASE16FLOAT:request_time})/s+(%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"                     ]    }geoip {                        source => "http_x_forwarded_for"                        target => "geoip"                        database => "/usr/local/logstash-2.3.4/etc/GeoLiteCity.dat"                        add_field => [ "aaaaaa", "%{[geoip][location][0]}" ]                        add_field => [ "bbbbbb", "%{[geoip][location][1]}" ]                }}output {        stdout {                        codec => rubydebug                } }[elk@Vsftp logstash]$ logstash -f logstash.conf Settings: Default pipeline workers: 4Pipeline main started10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103{                 "message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] /"GET / HTTP/1.1/" - 200 23388 /"/" /"Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30/" 0.001 101.226.125.103",                "@version" => "1",              "@timestamp" => "2017-02-08T01:47:32.656Z",                    "host" => "Vsftp",                "clientip" => "10.168.255.134",                    "time" => "09/Oct/2016:15:28:52 +0800",                    "verb" => "GET",                 "request" => "/",             "httpversion" => "1.1",        "http_status_code" => "200",                   "bytes" => "23388",         "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",            "request_time" => "0.001",    "http_x_forwarded_for" => "101.226.125.103",	                   "geoip" => {                      "ip" => "101.226.125.103",           "country_code2" => "CN",           "country_code3" => "CHN",            "country_name" => "China",          "continent_code" => "AS",             "region_name" => "23",               "city_name" => "Shanghai",                "latitude" => 31.045600000000007,               "longitude" => 121.3997,                "timezone" => "Asia/Shanghai",        "real_region_name" => "Shanghai",                "location" => [            [0] 121.3997,            [1] 31.045600000000007        ]    },		                  "aaaaaa" => 121.3997,                  "bbbbbb" => 31.045600000000007}变量值内插:[elk@Vsftp logstash]$ cat logstash.conf input {   stdin{}   }filter {    grok {        match =>[              "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request}/?.* HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"(?<http_referer>/S+)/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",              "message" , "%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"(?<http_referer>/S+)/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} (?<http_url>/S+)/s+HTTP/%{NUMBER:httpversion}/"/s+/-/s+%{NUMBER:http_status_code}/s+%{NUMBER:bytes}/s+/"/-/"/s+/"(?<http_user_agent>(/S+))/"/s+(%{BASE16FLOAT:request_time})/s+(%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} /[%{HTTPDATE:time}/] /"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}/" /- %{NUMBER:http_status_code} %{NUMBER:bytes} /"/" /"(?<http_user_agent>(/S+/s+)*/S+)/" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"                     ]    }geoip {                        source => "http_x_forwarded_for"                        target => "geoip"                        database => "/usr/local/logstash-2.3.4/etc/GeoLiteCity.dat"                        add_field => [ "kkkkkkk", "[geoip][location][0]"]                        add_field => [ "hhhhhhh", "[geoip][location][1]" ]                }}output {        stdout {                        codec => rubydebug                } }[elk@Vsftp logstash]$ logstash -f logstash.conf Settings: Default pipeline workers: 4Pipeline main started10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103{                 "message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] /"GET / HTTP/1.1/" - 200 23388 /"/" /"Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30/" 0.001 101.226.125.103",                "@version" => "1",              "@timestamp" => "2017-02-08T01:49:49.034Z",                    "host" => "Vsftp",                "clientip" => "10.168.255.134",                    "time" => "09/Oct/2016:15:28:52 +0800",                    "verb" => "GET",                 "request" => "/",             "httpversion" => "1.1",        "http_status_code" => "200",                   "bytes" => "23388",         "http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",            "request_time" => "0.001",    "http_x_forwarded_for" => "101.226.125.103",                   "geoip" => {                      "ip" => "101.226.125.103",           "country_code2" => "CN",           "country_code3" => "CHN",            "country_name" => "China",          "continent_code" => "AS",             "region_name" => "23",               "city_name" => "Shanghai",                "latitude" => 31.045600000000007,               "longitude" => 121.3997,                "timezone" => "Asia/Shanghai",        "real_region_name" => "Shanghai",                "location" => [            [0] 121.3997,            [1] 31.045600000000007        ]    },                 "kkkkkkk" => "[geoip][location][0]",                 "hhhhhhh" => "[geoip][location][1]"				 				 	必须使用        add_field => [ "aaaaaa", "%{[geoip][location][0]}" ]                        add_field => [ "bbbbbb", "%{[geoip][location][1]}" ]}