第二十一讲总体流程梳理nginx日志收集与geoip出图

《ELKstack业实战》基础篇第19讲nginx日志1、采用拓扑Redis队列：IP：00:放在log_formatbackend'$http_x_forwarded_for[$time_local]''"$host""$request"$status$body_bytes_sent''"$http_referer""$http_user_agent"'Agent配置[luohui@BJ-huasuan-h-web-07~]$cat/home/luohui/logstash-input{filepath=>["/home/data/logs/access.log"]type=>"nginx_access"}}outputif[type]=="nginx_access"{redis{host=>data_type=>"list"key=>"nginx"}}} [root@mailetc]#catlogstash_nginx.confinput{redishost=>port=>6379data_type=>"list"key=>"nginx"}}filtergrok

match{"message"=>"%{IPORHOST:ip}p}\]%{NOTSPACE:http_name}\"(?:%{WORD:verb}%{NOTSPACE:request}(?:HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\"%{NUMBER:response}(?:%{NUMBER:bytes:float}|-)%{QS:referrer}%{QS:agent}"}}datematch=>["timestamp","dd/MMM/YYYY:HH:mm:ssZ"}geoipsource=>"ip"target=>"geoip"add_field=>["[geoip][coordinates]","%{[geoip][longitude]}"]add_field=>["[geoip][coordinates]","%{[geoip][latitude]}"}mutateconvert=>["[geoip][coordinates]",}}output{action=>hostsindex=>"logstash-nginx-}}。logstash图表显示Ip情况：最nginxpatterns1nginxlog_formatmain'$remote_addr-$remote_user[$time_local]"$request"''$status$body_bytes_sent"$http_referer"''"$http_user_agent""$http_x_forwarded_for"'; [root@controllerlogstash-5.0.0]#cdvendor/bundle/jruby/1.9/gems/logstash-pattNGUSERNAME[a-zA-Z\.\@\-\+_%]+NGUSER%{NGUSERNAME}NGINXACCESS%{IPORHOST:ip}-%{NGUSER:remote_user}\[%{HTTPDATE:timestamp}\]\"(?:%{WORD:verb}%{NOTSPACE:request}(?:HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\"%{NUMBER:response}(?:%{NUMBER:bytes}|-)%{QS:referrer}%{QS:agent}%{NOTSPACE:http_x_forwarded_for}3inputoutput插件测试：[root@controlleretccatnginx.conf}filtergrokmatch=>{"message"=>"%{NGINXACCESS}"}datematch=>["timestamp","dd/MMM/YYYY:HH:mm:ssZ"}geoipsource=>"ip"target=>"geoip"add_field=>["[geoip][coordinates]","%{[geoip][longitude]}"]add_field=>["[geoip][coordinates]","%{[geoip][latitude]}"}mutateconvert=>["[geoip][coordinates]",}}outputelasticsearch{action=>hosts=>index=>"logstash-nginx-}}49-- ]"GET/favicon.icoHTTP/1.1"571"-""Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likecko)Chrome/45.0.2454.101Safari/537.36""-{"request"=>"agent"=>"\"Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/45.0.2454.101Safari/537.36\"","geoip"=>"timezone"=>"Asia/Shanghai","ip"=>"latitude"=>39.9289,"coordinates"=>[[0][1]"continent_code"=>"AS","city_name"=>"Beijing","country_code2"=>"CN","country_name"=>"","dma_code"=>nil,"country_code3"=>"region_name"=>"Beijing","location"=>[[0][1]"postal_code"=>nil,"longitude"=>116.3883,"region_code"=>"verb"=>"message"=>"49-- ]"GET/favicon.icoHTTP/1.1\"404571\"-\"\"Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/45.0.2454.101\"-"remote_user"=>"-"referrer"=>"\"-"@timestamp"=>2017-01-"response"=>"bytes"=>"571","ip"=>"49","@version"=>"host"=>"controller","http_x_forwarded_for"=>"\"-\"","httpversion"=>"timestamp"=> }log_formatmain'$remote_addr-$remote_user[$time_local]"$request"''$status$body_bytes_sent"$http_referer"''"$http_user_agent""$http_x_forwarded_for"$request_time28-- ]"GET/poweredby.pngHTTP/1.1"040"""Mozilla/4.0(compatible;MSIE7.0;WindowsNT6.1;WOW64;Trident/4.0;SLCC2;.NETCLR2.0.50727;.NETCLR3.5.30729;.NETCLR3.0.30729;MediaCenterPC6.0;InfoPath.3;.NET4.0C;.NET4.0E)""-"[root@controllerpatterns]#catnginxNGUSERNAME[a-zA-Z\.\@\-\+_%]+NGUSERNGINXACCESS%{IPORHOST:ip}-%{NGUSER:remote_user}\[%{HTTPDATE:timestamp}\]\"(?:%{WORD:verb}%{NOTSPACE:request}(?:HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\"%{NUMBER:response}(?:%{NUMBER:bytes:float}|-)%{QS:referrer}{"request"=>"agent"=>"\"Mozilla/4.0(compatible;MSIE7.0;Windows6.1;WOW64;Trident/4.0;SLCC2;.NETCLR2.0.50727;.NETCLR3.5.30729;.NETCLR3.0.30729;MediaCenterPC6.0;InfoPath.3;.NET4.0C;.NET4.0E)\"","geoip"=>"verb"=>"message"=>"28-- \"GET/poweredby.pngHTTP/1.1\"3040\"\"\"Mozilla/4.0(compatible;MSIE7.0;WindowsNT6.1;WOW64;Trident/4.0;SLCC2;.NETCLR2.0.50727;.NETCLR3.5.30729;.NETCLR3.0.30729;MediaCenterPC6.0;.NET4.0C;.NET4.0E)\"\"-\""tags"=>[0]"remote_user"=>"-"referrer"=>"\""@timestamp"=>2017-01-"request_time"=>"response"=>"bytes"=>"ip"=>"@version"=>"host"=>"controller","http_x_forwarded_for"=>"\"-\"","httpversion"=>"timestamp"=> }log_formatjsonaccess_log/etc/nginx/logs/access_nginx.jsontp://35/","agent":"Mozilla/4.0(compatible;MSIE7.0;WindowsNT6.1;WOW64;Trident/4.0;SLCC2;.NETCLR2.0.50727;.NETCLR3.5.30729;.NETCLR3.0.30729;MediaCenterPC6.0;InfoPath.3;.NET4.0C;.NET4.0E)","status":"304[root@controllerlogstash-5.0.0]#catinputfile type=>"nginx-path=>"/etc/nginx/logs/access_nginx.json"start_position=>"beginning"codecjson"#这里指定codec}}filterif[type]=="nginx-access"{geoip