logstashgrokmysql_Logstashgrok正则表达式

我在使用logstash的时候&＃xff0c;为了更细致的切割日志&＃xff0c;会写一些正则表达式。

使用方法input {

file {

type &＃61;> "billin"

path &＃61;> "/data/logs/product/result.log"

}

}

filter {

grok {

type &＃61;> "billin"

pattern &＃61;> "%{BILLINCENTER}"

patterns_dir &＃61;> "/data/logstash/patterns/my_patterns"

}

}

output {

redis {

host &＃61;> "192.168.50.13"

data_type &＃61;>"list"

key &＃61;> "logstash:redis"

}

}

以下内容为正则表达式文件&＃xff1a;cat my_patterns

TAB \t

META \-&＃43;

WZ ([^ ]*)

IPPORT %{IP}:%{POSINT}|%{META}

REQUEST (?:/[A-Za-z0-9$.&＃43;!*&＃39;(),~:#%_-]*)&＃43;\?[A-Za-z0-9$.&＃43;!*&＃39;(),~#%&/&＃61;:;_-]*

TY (?:(?

#EVERYURL ((\w&＃43;://)?([^\.]&＃43;)(\.[^/:]&＃43;)(:\d*)?([^#]*))|-

#EVERYURL (((\w&＃43;://)?([^\.]&＃43;)(\.[^/:]&＃43;)?([^#]*))&＃43;)|(\w&＃43;)|-

#EVERYURL ((\w&＃43;://)?([^\.]&＃43;)(\.[^/:]&＃43;)?([^#]*))&＃43;)|-

EVERYURL (http://&＃43;[\w\d:#&＃64;%/;$()~_?\&＃43;-&＃61;\\\.&]&＃43;)|(-)

#Logformat

########nginx access log example########

#122.137.199.113"122.137.199.113"www.xxxx.com172.16.10.110172.16.12.114:8018/Jun/2013:15:51:03 &＃43;0800GET /g/getSaleCounts.do?rnd&＃61;1371541857448&showStatus&＃61;true&goodsIds&＃61;215abd2e8fa95bc8 HTTP/1.120078"http://www.xxxx.com/goods-215abd2e8fa95bc8.html""Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; NP06)""a8fdb711-a695-43bd-abdd-a224fb07350d"

###############################

NGINXACCESSLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{IPPORT:upstrem_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:guid}

#picture p0.xxxx.com access log . 2012.07.19 add

PICLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}

#iis log format 20120618 add

###########iis log example###############

#2013-06-18 08:00:00 172.16.10.233 GET /js/functions.js - 80 - 117.136.34.2 Mozilla/5.0&＃43;(Linux;&＃43;U;&＃43;Android&＃43;4.1.2;&＃43;zh-CN;&＃43;LT22i&＃43;Build/6.2.A.0.400)&＃43;AppleWebKit/534.31&＃43;(KHTML,&＃43;like&＃43;Gecko)&＃43;UCBrowser/9.0.1.275&＃43;U3/0.8.0&＃43;Mobile&＃43;Safari/534.31 200 0 0 0

###################################

IISLOG %{DATE_EU:log_date} %{TIME:log_time} %{IP:server_ip} %{WORD:verb} %{URIPATH:uri_stem} %{WZ:uri_query} %{POSINT:s_port} %{WZ:cs_username} %{IP:c_ip} %{WZ:agent} %{POSINT:request} %{POSINT:substatus} %{POSINT:win32_status} %{POSINT:time_taken}

#2012/07/12 add

ZW \w&＃43;

###java date example

# 2012-11-27 14:52:42

############

JAVA_DATE %{DATE_EU} %{TIME}

EARTHLOG \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":"%{ZW:desc}","dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\]

EAGLEUPDATE \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code},"orderId":"%{ZW:orderId}"\}\]

EAGLELOGIN \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\]

#2012/10/23 add

LJF (-\s&＃43;-)

RESINLOG %{IP:remote_ip}%{SPACE}%{NUMBER}%{SPACE}%{LJF}%{SPACE}\[%{HTTPDATE:timestamp}\]%{SPACE}"%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER}"%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:session}

#RESINLOG %{IP:ip} %{NUMBER} - - \[%{HTTPDATE:time}\] "%{WORD:verb} %{WZ:request} HTTP/%{NUMBER}" %{NUMVER:response} %{NUMBER:bytes} %{QS:uri} %{QS:agent} %{QS:session}

#2012/11/13 add

DKH (\{.*\})

STOREGREP (\[\/\/\/ \- \] INFO \-)

DHMH ([^;|&＃61;]*)

CENTERLOG %{JAVA_DATE} %{STOREGREP} BId&＃61;%{NUMBER:bid};BR&＃61;%{DHMH:br};BP&＃61;%{DKH:bp}

#2012/11/20 add

JAVAGREP (\[\/\/\/ \- \])

ORDERCENTERERR %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{QS:message}

ORDERCENTERRESULT %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{DKH:message}

#2012/11/27 add

#####log example#######

#2013-06-18 15:28:12 INFO :{message:媒体传递的参数{"uid":["0"],"cid":["A100054947||0000"],"url":["http://www.xxxx.com/?from&＃61;lianmeng-weiyi"],"src":["weiyi"]}}

#

PARTNER %{JAVA_DATE:timestamp} %{WORD:level} :%{DKH:message}

#2012/11/28 add

PARTNERAPI %{JAVA_DATE:timestamp} %{WZ:level} :%{DKH:message}

#2013/06/18 add

#pattern all in the &＃39;[adskfjl }{\]&＃39;

FKH ([^;]*)

#######aether.log#####

#[2013-06-18 15:27:29] [INFO] [com.tuan.web.controller.IndexController] [{message:setHotStore#hot store size:5}]

AETHERLOG \[%{JAVA_DATE:timestamp}\] \[%{WZ:level}\] \[%{WZ:method}\] %{FKH:message}