logstash overwrite

@Jestin  April 03, 2020

logstash overwrite

测试日志

2020-04-02 13:05:04.872 [http-nio-7777-exec-12] INFO  24f7db0b88f047eea361d5493c567349 com.addx.iotcamera.aop.LogRequestAndResponseAspect - Rest API(24f7db0b88f047eea361d5493c567349): http://api.addx.live/lambda/updatehardwareinfo
Body:{"batteryEvent":0,"deviceType":"AIC","sdCardStatus":{"total":0,"formatStatus":1,"free":0},"serialNumber":"fd06c10707e1aab4f978f02678bde19c","charge":0,"buildTime":"2020-03-07-15-26-29","ip":"192.168.0.105","mcuVersion":"111","battery":47,"version":"0.2.22","userSn":"AICVAA8RMAM0400","MAC":"58:b3:fc:5e:7a:15","ap":"TP-LINK_EA3","uid":"fd06c10707e1aab4f978f02678bde19c","por":2,"autoSuspendDebug":{"maxDiff":9950,"maxDiffSrc":7,"srcBits":1027},"svnVersion":"g3e7ff90","wifiRssi":-49,"wifiChannel":0,"live":0,"reportTime":1585832705}
2020-04-02 13:05:02.017 [http-nio-7777-exec-14] INFO  26a825125d0e46d7bbbc6f1490fa86ed com.addx.iotcamera.service.ReportLogService - {"msg":"AAAAA","reportType":"keepaliveLoopSend","reportGroup":"live","seconds":30,"serialNumber":"e8fca248692ccc816c5fd518f7b419e1","reporter":"sys","userId":735}
2020-04-02 13:05:02.189 [http-nio-7777-exec-2] INFO  a2144a0159c94d8b899cb35fa70f39ff com.addx.iotcamera.aop.LogRequestAndResponseAspect - Rest API(a2144a0159c94d8b899cb35fa70f39ff): http://api.addx.live/lambda/replyrequest
Body:{"requestTime":1585832699,"msg":"success","serialNumber":"e8fca248692ccc816c5fd518f7b419e1","code":0,"id":"7025725ac4f74cc487441192a9701a06"}

logstash配置

input {
    file {
        path => ["/root/test.log"]
        start_position => "beginning"
        type => "test"
        codec => multiline {
            pattern => "^[0-9]{4}-[0-9]{2}-[0-9]{2}"
            negate => "true"
            what => "previous"
        }
    }
filter {
    if [type] == "test" {
        grok {
            match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:class} %{LOGLEVEL:loglevel}\s\s?( |%{NOTSPACE:requestid}|) %{NOTSPACE:logger} - %{JAVALOGMESSAGE:message}" }
            overwrite => [ "message" ]
        }
    }
    date {
        match => ["timestamp", "yyyy-MM-dd HH🇲🇲ss.SSS"]
        target => "@timestamp"
        remove_field => ["timestamp"]
    }
}
output {
    stdout {
       codec => rubydebug
    }
}

匹配后输出

{
          "tags" => [
        [0] "multiline"
    ],
    "@timestamp" => 2020-04-02T13:05:04.872Z,
          "path" => "/root/test.log",
      "@version" => "1",
     "requestid" => "24f7db0b88f047eea361d5493c567349",
        "logger" => "com.addx.iotcamera.aop.LogRequestAndResponseAspect",
      "loglevel" => "INFO",
       "message" => "Rest API(24f7db0b88f047eea361d5493c567349): http://api.addx.live/lambda/updatehardwareinfo\nBody:{"batteryEvent":0,"deviceType":"AIC","sdCardStatus":{"total":0,"formatStatus":1,"free":0},"serialNumber":"fd06c10707e1aab4f978f02678bde19c","charge":0,"buildTime":"2020-03-07-15-26-29","ip":"192.168.0.105","mcuVersion":"111","battery":47,"version":"0.2.22","userSn":"AICVAA8RMAM0400","MAC":"58:b3:fc:5e:7a:15","ap":"TP-LINK_EA3","uid":"fd06c10707e1aab4f978f02678bde19c","por":2,"autoSuspendDebug":{"maxDiff":9950,"maxDiffSrc":7,"srcBits":1027},"svnVersion":"g3e7ff90","wifiRssi":-49,"wifiChannel":0,"live":0,"reportTime":1585832705}",
          "host" => "elk-cn",
          "type" => "iot",
         "class" => "[http-nio-7777-exec-12]"
}
{
    "@timestamp" => 2020-04-02T13:05:02.017Z,
          "path" => "/root/test.log",
      "@version" => "1",
     "requestid" => "26a825125d0e46d7bbbc6f1490fa86ed",
        "logger" => "com.addx.iotcamera.service.ReportLogService",
      "loglevel" => "INFO",
       "message" => "{"msg":"AAAAAA","reportType":"keepaliveLoopSend","reportGroup":"live","seconds":30,"serialNumber":"e8fca248692ccc816c5fd518f7b419e1","reporter":"sys","userId":735}",
          "host" => "elk-cn",
          "type" => "iot",
         "class" => "[http-nio-7777-exec-14]"
}
^C[WARN ] 2020-04-02 16:57:27.565 [SIGINT handler] runner - SIGINT received. Shutting down.
[INFO ] 2020-04-02 16:57:27.656 [Converge PipelineAction::Stop<main>] observingtail - QUIT - closing all files and shutting down.
{
          "tags" => [
        [0] "multiline"
    ],
    "@timestamp" => 2020-04-02T13:05:02.189Z,
          "path" => "/root/test.log",
      "@version" => "1",
     "requestid" => "a2144a0159c94d8b899cb35fa70f39ff",
        "logger" => "com.addx.iotcamera.aop.LogRequestAndResponseAspect",
      "loglevel" => "INFO",
       "message" => "Rest API(a2144a0159c94d8b899cb35fa70f39ff): http://api.addx.live/lambda/replyrequest\nBody:{"requestTime":1585832699,"msg":"success","serialNumber":"e8fca248692ccc816c5fd518f7b419e1","code":0,"id":"7025725ac4f74cc487441192a9701a06"}",
          "host" => "elk-cn",
          "type" => "iot",
         "class" => "[http-nio-7777-exec-2]"
}


添加新评论