一、背景
- 当项目中用到集群环境时,一个springboot的应用,会发布到多个tomcat中,在排除故障的时候,必须要每个tomcat都登录上去查看,非常麻烦。所以就有了用filebeat+redis+ELK将分布式日志集中起来,分析的想法。
二、环境介绍
- 系统: Centos7
- 框架:springboot+logback
- 在springboot项目中做到了分不同环境执行不同log配置。dev开发环境只是简单的输出日志在控制台,test以及production环境输出到文件中,每个项目的日志都放在自己相应的文件夹下,并且做到按天分隔日志,当前正在写日志的log文件命令为info.log,使用filebeat去抓取此文件日志。
-
结构
- log-test.xml配置
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<property resource="application.properties" />
<!-- 项目名, 在application.properties中定义的 -->
<contextName>${projectName}</contextName>
<!-- 日志级别 -->
<property name="logLevel" value="INFO"></property>
<!-- 最大保存时间 -->
<property name="maxHistory" value="180"/>
<!-- 异步缓冲队列的深度,该值会影响性能.默认值为256 -->
<property name="queueSize" value="512"></property>
<!-- 日志路径,${catalina.home}是该项目当前tomcat的路径。 -->
<property name="logPath" value="${catalina.home}/logs/${projectName}"/>
<property name = "CONSOLE_PATTERN" value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %-40.40logger{35} - %msg%n"></property>
<!-- 控制台输出 -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出:%d表示日期,%thread表示线程名,%-5level:级别从左显示5个字符宽度%msg:日志消息,%n是换行符-->
<pattern>${CONSOLE_PATTERN}</pattern>
</encoder>
</appender>
<!-- 日志记录器,日期滚动记录 -->
<appender name="FILE_INFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
<!-- 正在记录的日志文件的路径及文件名 -->
<file>${logPath}/info.log</file>
<!-- 日志记录器的滚动策略,按日期记录 -->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- 归档的日志文件的路径-->
<fileNamePattern>${logPath}/info.%d{yyyy-MM-dd}.log</fileNamePattern>
<maxHistory>${maxHistory}</maxHistory>
</rollingPolicy>
<!-- 追加方式记录日志 -->
<append>true</append>
<!-- 日志文件的格式 -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>${CONSOLE_PATTERN}</pattern>
</encoder>
<!-- 此日志文件只记录info、warn、error 级别的 -->
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>info</level>
</filter>
</appender>
<appender name="ASYNC_LOG_INFO" class="ch.qos.logback.classic.AsyncAppender">
<!-- 不丢失日志.默认的,如果队列的80%已满,则会丢弃TRACT、DEBUG、INFO级别的日志 -->
<discardingThreshold>0</discardingThreshold>
<!-- 更改默认的队列的深度,该值会影响性能.默认值为256 -->
<queueSize>${queueSize}</queueSize>
<appender-ref ref="FILE_INFO"/>
</appender>
<root level="${logLevel}">
<!-- appender referenced after it is defined -->
<appender-ref ref="STDOUT"/>
<appender-ref ref="ASYNC_LOG_INFO"/>
</root>
</configuration>
-
贴一下application.properties中配置的projectName
-
最终的日志文件路径如下
- 日志内容如下
2019-04-12 15:20:31.828 [o-9099-exec-702] INFO o.s.web.servlet.DispatcherServlet - FrameworkServlet 'dispatcherServlet': initialization started
2019-04-12 15:20:31.853 [o-9099-exec-702] INFO o.s.web.servlet.DispatcherServlet - FrameworkServlet 'dispatcherServlet': initialization completed in 24 ms
2019-04-12 15:20:31.902 [o-9099-exec-702] ERROR o.s.b.w.servlet.support.ErrorPageFilter - Forwarding to error page from request [/user/test1] due to exception [null]
java.lang.NullPointerException: null
at com.my.test.controller.UserController.test(UserController.java:61)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:209)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:136)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:102)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:877)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:783)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:991)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:925)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:974)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:866)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:635)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:851)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:230)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:109)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:93)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.boot.web.servlet.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:130)
at org.springframework.boot.web.servlet.support.ErrorPageFilter.access$000(ErrorPageFilter.java:66)
at org.springframework.boot.web.servlet.support.ErrorPageFilter$1.doFilterInternal(ErrorPageFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.boot.web.servlet.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:123)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:200)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:478)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:80)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:624)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:341)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:783)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:798)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1441)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
三、配置filebeat收集info文件日志
- 编辑filebeat.yml
vi /etc/filebeat/filebeat.yml
- 输入以下内容
###################### Filebeat Configuration Example #########################
# This file is an example configuration file highlighting only the most common
# options. The filebeat.reference.yml file from the same directory contains all the
# supported options with more comments. You can use it as a reference.
#
# You can find the full configuration reference here:
# https://www.elastic.co/guide/en/beats/filebeat/index.html
# For more available modules and options, please see the filebeat.reference.yml sample
# configuration file.
#=========================== Filebeat inputs =============================
filebeat.inputs:
#=========== nginx access-json日志 ==============
- type: log
enabled: true
paths:
- /var/log/nginx/access-json.log #指明读取文件的位置
tags: ["nginx-access"] #用于logstash过滤
#=========== nginx error日志 ==============
- type: log
enabled: true
paths:
- /var/log/nginx/error.log #指明读取文件的位置
tags: ["nginx-error"] #用于logstash过滤
#=========== tomcat access日志 ==============
- type: log
enabled: true
paths:
- /root/server/apache-tomcat-aic/logs/localhost_access_log.Y-M-D.txt #指明tomcat-access日志文件的位置
tags: ["tomcat-access"] #用于logstash过滤
#=========== tomcat job项目日志 ==============
- type: log
enabled: true
paths:
- /root/server/apache-tomcat-aic/logs/aic-job/info.log #指明Springboot项目日志文件的位置
multiline:
pattern: '^\s*(\d{4}|\d{2})\-(\d{2}|[a-zA-Z]{3})\-(\d{2}|\d{4})' # 指定匹配的表达式(匹配以 2017-11-15 08:04:23:889 时间格式开头的字符串)
negate: true # 是否匹配到
match: after # 合并到上一行的末尾, 为了error日志
max_lines: 1000 # 最大的行数
timeout: 30s # 如果在规定的时候没有新的日志事件就不等待后面的日志
tags: ["aic-job"] #用于logstash过滤
#============================= Filebeat modules ===============================
filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml
# Set to true to enable config reloading
reload.enabled: false
# Period on which files under path should be checked for changes
#reload.period: 10s
#==================== Elasticsearch template setting ==========================
setup.template.settings:
index.number_of_shards: 3
#index.codec: best_compression
#_source.enabled: false
#================================ Outputs =====================================
# Configure what output to use when sending the data collected by the beat.
#-------------------------- Redis output ------------------------------
output.redis:
hosts: ["192.168.1.110:6379"] #输出到redis的机器
password: "123456"
key: "filebeat:test16" #redis中日志数据的key值ֵ
db: 0
timeout: 5
#================================ Processors =====================================
# Configure processors to enhance or manipulate events generated by the beat.
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
四、配置logstash过滤项目info文件日志
- 新建一个logstash的配置文件
vi /etc/logstash/conf.d/myJob.conf
- 输入以下内容
input {
redis {
data_type =>"list"
key =>"filebeat:test16"
host =>"192.168.1.110"
port => 6379
password => "123456"
threads => "8"
db => 0
#codec => json
}
}
filter {
if "aic-job" in [tags]{
grok {
match => ["message", "%{TIMESTAMP_ISO8601:time}\s* \s*%{NOTSPACE:thread-id}\s* \s*%{LOGLEVEL:level}\s* \s*%{JAVACLASS:class}\s* \- \s*%{JAVALOGMESSAGE:logmessage}\s*"]
}
}
mutate {
remove_field => "log"
remove_field => "beat"
remove_field => "meta"
remove_field => "prospector"
remove_field => "[host][os]"
}
}
output {
if "aic-job" in [tags]{
elasticsearch {
hosts => ["192.168.1.110:9200"]
index => "logstash-test16-tomcat-aic-job-%{+yyyy.MM.dd}"
}
}
}
- 重启logstash,查看启动日志是否报错。
systemctl restart logstash #重启
tail -f /var/log/logstash/logstash-plain.log #查看运行日志