ELK - log4j2自定义Layout以json格式输出日志到logstash，支持收集多个项目的日志

logstash 新增配置文件 log4j2.conf

input {
#开启远程输入日志服务
    tcp {
        port => "4560"
        mode => "server"
        type => "log4j2"
    }
}

filter {
#将日志转成json对象
    json {
        source => "message"
    }
#将远程客户端的日志时间设置为插入时间，不设置默认为当前系统时间，可能会存在时间误差
    date {
        match => ["time", "yyyy-MM-dd HH:mm:ss.SSS"]
        remove_field => ["time"]
    }
}

output {
    elasticsearch {
        hosts => ["192.168.1.138:9200"]
        index  => "application-%{+YYYY.MM.dd}"
    }
}

linux 系统下启动 logstash

bin/logstash -f config/log4j.conf

在项目代码里添加自定义Layout类

import com.fasterxml.jackson.core.JsonProcessingException;
import com.gzkit.core.utils.JacksonUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.Node;
import org.apache.logging.log4j.core.config.plugins.*;
import org.apache.logging.log4j.core.layout.AbstractStringLayout;
import org.apache.logging.log4j.core.layout.PatternLayout;
import org.apache.logging.log4j.core.layout.PatternSelector;
import org.apache.logging.log4j.core.pattern.RegexReplacement;

import java.io.File;
import java.nio.charset.Charset;

/**
 * ELK日志格式
 * @Author lnk
 * @Date 2018/3/13
 */
@Plugin(name = "ElkJsonPatternLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true)
public class ElkJsonPatternLayout extends AbstractStringLayout {
/** 项目路径 */
private static String PROJECT_PATH;

private PatternLayout patternLayout;

private String projectName;
private String logType;

static {
        PROJECT_PATH = new File("").getAbsolutePath();
    }

private ElkJsonPatternLayout(Configuration config, RegexReplacement replace, String eventPattern,
                                 PatternSelector patternSelector, Charset charset, boolean alwaysWriteExceptions,
boolean noConsoleNoAnsi, String headerPattern, String footerPattern, String projectName, String logType) {
super(config, charset,
                PatternLayout.createSerializer(config, replace, headerPattern, null, patternSelector, alwaysWriteExceptions,
                        noConsoleNoAnsi),
                PatternLayout.createSerializer(config, replace, footerPattern, null, patternSelector, alwaysWriteExceptions,
                        noConsoleNoAnsi));

this.projectName = projectName;
this.logType = logType;
this.patternLayout = PatternLayout.newBuilder()
                .withPattern(eventPattern)
                .withPatternSelector(patternSelector)
                .withConfiguration(config)
                .withRegexReplacement(replace)
                .withCharset(charset)
                .withAlwaysWriteExceptions(alwaysWriteExceptions)
                .withNoConsoleNoAnsi(noConsoleNoAnsi)
                .withHeader(headerPattern)
                .withFooter(footerPattern)
                .build();
    }

@Override
public String toSerializable(LogEvent event) {
//在这里处理日志内容
        String message = patternLayout.toSerializable(event);
        String jsonStr = new JsonLoggerInfo(projectName, message, event.getLevel().name(), logType, event.getTimeMillis()).toString();
return jsonStr + "\n";
    }

@PluginFactory
public static ElkJsonPatternLayout createLayout(
            @PluginAttribute(value = "pattern", defaultString = PatternLayout.DEFAULT_CONVERSION_PATTERN) final String pattern,
@PluginElement("PatternSelector") final PatternSelector patternSelector,
@PluginConfiguration final Configuration config,
@PluginElement("Replace") final RegexReplacement replace,
// LOG4J2-783 use platform default by default, so do not specify defaultString for charset
@PluginAttribute(value = "charset") final Charset charset,
@PluginAttribute(value = "alwaysWriteExceptions", defaultBoolean = true) final boolean alwaysWriteExceptions,
@PluginAttribute(value = "noConsoleNoAnsi", defaultBoolean = false) final boolean noConsoleNoAnsi,
@PluginAttribute("header") final String headerPattern,
@PluginAttribute("footer") final String footerPattern,
@PluginAttribute("projectName") final String projectName,
@PluginAttribute("logType") final String logType) {


return new ElkJsonPatternLayout(config, replace, pattern, patternSelector, charset,
                alwaysWriteExceptions, noConsoleNoAnsi, headerPattern, footerPattern, projectName, logType);
    }

/**
 * 输出的日志内容
 */
public static class JsonLoggerInfo{
/** 项目名 */
private String projectName;
/** 项目目录路径 */
private String projectPath;
/** 日志信息 */
private String message;
/** 日志级别 */
private String level;
/** 日志分类 */
private String logType;
/** 日志时间 */
private String time;

public JsonLoggerInfo(String projectName, String message, String level, String logType, long timeMillis) {
this.projectName = projectName;
this.projectPath = PROJECT_PATH;
this.message = message;
this.level = level;
this.logType = logType;
this.time = DateFormatUtils.format(timeMillis, "yyyy-MM-dd HH:mm:ss.SSS");
        }

public String getProjectName() {
return projectName;
        }

public String getProjectPath() {
return projectPath;
        }

public String getMessage() {
return message;
        }

public String getLevel() {
return level;
        }

public String getLogType() {
return logType;
        }

public String getTime() {
return time;
        }

@Override
public String toString() {
try {
return new ObjectMapper().writeValueAsString(this);
            } catch (JsonProcessingException e) {
                e.printStackTrace();
            }
return null;
        }
    }
}

修改 log4j2.xml 添加下面几个节点

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" monitorInterval="30" packages="com.xxx.logging.log4j">
<Properties>
<Property name="PROJECT_NAME">my-project</Property>
<Property name="ELK_LOG_PATTERN">${sys:PID} %c{3.}.%t %m%n</Property>
</Properties>
<Appenders>
<!-- 输出业务日志到ELK日志系统 -->
<Socket name="elk" host="192.168.1.138" port="4560">
<ElkJsonPatternLayout pattern="${ELK_LOG_PATTERN}" projectName="${PROJECT_NAME}" logType="business" />
</Socket>
</Appenders>
<Loggers>
<Logger name="com.xxx" level="info">
<AppenderRef ref="elk"/>
</Logger>
</Loggers>
</Configuration>

1. 根节点 Configuration 添加属性 packages 值为自定义Layout类的包路径，packages属性是扫描指定包下log4j的自定义扩展类。
2. 节点 Appenders 添加节点 Socket 使用 Socket 方式输出去远程 logstath。
3. 节点 Logger 指定日志使用Socket输出方式。
4. 上面配置文件中只贴出了关键配置内容，并不适合直接应用到实现项目中，将上面关键配置添加到自己项目的配置中即可。

在 Kibana 查看日志
1. 先使用log4j生成几条日志。
2. 进入 Kibana 添加新的 index索引 application-*，并刷一下字段，以便json日志的字段生效。
3. 然后去看application-*，可以根据 projectName 字段过滤查看不同项目的日志。