package net.ripe.hadoop.pcap; import java.io.DataInputStream;
import java.io.IOException; import com.google.common.hash.Hashing; import net.ripe.hadoop.pcap.packet.HashPayloadPacket;
import net.ripe.hadoop.pcap.packet.Packet; public class HashPayloadPcapReader extends PcapReader {
public HashPayloadPcapReader(DataInputStream is) throws IOException {
super(is);
} @Override
protected Packet createPacket() {
return new HashPayloadPacket();
} @Override
protected boolean isReassemble() {
return true;
} @Override
protected boolean isPush() {
return false;
} @Override
protected void processPacketPayload(Packet packet, byte[] payload) {
if (payload.length > 0) {
packet.put(HashPayloadPacket.PAYLOAD_SHA1_HASH, Hashing.sha1().hashBytes(payload).toString());
packet.put(HashPayloadPacket.PAYLOAD_SHA256_HASH, Hashing.sha256().hashBytes(payload).toString());
packet.put(HashPayloadPacket.PAYLOAD_SHA512_HASH, Hashing.sha512().hashBytes(payload).toString());
packet.put(HashPayloadPacket.PAYLOAD_MD5_HASH, Hashing.md5().hashBytes(payload).toString());
}
}
}
HttpPcapReader
package net.ripe.hadoop.pcap; import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.util.LinkedList; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.Header;
import org.apache.http.HttpClientConnection;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestFactory;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseFactory;
import org.apache.http.impl.DefaultHttpRequestFactory;
import org.apache.http.impl.DefaultHttpResponseFactory;
import org.apache.http.impl.conn.DefaultClientConnection;
import org.apache.http.impl.io.AbstractSessionInputBuffer;
import org.apache.http.impl.io.AbstractSessionOutputBuffer;
import org.apache.http.impl.io.DefaultHttpRequestParser;
import org.apache.http.impl.io.DefaultHttpResponseParser;
import org.apache.http.io.HttpMessageParser;
import org.apache.http.io.SessionInputBuffer;
import org.apache.http.io.SessionOutputBuffer;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams; import com.google.common.base.Joiner; import net.ripe.hadoop.pcap.packet.HttpPacket;
import net.ripe.hadoop.pcap.packet.Packet; public class HttpPcapReader extends PcapReader{
public static final Log LOG = LogFactory.getLog(HttpPcapReader.class); public static final int HTTP_PORT = 80;
public static final String HEADER_PREFIX = "header_"; private HttpParams params = new BasicHttpParams();
private HttpRequestFactory reqFactory = new DefaultHttpRequestFactory();
private HttpResponseFactory respFactory = new DefaultHttpResponseFactory(); public HttpPcapReader(DataInputStream is) throws IOException {
super(is);
} @Override
protected Packet createPacket() {
return new HttpPacket();
} @Override
protected boolean isReassemble() {
return true;
} @Override
protected boolean isPush() {
return false;
} @Override
protected void processPacketPayload(Packet packet, final byte[] payload) {
HttpPacket httpPacket = (HttpPacket)packet;
Integer srcPort = (Integer)packet.get(Packet.SRC_PORT);
Integer dstPort = (Integer)packet.get(Packet.DST_PORT);
if ((HTTP_PORT == srcPort || HTTP_PORT == dstPort) &&
packet.containsKey(Packet.REASSEMBLED_FRAGMENTS) &&
PROTOCOL_TCP.equals(packet.get(Packet.PROTOCOL))) {
final SessionInputBuffer inBuf = new AbstractSessionInputBuffer() {
{
init(new ByteArrayInputStream(payload), 1024, params);
} @Override
public boolean isDataAvailable(int timeout) throws IOException {
return true;
}
};
final SessionOutputBuffer outBuf = new AbstractSessionOutputBuffer() {}; if (HTTP_PORT == srcPort) {
HttpMessageParser<HttpResponse> parser = new DefaultHttpResponseParser(inBuf, null, respFactory, params); HttpClientConnection conn = new DefaultClientConnection() {
{
init(inBuf, outBuf, params);
} @Override
protected void assertNotOpen() {} @Override
protected void assertOpen() {}
}; try {
HttpResponse response = parser.parse();
conn.receiveResponseEntity(response);
propagateHeaders(httpPacket, response.getAllHeaders());
} catch (IOException e) {
LOG.error("IOException when decoding HTTP response", e);
} catch (HttpException e) {
LOG.error("HttpException when decoding HTTP response", e);
}
} else if (HTTP_PORT == dstPort) {
HttpMessageParser<HttpRequest> parser = new DefaultHttpRequestParser(inBuf, null, reqFactory, params);
try {
HttpRequest request = parser.parse();
propagateHeaders(httpPacket, request.getAllHeaders());
} catch (IOException e) {
LOG.error("IOException when decoding HTTP request", e);
} catch (HttpException e) {
LOG.error("HttpException when decoding HTTP request", e);
}
}
}
} private void propagateHeaders(HttpPacket packet, Header[] headers) {
LinkedList<String> headerKeys = new LinkedList<String>();
for (Header header : headers) {
String headerKey = HEADER_PREFIX + header.getName().toLowerCase();
packet.put(headerKey, header.getValue());
}
packet.put(HttpPacket.HTTP_HEADERS, Joiner.on(',').join(headerKeys));
}
}
DnsPcapReader
package net.ripe.hadoop.pcap; import java.io.DataInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List; import net.ripe.hadoop.pcap.packet.DnsPacket;
import net.ripe.hadoop.pcap.packet.Packet; import org.xbill.DNS.Header;
import org.xbill.DNS.Message;
import org.xbill.DNS.Opcode;
import org.xbill.DNS.Rcode;
import org.xbill.DNS.Record;
import org.xbill.DNS.Section;
import org.xbill.DNS.Flags; public class DnsPcapReader extends PcapReader {
public static final int DNS_PORT = 53; public DnsPcapReader(DataInputStream is) throws IOException {
super(is);
} @Override
protected Packet createPacket() {
return new DnsPacket();
} @Override
protected boolean isReassemble() {
return true;
} @Override
protected boolean isPush() {
return false;
} @Override
protected void processPacketPayload(Packet packet, byte[] payload) {
DnsPacket dnsPacket = (DnsPacket)packet; if (DNS_PORT == (Integer)packet.get(Packet.SRC_PORT) || DNS_PORT == (Integer)packet.get(Packet.DST_PORT)) {
if (PROTOCOL_TCP.equals(packet.get(Packet.PROTOCOL)) &&
payload.length > 2) // TODO Support DNS responses with multiple messages (as used for XFRs)
payload = Arrays.copyOfRange(payload, 2, payload.length); // First two bytes denote the size of the DNS message, ignore them
try {
Message msg = new Message(payload);
Header header = msg.getHeader();
dnsPacket.put(DnsPacket.QUERYID, header.getID());
dnsPacket.put(DnsPacket.FLAGS, header.printFlags());
dnsPacket.put(DnsPacket.QR, header.getFlag(Flags.QR));
dnsPacket.put(DnsPacket.OPCODE, Opcode.string(header.getOpcode()));
dnsPacket.put(DnsPacket.RCODE, Rcode.string(header.getRcode()));
dnsPacket.put(DnsPacket.QUESTION, convertRecordToString(msg.getQuestion()));
dnsPacket.put(DnsPacket.QNAME, convertRecordOwnerToString(msg.getQuestion()));
dnsPacket.put(DnsPacket.QTYPE, convertRecordTypeToInt(msg.getQuestion()));
dnsPacket.put(DnsPacket.ANSWER, convertRecordsToStrings(msg.getSectionArray(Section.ANSWER)));
dnsPacket.put(DnsPacket.AUTHORITY, convertRecordsToStrings(msg.getSectionArray(Section.AUTHORITY)));
dnsPacket.put(DnsPacket.ADDITIONAL, convertRecordsToStrings(msg.getSectionArray(Section.ADDITIONAL)));
} catch (Exception e) {
// If we cannot decode a DNS packet we ignore it
}
}
} private String convertRecordToString(Record record) {
if (record == null)
return null; String recordString = record.toString();
recordString = normalizeRecordString(recordString);
return recordString;
} private String convertRecordOwnerToString(Record record) {
if (record == null)
return null;
String ownerString = record.getName().toString();
ownerString = ownerString.toLowerCase();
return ownerString;
} private int convertRecordTypeToInt(Record record) {
if (record == null)
return -1;
return record.getType();
} private List<String> convertRecordsToStrings(Record[] records) {
if (records == null)
return null; ArrayList<String> retVal = new ArrayList<String>(records.length);
for (Record record : records)
retVal.add(convertRecordToString(record));
return retVal;
} protected String normalizeRecordString(String recordString) {
if (recordString == null)
return null; // Reduce everything that is more than one whitespace to a single whitespace
recordString = recordString.replaceAll("\\s{2,}", " ");
// Replace tabs with a single whitespace
recordString = recordString.replaceAll("\\t{1,}", " ");
return recordString;
}
}
HashPayloadPcapReader的更多相关文章
随机推荐
-
PAT 03-树1 树的同构 (25分)
给定两棵树T1和T2.如果T1可以通过若干次左右孩子互换就变成T2,则我们称两棵树是"同构"的.例如图1给出的两棵树就是同构的,因为我们把其中一棵树的结点A.B.G的左右孩子互换后 ...
-
java产生随机数的几种方式
java产生随机数的几种方式 一.在j2se里我们可以使用Math.random()方法来产生一个随机数,这个产生的随机数是0-1之间的一个double,我们可以把他乘以一定的数,比如说乘以100,他 ...
-
Office Online简介
一.什么是Office Online Office Online 将 Microsoft Office 体验扩展到了 Web 浏览器,这使您可以直接在存储文档的 SharePoint 网站上处理文档, ...
-
rsyslog 直接读取日志,当日志截断后,不会继续发送
rsyslog web机器上日志被截断,那么就不会发送到rsyslog服务器 因为imfile记录了offset,然后你直接>导致offset还没到
-
Java8 Lamdba表达式 001
在一个已经存在的编程语言里非常少有对现有的生态系统起重大影响的新特性.Lambda表达式对于Java语言就是这样的意义的存在.简单来说,Lambda表达式提供了便利的方式去创建一个匿名的功能.提供了一 ...
-
【Spark2.0源码学习】-3.Endpoint模型介绍
Spark作为分布式计算框架,多个节点的设计与相互通信模式是其重要的组成部分. 一.组件概览 对源码分析,对于设计思路理解如下: RpcEndpoint: ...
-
Python 中 and 和 or 的短路原则
对于 and 来说: 如果第一个条件的结论为假,那么 and 前后两个条件组成的表达式计算结果一定为假,后面的条件计算机不会进行计算 对于 or 来说: 如果第一个条件的结论为真,那么 or 前后两个 ...
-
talend openstudio 在OracleInput组件中guess Schema 出现Database connection is failed 的错误
错误描述: talend openstudio 在OracleInput组件中guess Schema 出现Database connection is failed 的错误. 查看错误详情,发现错误 ...
-
Linux安装模式AppImage,Flatpak,Snap整理
本文只谈Linux世界用户较多的前2大主要分支, RedHat Red Hat Enterprise Linux 简称RHEL rpm (RedHat, CentOS, Fedora, Oracle. ...
-
腾讯微信被怼,iOS版微信不能打赏了
2017年4月19日,估计很多有着大量粉丝的微信自媒体作者会感到很不爽,因为他们的苹果粉丝再也无法很爽快地.肆意.任性地打赏他们了,按目前iphone手机的占有率,估计打赏率会掉一半以上. 据微信派微 ...