因需求，现需分词接口，故记录之。
1、需要依赖：
<!-- https://mvnrepository.com/artifact/com.janeluo/ikanalyzer -->
<dependency>
<groupId>com.janeluo</groupId>
<artifactId>ikanalyzer</artifactId>
<version>2012_u6</version>
</dependency>
maven依赖
2、完整代码如下：
public JSONArray entropy(String content, Integer quantity) throws Exception {
List<String> words = extract(DelHtmlTagUtil.delHTMLTag(content), quantity);
JSONArray array = calculateWordEntropy(words);
return array;
}
/**
* 传入String类型的文章，智能提取单词放入list
*
* @param content 传入分词的内容
* @param quantity 截取关键字在几个单词以上的数量，默认为1
* @return
*/
private List<String> extract(String content, Integer quantity) throws IOException {
List<String> list = Lists.newArrayList();
StringReader reader = new StringReader(content);
IKSegmenter ik = new IKSegmenter(reader, true);
Lexeme lex = null;
while ((lex = ik.next()) != null) {
//String typeString = lex.getLexemeTypeString(); 词语类型
String word = lex.getLexemeText();
if (word.length() > quantity) {//判断截取关键字在几个单词以上的数量
list.add(word);
}
}
return list;
}
private JSONArray calculateWordEntropy(List<String> words) throws Exception{
int length = words.size();
ArrayList<String[]> wordList = new ArrayList<String[]>();
// 将分好的词每3个一组存到数组中
for (int i = 0; i < length; i++) {
String[] wordSeg = new String[3];
if (i == 0) {
wordSeg[0] = "null";
wordSeg[1] = words.get(i);
wordSeg[2] = words.get(i + 1);
} else if (i == length - 1) {
wordSeg[0] = words.get(i - 1);
wordSeg[1] = words.get(i);
wordSeg[2] = "null";
} else {
wordSeg[0] = words.get(i - 1);
wordSeg[1] = words.get(i);
wordSeg[2] = words.get(i + 1);
}
wordList.add(wordSeg);
}
// 去除重复的词
List<String> lists = Lists.newArrayList();
for (int l = 0; l < length; l++) {
lists.add(words.get(l));
}
List<String> tempList = Lists.newArrayList();
for (String str : lists) {
if (!(tempList.contains(str))) {
tempList.add(str);
}
}
String[] wordClean = new String[tempList.size()];
for (int m = 0; m < tempList.size(); m++) {
wordClean[m] = tempList.get(m);
}
// 统计每个词的词频
int[] frequent = new int[wordClean.length];
for (int j = 0; j < wordClean.length; j++) {
int count = 0;
for (int k = 0; k < words.size(); k++) {
if (wordClean[j].equals(words.get(k))) {
count++;
}
}
frequent[j] = count;
}
// 将三元组中中间的那个词相同的存到一个list中，然后计算该词的信息熵
double[] allEntropy = new double[wordClean.length];
for (int n = 0; n < wordClean.length; n++) {
ArrayList<String[]> wordSegList = new ArrayList<String[]>();
int count = 1;
for (int p = 0; p < wordList.size(); p++) {
String[] wordSegStr = wordList.get(p);
if (wordSegStr[1].equals(wordClean[n])) {
count++;
wordSegList.add(wordSegStr);
}
}
String[] leftword = new String[wordSegList.size()];
String[] rightword = new String[wordSegList.size()];
// 计算左信息熵
for (int i = 0; i < wordSegList.size(); i++) {
String[] left = wordSegList.get(i);
leftword[i] = left[0];
}
// 去除左边重复的词
List<String> listsLeft = new ArrayList<String>();
for (int l = 0; l < leftword.length; l++) {
listsLeft.add(leftword[l]);
}
List<String> tempListLeft = new ArrayList<String>();
for (String str : listsLeft) {
if (!(tempListLeft.contains(str))) {
tempListLeft.add(str);
}
}
String[] leftWordClean = new String[tempListLeft.size()];
for (int m = 0; m < tempListLeft.size(); m++) {
leftWordClean[m] = tempListLeft.get(m);
}
// 统计左边每个词的词频
int[] leftFrequent = new int[leftWordClean.length];
for (int j = 0; j < leftWordClean.length; j++) {
int leftcount = 0;
for (int k = 0; k < leftword.length; k++) {
if (leftWordClean[j].equals(leftword[k])) {
leftcount++;
}
}
leftFrequent[j] = leftcount;
}
// 计算左熵值
double leftEntropy = 0;
for (int i = 0; i < leftFrequent.length; i++) {
double a = (double) leftFrequent[i] / count;
double b = Math.log((double) leftFrequent[i] / count);
leftEntropy += -a * b;
// leftEntropy +=
// (-(double)(leftFrequent[i]/count))*Math.log((double)(leftFrequent[i]/count));
}
// 计算右信息熵
for (int i = 0; i < wordSegList.size(); i++) {
String[] right = wordSegList.get(i);
rightword[i] = right[2];
}
// 去除右边重复的词
List<String> listsRight = new ArrayList<String>();
for (int l = 0; l < rightword.length; l++) {
listsRight.add(rightword[l]);
}
List<String> tempListRight = new ArrayList<String>();
for (String str : listsRight) {
if (!(tempListRight.contains(str))) {
tempListRight.add(str);
}
}
String[] rightWordClean = new String[tempListRight.size()];
for (int m = 0; m < tempListRight.size(); m++) {
rightWordClean[m] = tempListRight.get(m);
}
// 统计右边每个词的词频
int[] rightFrequent = new int[rightWordClean.length];
for (int j = 0; j < rightWordClean.length; j++) {
int rightcount = 0;
for (int k = 0; k < rightword.length; k++) {
if (rightWordClean[j].equals(rightword[k])) {
rightcount++;
}
}
rightFrequent[j] = rightcount;
}
// 计算右熵值
double rightEntropy = 0.0;
for (int i = 0; i < rightFrequent.length; i++) {
double a = (double) rightFrequent[i] / count;
double b = Math.log((double) rightFrequent[i] / count);
rightEntropy += -a * b;
// rightEntropy +=
// (-(double)(rightFrequent[i]/count))*Math.log((double)(rightFrequent[i]/count));
}
// 计算词的总信息熵
double wordEntropy = leftEntropy + rightEntropy;
allEntropy[n] = wordEntropy;
}
JSONArray list = new JSONArray();
for (int i = 0; i < allEntropy.length; i++) {
JSONObject obj = new JSONObject();
obj.put("name", wordClean[i]);
obj.put("entropy", allEntropy[i]);
list.add(obj);
}
Collections.sort(list, (o1, o2) -> {
Double d1 = ((JSONObject) o1).getDouble("entropy");
Double d2 = ((JSONObject) o2).getDouble("entropy");
return d2.compareTo(d1);
});
return list;
}
处理代理