一、概念认识
1、常用的Analyer
SimpleAnalyzer、StopAnalyzer、WhitespaceAnalyzer、StandardAnalyzer
2、TokenStream
分词器做好处理之后得到的一个流，这个流中存储了分词的各种信息，可以通过TokenStream有效的获取到分词单元信息生成的流程
在这个流中所需要存储的数据
3、Tokenizer
主要负责接收字符流Reader,将Reader进行分词操作。有如下一些实现类
4、TokenFilter
将分词的语汇单元，进行各种各样过滤
5、内置常用分词器分词进行分词的差异
|
1
2
3
4
5
6
7
8
9
10
11
12
13
|
public static void displayToken(String str,Analyzer a) {
try {
TokenStream stream = a.tokenStream("content",new StringReader(str));
//创建一个属性，这个属性会添加流中，随着这个TokenStream增加
CharTermAttribute cta = stream.addAttribute(CharTermAttribute.class);
while(stream.incrementToken()) {
System.out.print("["+cta+"]");
}
System.out.println();
} catch (IOException e) {
e.printStackTrace();
}
}
|
|
1
2
3
4
5
6
7
8
9
|
public Map<String,Analyzer> toMap(String[] str,Analyzer ... analyzers){
Map<String,Analyzer> analyzerMap = new HashMap<String,Analyzer>();
int i =0;
for(Analyzer a : analyzers){
analyzerMap.put(str[i], a);
i++;
}
return analyzerMap;
}
|
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
|
@Test public void test01() {
String[] str ={"StandardAnalyzer","StopAnalyzer","SimpleAnalyzer","WhitespaceAnalyzer"};
Map<String,Analyzer> analyzerMap = new HashMap<String,Analyzer>();
Analyzer a1 = new StandardAnalyzer(Version.LUCENE_35);
Analyzer a2 = new StopAnalyzer(Version.LUCENE_35);
Analyzer a3 = new SimpleAnalyzer(Version.LUCENE_35);
Analyzer a4 = new WhitespaceAnalyzer(Version.LUCENE_35);
analyzerMap = toMap(str,a1,a2,a3,a4);
String txt = "this is my house,I am come from bilibili qiansongyi," +
"My email is dumingjun@gmail.com,My QQ is 888168";
for(String analyzer : analyzerMap.keySet()){
System.out.println(analyzer);
AnalyzerUtils.displayToken(txt, analyzerMap.get(analyzer));
System.out.println("==============================");
}
}
|
6、中文分词
|
1
2
3
4
5
6
7
8
9
|
public void toMap(String txt,Analyzer ... analyzers){
for(Analyzer a : analyzers){
int start = a.toString().lastIndexOf(".")+1;
int end = a.toString().lastIndexOf("@")-1;
System.out.println(a.toString().substring(start, end));
AnalyzerUtils.displayToken(txt, a);
System.out.println("====================");
}
}
|
|
1
2
3
4
5
6
7
8
9
10
|
public void test02() {
Analyzer a1 = new StandardAnalyzer(Version.LUCENE_35);
Analyzer a2 = new StopAnalyzer(Version.LUCENE_35);
Analyzer a3 = new SimpleAnalyzer(Version.LUCENE_35);
Analyzer a4 = new WhitespaceAnalyzer(Version.LUCENE_35);
Analyzer a5 = new MMSegAnalyzer(new File("D:\\lucene\\mmseg4j\\data"));
String txt = "我来自中国广东省广州市天河区的小白";
toMap(txt,a1,a2,a3,a4,a5);
}
|
7、位置增量、位置偏移量、分词单元、分词器的类型
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
public static void displayAllTokenInfo(String str,Analyzer a) {
try {
TokenStream stream = a.tokenStream("content",new StringReader(str));
//位置增量的属性，存储语汇单元之间的距离
PositionIncrementAttribute pia =
stream.addAttribute(PositionIncrementAttribute.class);
//每个语汇单元的位置偏移量
OffsetAttribute oa = stream.addAttribute(OffsetAttribute.class);
//存储每一个语汇单元的信息（分词单元信息）
CharTermAttribute cta = stream.addAttribute(CharTermAttribute.class);
//使用的分词器的类型信息
TypeAttribute ta = stream.addAttribute(TypeAttribute.class);
for(;stream.incrementToken();) {
System.out.print(pia.getPositionIncrement()+":");
System.out.print(cta+"["+oa.startOffset()+"-"+oa.endOffset()+"]-->"+ta.type()+"\n");
}
} catch (Exception e) {
e.printStackTrace();
}
}
|
8、停用分词器
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
|
public class MyStopAnalyzer extends Analyzer {
@SuppressWarnings("rawtypes")
private Set stops;
@SuppressWarnings("unchecked")
public MyStopAnalyzer(String[]sws) {
//会自动将字符串数组转换为Set
stops = StopFilter.makeStopSet(Version.LUCENE_35, sws, true);
//将原有的停用词加入到现在的停用词
stops.addAll(StopAnalyzer.ENGLISH_STOP_WORDS_SET);
}
public MyStopAnalyzer() {
//获取原有的停用词
stops = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
}
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
//为这个分词器设定过滤链和Tokenizer
return new StopFilter(Version.LUCENE_35,
new LowerCaseFilter(Version.LUCENE_35,
new LetterTokenizer(Version.LUCENE_35,reader)), stops);
}
} |
|
1
2
3
4
5
6
7
8
|
@Test public void test04() {
Analyzer a1 = new MyStopAnalyzer(new String[]{"I","you","hate"});
Analyzer a2 = new MyStopAnalyzer();
String txt = "how are you thank you I hate you";
AnalyzerUtils.displayToken(txt, a1);
AnalyzerUtils.displayToken(txt, a2);
}
|
9、简单实现同义词索引
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
|
public class MySameAnalyzer extends Analyzer {
private SamewordContext samewordContext;
public MySameAnalyzer(SamewordContext swc) {
samewordContext = swc;
}
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
Dictionary dic = Dictionary.getInstance("D:\\lucene\\mmseg4j\\data");
return new MySameTokenFilter(
new MMSegTokenizer(new MaxWordSeg(dic), reader),samewordContext);
}
} |
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
|
public class MySameTokenFilter extends TokenFilter {
private CharTermAttribute cta = null;
private PositionIncrementAttribute pia = null;
private AttributeSource.State current;
private Stack<String> sames = null;
private SamewordContext samewordContext;
protected MySameTokenFilter(TokenStream input,SamewordContext samewordContext) {
super(input);
cta = this.addAttribute(CharTermAttribute.class);
pia = this.addAttribute(PositionIncrementAttribute.class);
sames = new Stack<String>();
this.samewordContext = samewordContext;
}
@Override
public boolean incrementToken() throws IOException {
if(sames.size()>0) {
//将元素出栈，并且获取这个同义词
String str = sames.pop();
//还原状态
restoreState(current);
cta.setEmpty();
cta.append(str);
//设置位置0
pia.setPositionIncrement(0);
return true;
}
if(!this.input.incrementToken()) return false;
if(addSames(cta.toString())) {
//如果有同义词将当前状态先保存
current = captureState();
}
return true;
}
private boolean addSames(String name) {
String[] sws = samewordContext.getSamewords(name);
if(sws!=null) {
for(String str:sws) {
sames.push(str);
}
return true;
}
return false;
}
} |
|
1
2
3
4
5
6
7
8
9
10
11
12
13
|
public class SimpleSamewordContext2 implements SamewordContext {
Map<String,String[]> maps = new HashMap<String,String[]>();
public SimpleSamewordContext2() {
maps.put("中国",new String[]{"天朝","大陆"});
}
@Override
public String[] getSamewords(String name) {
return maps.get(name);
}
} |
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
|
@Test public void test05() {
try {
Analyzer a2 = new MySameAnalyzer(new SimpleSamewordContext2());
String txt = "我来自中国广东省广州市天河区的小白";
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir,new IndexWriterConfig(Version.LUCENE_35, a2));
Document doc = new Document();
doc.add(new Field("content",txt,Field.Store.YES,Field.Index.ANALYZED));
writer.addDocument(doc);
writer.close();
IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
TopDocs tds = searcher.search(new TermQuery(new Term("content","咱")),10);
// Document d = searcher.doc(tds.scoreDocs[0].doc);// System.out.println(d.get("content")); AnalyzerUtils.displayAllTokenInfo(txt, a2);
} catch (CorruptIndexException e) {
e.printStackTrace();
} catch (LockObtainFailedException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
|
