模糊查询-〉数据库全文检索-〉Lucene
一元分词（lucene内置）
Analyzer analyzer = new CJKAnalyzer();
TokenStream tokenStream = analyzer.TokenStream("", new StringReader("北京，Hi欢迎你们大家"));
Lucene.Net.Analysis.Token token = null;
while ((token = tokenStream.Next()) != null)
{
Console.WriteLine(token.TermText());
}
二元分词对汉字没有多大作用
using System.Collections;
using System.IO; using Lucene.Net.Analysis; namespace NSharp.SearchEngine.Lucene.Analysis.Cjk
{
/**/
/**
* Filters CJKTokenizer with StopFilter.
*
* @author Che, Dong
*/
public class CJKAnalyzer : Analyzer
{
//~ Static fields/initializers --------------------------------------------- /**/
/**
* An array containing some common English words that are not usually
* useful for searching and some double-byte interpunctions.
*/
public static string[] STOP_WORDS = {
"a", "and", "are", "as", "at", "be",
"but", "by", "for", "if", "in",
"into", "is", "it", "no", "not",
"of", "on", "or", "s", "such", "t",
"that", "the", "their", "then",
"there", "these", "they", "this",
"to", "was", "will", "with", "",
"www"
}; //~ Instance fields -------------------------------------------------------- /**/
/**
* stop word list
*/
private Hashtable stopTable; //~ Constructors ----------------------------------------------------------- /**/
/**
* Builds an analyzer which removes words in {@link #STOP_WORDS}.
*/
public CJKAnalyzer()
{
stopTable = StopFilter.MakeStopSet(STOP_WORDS);
} /**/
/**
* Builds an analyzer which removes words in the provided array.
*
* @param stopWords stop word array
*/
public CJKAnalyzer(string[] stopWords)
{
stopTable = StopFilter.MakeStopSet(stopWords);
} //~ Methods ---------------------------------------------------------------- /**/
/**
* get token stream from input
*
* @param fieldName lucene field name
* @param reader input reader
* @return TokenStream
*/
public override TokenStream TokenStream(string fieldName, TextReader reader)
{
TokenStream ts = new CJKTokenizer(reader);
return new StopFilter(ts, stopTable);
//return new StopFilter(new CJKTokenizer(reader), stopTable);
}
}
}
CJKAnalyzer
using System;
using System.Collections;
using System.IO; using Lucene.Net.Analysis; namespace NSharp.SearchEngine.Lucene.Analysis.Cjk
{
public class CJKTokenizer:Tokenizer
{
//~ Static fields/initializers --------------------------------------------- /**//** Max word length */
private static int MAX_WORD_LEN = ; /**//** buffer size: */
private static int IO_BUFFER_SIZE = ; //~ Instance fields -------------------------------------------------------- /**//** word offset, used to imply which character(in ) is parsed */
private int offset = ; /**//** the index used only for ioBuffer */
private int bufferIndex = ; /**//** data length */
private int dataLen = ; /**//**
* character buffer, store the characters which are used to compose <br>
* the returned Token
*/
private char[] buffer = new char[MAX_WORD_LEN]; /**//**
* I/O buffer, used to store the content of the input(one of the <br>
* members of Tokenizer)
*/
private char[] ioBuffer = new char[IO_BUFFER_SIZE]; /**//** word type: single=>ASCII double=>non-ASCII word=>default */
private string tokenType = "word"; /**//**
* tag: previous character is a cached double-byte character "C1C2C3C4"
* ----(set the C1 isTokened) C1C2 "C2C3C4" ----(set the C2 isTokened)
* C1C2 C2C3 "C3C4" ----(set the C3 isTokened) "C1C2 C2C3 C3C4"
*/
private bool preIsTokened = false; //~ Constructors ----------------------------------------------------------- /**//**
* Construct a token stream processing the given input.
*
* @param in I/O reader
*/
public CJKTokenizer(TextReader reader)
{
input = reader;
} //~ Methods ---------------------------------------------------------------- /**//**
* Returns the next token in the stream, or null at EOS.
* See http://java.sun.com/j2se/1.3/docs/api/java/lang/Character.UnicodeBlock.html
* for detail.
*
* @return Token
*
* @throws java.io.IOException - throw IOException when read error <br>
* hanppened in the InputStream
*
*/
public override Token Next()
{
/**//** how many character(s) has been stored in buffer */
int length = ; /**//** the position used to create Token */
int start = offset; while (true)
{
/**//** current charactor */
char c; offset++; /**//*
if (bufferIndex >= dataLen)
{
dataLen = input.read(ioBuffer); //Java中read读到最后不会出错，但.Net会，
bufferIndex = 0;
}
*/ if (bufferIndex >= dataLen )
{
if (dataLen== || dataLen>=ioBuffer.Length)//Java中read读到最后不会出错，但.Net会，所以此处是为了拦截异常
{
dataLen = input.Read(ioBuffer,,ioBuffer.Length);
bufferIndex = ;
}
else
{
dataLen=;
}
} if (dataLen ==)
{
if (length > )
{
if (preIsTokened == true)
{
length = ;
preIsTokened = false;
} break;
}
else
{
return null;
}
}
else
{
//get current character
c = ioBuffer[bufferIndex++];
} //if the current character is ASCII or Extend ASCII
if (IsAscii(c) || IsHALFWIDTH_AND_FULLWIDTH_FORMS(c))
{
if (IsHALFWIDTH_AND_FULLWIDTH_FORMS(c))
{
/**//** convert HALFWIDTH_AND_FULLWIDTH_FORMS to BASIC_LATIN */
int i = (int) c;
i = i - ;
c = (char) i;
}
//if the current character is a letter or "_" "+" "##region if the current character is a letter or "_" "+" "# // if the current character is a letter or "_" "+" "#"
if (char.IsLetterOrDigit(c) || ((c == '_') || (c == '+') || (c == '#')))
{ if (length == )
{
// "javaC1C2C3C4linux" <br>
// ^--: the current character begin to token the ASCII
// letter
start = offset - ;
}
else if (tokenType == "double")
{
// "javaC1C2C3C4linux" <br>
// ^--: the previous non-ASCII
// : the current character
offset--;
bufferIndex--;
tokenType = "single"; if (preIsTokened == true)
{
// there is only one non-ASCII has been stored
length = ;
preIsTokened = false; break;
}
else
{
break;
}
} // store the LowerCase(c) in the buffer
buffer[length++] = char.ToLower(c);
tokenType = "single";
// break the procedure if buffer overflowed!
if (length == MAX_WORD_LEN)
{
break;
}
}
else if (length > )
{
if (preIsTokened == true)
{
length = ;
preIsTokened = false;
}
else
{
break;
}
} }
else
{
// non-ASCII letter, eg."C1C2C3C4"#region // non-ASCII letter, eg."C1C2C3C4" // non-ASCII letter, eg."C1C2C3C4"
if (char.IsLetter(c))
{
if (length == )
{
start = offset - ;
buffer[length++] = c;
tokenType = "double";
}
else
{
if (tokenType == "single")
{
offset--;
bufferIndex--; //return the previous ASCII characters
break;
}
else
{
buffer[length++] = c;
tokenType = "double"; if (length == )
{
offset--;
bufferIndex--;
preIsTokened = true; break;
}
}
}
}
else if (length > )
{
if (preIsTokened == true)
{
// empty the buffer
length = ;
preIsTokened = false;
}
else
{
break;
}
}
}
} return new Token(new String(buffer, , length), start, start + length,
tokenType
);
} public bool IsAscii(char c)
{
return c< && c>=;
} public bool IsHALFWIDTH_AND_FULLWIDTH_FORMS(char c)
{
return c<=0xFFEF && c>=0xFF00;
}
}
}
CJKTokenizer
Analyzer analyzer = new PanGuAnalyzer();
TokenStream tokenStream = analyzer.TokenStream("", new StringReader("面向对象编程"));
Lucene.Net.Analysis.Token token = null;
while ((token = tokenStream.Next()) != null)
{
Console.WriteLine(token.TermText());
}
基于词库的分词算法：庖丁解牛（java用）、盘古分词(.net用)等，效率低
盘古分词要引用pangu.dll,pangu.lucene.Analyzer.dll,并拷贝词库
Analyzer analyzer = new PanGuAnalyzer();
TokenStream tokenStream = analyzer.TokenStream("", new StringReader("面向对象编程"));
Lucene.Net.Analysis.Token token = null;
while ((token = tokenStream.Next()) != null)
{
Console.WriteLine(token.TermText());
}
创建索引（读取查询文件并写入Lucene.net并生成索引文件）
string indexPath = @"C:\lucenedir";//注意和磁盘上文件夹的大小写一致，否则会报错。将创建的分词内容放在该目录下。
FSDirectory directory = FSDirectory.Open(new DirectoryInfo(indexPath), new NativeFSLockFactory());//指定索引文件(打开索引目录) FS指的是就是FileSystem
bool isUpdate = IndexReader.IndexExists(directory);//IndexReader:对索引进行读取的类。该语句的作用：判断索引库文件夹是否存在以及索引特征文件是否存在。
if (isUpdate)
{
//同时只能有一段代码对索引库进行写操作。当使用IndexWriter打开directory时会自动对索引库文件上锁。
//如果索引目录被锁定（比如索引过程中程序异常退出），则首先解锁（提示一下：如果我现在正在写着已经加锁了，但是还没有写完，这时候又来一个请求，那么不就解锁了吗？这个问题后面会解决）
if (IndexWriter.IsLocked(directory))
{
IndexWriter.Unlock(directory);
}
}
IndexWriter writer = new IndexWriter(directory, new PanGuAnalyzer(), !isUpdate, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED);//向索引库中写索引。这时在这里加锁。
for (int i = ; i <= ; i++)
{
string txt = File.ReadAllText(@"D:\讲课\0924班\OA\第七天\资料\测试文件\" + i + ".txt", System.Text.Encoding.Default);//注意这个地方的编码
Document document = new Document();//表示一篇文档。
//Field.Store.YES:表示是否存储原值。只有当Field.Store.YES在后面才能用doc.Get("number")取出值来.Field.Index. NOT_ANALYZED:不进行分词保存
document.Add(new Field("number", i.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED)); //Field.Index. ANALYZED:进行分词保存:也就是要进行全文的字段要设置分词 保存（因为要进行模糊查询） //Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS:不仅保存分词还保存分词的距离。
document.Add(new Field("body", txt, Field.Store.YES, Field.Index.ANALYZED, Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS));
writer.AddDocument(document); }
writer.Close();//会自动解锁。
directory.Close();//不要忘了Close，否则索引结果搜不到
搜索
string indexPath = @"C:\lucenedir";
string kw = "面向对象";
FSDirectory directory = FSDirectory.Open(new DirectoryInfo(indexPath), new NoLockFactory());
IndexReader reader = IndexReader.Open(directory, true);
IndexSearcher searcher = new IndexSearcher(reader);
//搜索条件
PhraseQuery query = new PhraseQuery();
//foreach (string word in kw.Split(' '))//先用空格，让用户去分词，空格分隔的就是词“计算机 专业”
//{
// query.Add(new Term("body", word));
//}
//query.Add(new Term("body","语言"));--可以添加查询条件，两者是add关系.顺序没有关系.
//query.Add(new Term("body", "大学生"));
query.Add(new Term("body", kw));//body中含有kw的文章
query.SetSlop();//多个查询条件的词之间的最大距离.在文章中相隔太远 也就无意义.（例如 “大学生”这个查询条件和"简历"这个查询条件之间如果间隔的词太多也就没有意义了。）
//TopScoreDocCollector是盛放查询结果的容器
TopScoreDocCollector collector = TopScoreDocCollector.create(, true);
searcher.Search(query, null, collector);//根据query查询条件进行查询，查询结果放入collector容器
ScoreDoc[] docs = collector.TopDocs(, collector.GetTotalHits()).scoreDocs;//得到所有查询结果中的文档,GetTotalHits():表示总条数 TopDocs(300, 20);//表示得到300（从300开始），到320（结束）的文档内容.
//可以用来实现分页功能
this.listBox1.Items.Clear();
for (int i = ; i < docs.Length; i++)
{
//
//搜索ScoreDoc[]只能获得文档的id,这样不会把查询结果的Document一次性加载到内存中。降低了内存压力，需要获得文档的详细内容的时候通过searcher.Doc来根据文档id来获得文档的详细内容对象Document.
int docId = docs[i].doc;//得到查询结果文档的id（Lucene内部分配的id）
Document doc = searcher.Doc(docId);//找到文档id对应的文档详细信息
this.listBox1.Items.Add(doc.Get("number") + "\n");// 取出放进字段的值
this.listBox1.Items.Add(doc.Get("body") + "\n");
this.listBox1.Items.Add("-----------------------\n");
}