（转）solr排序OOM解决方法

时间：2023-03-09 09:14:48
（转）solr排序OOM解决方法

转自 http://topcat.iteye.com/blog/1293650

问题

lucene使用排序时会将被排序字段全部加入内存再进行排序，当多次使用不同字段进行排序时会造成OOM问题

解决方案

修改lucene源码在每次查询完成后将排序所使用的FieldCache里的缓存清空

分别需要修改两个工程
  1. lucene2.4
  2. solr1.3

Lucene2.4中要修改的类或接口有

  • org.apache.lucene.index.IndexReader
添加成员变量
public String m_cacheKey = "cmhk_oom_bugfix";
  • org.apache.lucene.search.Searcher
添加抽象方法

abstract public void release();
在它的实现类中除org.apache.lucene.search.IndexSearcher外的都加上一个这个抽象方法的空实现
  • org.apache.lucene.search.IndexSearcher
添加方法
public void release() {
// 清除缓存数据
if (null !=
reader.m_cacheKey && !"".equals(reader.m_cacheKey))
{
if (null != FieldSortedHitQueue.Comparators)
{
FieldSortedHitQueue.Comparators.ReleaseCache(reader.m_cacheKey);
}
FieldCache.DEFAULT.ReleaseFieldCache(reader.m_cacheKey);
ExtendedFieldCache.EXT_DEFAULT.ReleaseExtendedFieldCache(reader.m_cacheKey);
// 立即回收垃圾
//
System.gc();
}
}
  • org.apache.lucene.search.FieldCache
添加方法
void ReleaseFieldCache(String
strCacheKey);
  • org.apache.lucene.search.FieldCacheImpl
添加方法
public void
ReleaseFieldCache(String strCacheKey)
{
bytesCache.ReleaseCache(strCacheKey);
shortsCache.ReleaseCache(strCacheKey);
intsCache.ReleaseCache(strCacheKey);
floatsCache.ReleaseCache(strCacheKey);
stringsCache.ReleaseCache(strCacheKey);
stringsIndexCache.ReleaseCache(strCacheKey);
autoCache.ReleaseCache(strCacheKey);
customCache.ReleaseCache(strCacheKey);
}

  • org.apache.lucene.search.FieldCacheImpl.Cache
添加方法
// 用于清除缓存
public
void ReleaseCache(String strCacheKey)
{
if (null != readerCache) {
synchronized (readerCache) {
if (null !=
readerCache.get(strCacheKey)) {
((Map)
readerCache.get(strCacheKey)).clear();
}
}
}
}
修改org.apache.lucene.search.FieldCacheImpl.Cache.get(IndexReader,
Object)方法
public Object get(IndexReader reader, Object key)
throws IOException {
Map
innerCache;
Object value;
synchronized (readerCache) {
//改键值为静态值
innerCache = (Map)readerCache.get(reader.m_cacheKey);
// innerCache = (Map)
readerCache.get(reader);
if
(innerCache == null) {
innerCache = new
HashMap();
//改键值为静态值
readerCache.put(reader.m_cacheKey,innerCache);
//readerCache.put(reader, innerCache);
value
= null;
} else {
value = innerCache.get(key);
}
if (value == null)
{
value = new
CreationPlaceholder();
innerCache.put(key,
value);
}
}
if
(value instanceof CreationPlaceholder)
{
synchronized (value)
{
CreationPlaceholder progress =
(CreationPlaceholder) value;
if
(progress.value == null)
{
progress.value = createValue(reader,
key);
synchronized
(readerCache) {
innerCache.put(key,
progress.value);
}
}
return
progress.value;
}
}
return
value;
}
}
  • org.apache.lucene.search.ExtendedFieldCache
添加方法
//用于清除缓存
void
ReleaseExtendedFieldCache(String
strCacheKey);
  • org.apache.lucene.search.ExtendedFieldCacheImpl
添加方法
// 用于清除缓存
public
void ReleaseExtendedFieldCache(String
strCacheKey)
{
longsCache.ReleaseCache(strCacheKey);
doublesCache.ReleaseCache(strCacheKey);
autoCache.ReleaseCache(strCacheKey);
}

solr1.3中要修改的类或接口

  • org.apache.solr.search.SolrIndexSearcher
添加方法
public void
release(){
searcher.release();
}
  • org.apache.solr.core.SolrCore
修改私有方法
private RefCounted<SolrIndexSearcher>
newHolder(SolrIndexSearcher newSearcher) {
RefCounted<SolrIndexSearcher> holder
= new
RefCounted<SolrIndexSearcher>(newSearcher) {
public void
close() {
try {
synchronized(searcherLock) {
// it's possible for someone to get a reference via the _searchers
queue
// and increment the refcount
while RefCounted.close() is being called.
// we check the refcount again to see if this has happened and
abort the close.
// This relies on the
RefCounted class allowing close() to be called every
// time the counter hits
zero.
resource.release();//释放FieldCache中的内存
if (refcount.get() >
0) return;
_searchers.remove(this);
}
resource.close();
}
catch (IOException e)
{
log.severe("Error closing searcher:"
+
SolrException.toStr(e));
}
}
};
holder.incref();
// set ref count to 1 to account for
this._searcher
return
holder;
}
  • org.apache.solr.util.RefCounted<Type>
修改方法org.apache.solr.util.RefCounted.decref()
public void
decref() {
// if (refcount.decrementAndGet() == 0)
{
close();
//
}
}

相关文章