前言
昨天写了个 《基于STSdb和fastJson的磁盘/内存缓存》，大家可以先看看。下午用到业务系统时候，觉得可以改进一下，昨晚想了一个晚上，刚才重新实现一下。
更新
1. 增加了对批量处理的支持，写操作速度提升5倍，读操作提升100倍
2. 增加了一个存储provider，可以选择不用STSdb做存储，而用物理文件/Dictionary。
3. 增加了空间回收
4. 增加了对并发的支持
需求
业务系统用的是数据库，数据量大，部分只读或相对稳定业务查询复杂，每次页面加载都要花耗不少时间(不讨论异步)，觉得可以做一下高速缓存，譬如用nosql那种key/value快速存取结果
目的
提供一个简单易用的解决缓存方案，可以根据数据的大小缓存到内存或者磁盘。
实现
存取
方法1. 基于STSdb，提供高效的Key/Value存取，支持磁盘/内存，对Key无限制
方法2. 基于直接物理文件/Dictionary。Key必须是基本类型，譬如int/long/uint/ulong/DateTime/string等。
代码
代码比较简单，花了2个小时写的，很多情况没考虑，譬如磁盘空间/内存不足，自动回收过期缓存等，这些留给大家做家庭作业吧。另外，为了发布方便，STSdb和fastJson的代码都合并到一个项目里。
BaseCahce.cs
这是一个抽象基类，提供存取接口。
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text; namespace Com.SuperCache.Engine
{
public abstract class BaseCache
{
protected internal const string KeyExpiration = "Expiration"; public abstract void Add<K>(string Category, K Key, object Data);
public abstract void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate);
public abstract void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate);
public abstract List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys);
public abstract V Get<K, V>(string Category, K Key);
public abstract void Recycle<K>( string Category, long Count);
}
}
CahceEngine.cs
主要调用缓存引擎
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using STSdb4.Database;
using fastJSON;
using System.IO; namespace Com.SuperCache.Engine
{
public enum CacheProviders
{
Default = 1,
Raw = 2
} public enum RecycleAlgorithms
{
None = 0,
//LRU = 1,
MRU = 2
} public enum RecycleModes
{
None = 0,
Passive = 1,
Active = 2
} public class CacheEngine
{
private BaseCache cacheProvider = null; public CacheEngine(string DataPath)
: this(CacheProviders.Default, DataPath, RecycleAlgorithms.None, 0, 0, RecycleModes.None)
{
} public CacheEngine(CacheProviders Provider, string DataPath, RecycleAlgorithms RecycleAlgorithm, int MaxCount, int Threshold, RecycleModes RecycleMode)
{
switch (Provider)
{
case CacheProviders.Default:
cacheProvider = new STSdbCache(DataPath, RecycleAlgorithm, MaxCount, Threshold, RecycleMode);
break;
case CacheProviders.Raw:
cacheProvider = new RawCache(DataPath, RecycleAlgorithm, MaxCount, Threshold, RecycleMode);
break;
default:
break;
}
} public void Add<K>(string Category, K Key, object Data)
{
cacheProvider.Add<K>(Category, Key, Data);
} public void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate)
{
cacheProvider.Add<K, V>(Category, Items, ExpirationDate);
} public void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate)
{
cacheProvider.Add<K>(Category, Key, Data, ExpirationDate);
} public List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys)
{
return cacheProvider.Get<K, V>(Category, Keys);
} public V Get<K, V>(string Category, K Key)
{
return cacheProvider.Get<K, V>(Category, Key);
} }
}
STSdbCache.cs
STSdb存储引擎
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using STSdb4.Database;
using fastJSON;
using System.IO; namespace Com.SuperCache.Engine
{
public class STSdbCache : BaseCache
{
private const string UsageStat = "SuperCacheUsageStat";
private string dataPath;
private static IStorageEngine memoryInstance = null;
private static object syncRoot = new object();
private bool isMemory = false;
private RecycleAlgorithms recycleAlgorithm;
private int maxCount;
private int threshold;
private RecycleModes recycleMode; public STSdbCache(string DataPath, RecycleAlgorithms RecycleAlgorithm, int MaxCount, int Threshold, RecycleModes RecycleMode)
{
dataPath = DataPath;
if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString()))
dataPath += Path.DirectorySeparatorChar; isMemory = string.IsNullOrEmpty(DataPath); recycleAlgorithm = RecycleAlgorithm;
maxCount = MaxCount;
threshold = Threshold;
recycleMode = RecycleMode;
} public override void Add<K>(string Category, K Key, object Data)
{
Add(Category, Key, Data, null);
} private IStorageEngine Engine
{
get
{
if (isMemory)
{
lock (syncRoot)
{
if (memoryInstance == null)
memoryInstance = STSdb.FromMemory();
}
return memoryInstance;
}
else
return STSdb.FromFile(GetFile(false), GetFile(true));
}
} private string GetExpirationTable(string Category)
{
return KeyExpiration + "_" + Category;
} public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate)
{
long count = 0; lock (syncRoot)
{
var engine = Engine;
var table = engine.OpenXIndex<K, string>(Category);
var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category)); //track recycle
IIndex<K, int> usage = null;
if (recycleAlgorithm != RecycleAlgorithms.None)
usage = engine.OpenXIndex<K, int>(UsageStat); Items.ForEach(i =>
{
var key = i.Key;
var data = i.Value;
//will only serialize object other than string
var result = typeof(V) == typeof(string) ? data as string : JSON.Instance.ToJSON(data);
table[key] = result;
table.Flush(); //specify expiration
//default 30 mins to expire from now
var expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate;
expiration[key] = expirationDate;
expiration.Flush(); //track recycle
if (usage != null)
{
usage[key] = 0;
if (recycleMode == RecycleModes.Active)
Recycle<K>(Category, usage.Count());
}
}); if (usage != null)
count = usage.Count(); engine.Commit(); //only dispose disk-based engine
if (!isMemory)
engine.Dispose();
} if (recycleMode == RecycleModes.Passive)
Recycle<K>(Category, count);
} public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate)
{
Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate);
} private string GetFile(bool IsData)
{
if (!Directory.Exists(dataPath))
Directory.CreateDirectory(dataPath);
return dataPath + "SuperCache." + (IsData ? "dat" : "sys");
} public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys)
{
var result = new List<KeyValuePair<K, V>>();
lock (syncRoot)
{
var engine = Engine;
var table = engine.OpenXIndex<K, string>(Category);
var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category));
var isCommitRequired = false; //track recycle
IIndex<K, int> usage = null;
if (recycleAlgorithm != RecycleAlgorithms.None)
usage = engine.OpenXIndex<K, int>(UsageStat); Keys.ForEach(key =>
{
string buffer;
V value;
if (table.TryGet(key, out buffer))
{
//will only deserialize object other than string
value = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer);
bool needUpdate = true;
DateTime expirationDate;
//get expiration date
if (expiration.TryGet(key, out expirationDate))
{
//expired
if (expirationDate < DateTime.Now)
{
value = default(V);
table.Delete(key);
table.Flush();
expiration.Delete(key);
expiration.Flush();
isCommitRequired = true;
needUpdate = false;
}
} //track recycle
if (usage != null && needUpdate)
{
usage[key]++;
isCommitRequired = true;
}
}
else
value = default(V); result.Add(new KeyValuePair<K, V>(key, value));
}); //only need to commit write actions
if (isCommitRequired)
engine.Commit(); //only dispose disk-based engine
if (!isMemory)
engine.Dispose();
}
return result;
} public override V Get<K, V>(string Category, K Key)
{
var buffer = Get<K, V>(Category, new K[] { Key });
var result = buffer.FirstOrDefault();
return result.Value;
} public override void Recycle<K>(string Category, long Count)
{
if (Count < maxCount)
return; switch (recycleAlgorithm)
{
case RecycleAlgorithms.MRU:
lock (syncRoot)
{
var engine = Engine;
var table = engine.OpenXIndex<K, string>(Category);
var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category));
var usage = engine.OpenXIndex<K, int>(UsageStat);
//find out expired items
var expired = expiration.Where(e => e.Value < DateTime.Now);
expired.ForEach(e =>
{
table.Delete(e.Key);
expiration.Delete(e.Key);
usage.Delete(e.Key);
}); //find out least used items
var leastUsed = usage.OrderByDescending(s => s.Value).Skip(maxCount - threshold);
leastUsed.ForEach(u =>
{
table.Delete(u.Key);
expiration.Delete(u.Key);
usage.Delete(u.Key);
}); table.Flush();
expiration.Flush();
usage.Flush();
engine.Commit(); if (!isMemory)
engine.Dispose();
}
break;
default:
break;
}
}
}
}
RawCache.cs
物理文件/Dictionary引擎
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using STSdb4.Database;
using fastJSON;
using System.IO; namespace Com.SuperCache.Engine
{
public class RawCache : BaseCache
{
private const string ExpirationFileExtension = "exp";
private const string DataFileExtension = "dat";
private const string statFile = "SuperCache.sta";
private string dataPath;
private static Dictionary<string, object> memoryData = new Dictionary<string, object>();
private static Dictionary<string, DateTime?> memoryExpiration = new Dictionary<string, DateTime?>();
private static object syncRoot = new object();
private bool isMemory = false;
private RecycleAlgorithms recycleAlgorithm;
private int maxCount;
private int threshold;
private static Dictionary<string, KeyValue> usageStat = new Dictionary<string, KeyValue>();
private Dictionary<string, KeyValuePair<DateTime, string>> expiredFiles = new Dictionary<string, KeyValuePair<DateTime, string>>();
private RecycleModes recycleMode; public RawCache(string DataPath, RecycleAlgorithms RecycleAlgorithm, int MaxCount, int Threshold, RecycleModes RecycleMode)
{
dataPath = DataPath;
if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString()))
dataPath += Path.DirectorySeparatorChar; isMemory = string.IsNullOrEmpty(DataPath); recycleAlgorithm = RecycleAlgorithm;
maxCount = MaxCount;
threshold = Threshold;
recycleMode = RecycleMode;
} public override void Add<K>(string Category, K Key, object Data)
{
Add(Category, Key, Data, null);
} private string GetExpirationTable(string Category)
{
return KeyExpiration + "_" + Category;
} public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate)
{
long count = 0; lock (syncRoot)
{
Items.ForEach(i =>
{
var key = i.Key;
var data = i.Value;
var cacheKey = GetKey(Category, key.ToString()); if (isMemory)
{
memoryData[cacheKey] = data;
memoryExpiration[cacheKey] = ExpirationDate; //recycle algo
switch (recycleAlgorithm)
{
case RecycleAlgorithms.MRU:
usageStat[cacheKey] = new KeyValue(string.Empty, 0);
if (recycleMode == RecycleModes.Active)
Recycle<K>(Category, memoryData.Count);
break;
default:
break;
}
}
else
{
//will only serialize object other than string
var result = typeof(V) == typeof(string) ? data as string : JSON.Instance.ToJSON(data);
var fileKey = key.ToString();
var dataFile = GetFile(Category, fileKey, true);
bool exists = File.Exists(dataFile);
File.WriteAllText(dataFile, result); //specify expiration
//default 30 mins to expire from now
var expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate;
var expirationFile = GetFile(Category, fileKey, false);
File.WriteAllText(expirationFile, expirationDate.ToString()); //recycle algo
if (recycleAlgorithm != RecycleAlgorithms.None)
{
var statFilePath = dataPath + statFile;
if (File.Exists(statFilePath))
{
var buffer = File.ReadAllText(statFilePath);
count = Convert.ToInt32(buffer);
}
if (!exists)
{
count++;
File.WriteAllText(statFilePath, count.ToString());
}
switch (recycleAlgorithm)
{
case RecycleAlgorithms.MRU:
usageStat[cacheKey] = new KeyValue(expirationFile, 0);
expiredFiles[cacheKey] = new KeyValuePair<DateTime, string>(expirationDate, expirationFile);
if (recycleMode == RecycleModes.Active)
Recycle<K>(Category, count);
break;
default:
break;
}
}
}
}); if (recycleAlgorithm != RecycleAlgorithms.None && recycleMode == RecycleModes.Passive)
{
if (isMemory)
count = memoryData.Count;
Recycle<K>(Category, count);
}
}
} public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate)
{
Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate);
} private string GetFile(string Category, string FileName, bool IsData)
{
var path = dataPath + Category.NormalizeFileName() + @"\";
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
return path + FileName.NormalizeFileName() + "." + (IsData ? "dat" : ExpirationFileExtension);
} private string GetKey(string Category, string Key)
{
return Category + "_" + Key;
} public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys)
{
var result = new List<KeyValuePair<K, V>>();
lock (syncRoot)
{
Keys.ForEach(key =>
{
string buffer;
V value;
var cacheKey = GetKey(Category, key.ToString());
if (isMemory)
{
object memBuffer;
if (memoryData.TryGetValue(cacheKey, out memBuffer))
{
//track recycle
switch (recycleAlgorithm)
{
case RecycleAlgorithms.MRU:
usageStat[cacheKey].Value++;
break;
default:
break;
} value = (V)memBuffer;
DateTime? expirationDate;
if (memoryExpiration.TryGetValue(cacheKey, out expirationDate))
{
//expired
if (expirationDate != null && (DateTime)expirationDate < DateTime.Now)
{
value = default(V);
memoryData.Remove(cacheKey);
memoryExpiration.Remove(cacheKey);
}
}
}
else
value = default(V);
}
else
{
var dataFilePath = GetFile(Category, key.ToString(), true);
if (File.Exists(dataFilePath))
{
buffer = File.ReadAllText(dataFilePath); //track recycle
switch (recycleAlgorithm)
{
case RecycleAlgorithms.MRU:
usageStat[cacheKey].Value++;
break;
default:
break;
} //will only deserialize object other than string
value = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer);
DateTime expirationDate;
var expirationFilePath = GetFile(Category, key.ToString(), false);
if (File.Exists(expirationFilePath))
{
buffer = File.ReadAllText(expirationFilePath);
expirationDate = Convert.ToDateTime(buffer);
//expired
if (expirationDate < DateTime.Now)
{
value = default(V);
File.Delete(dataFilePath);
File.Delete(expirationFilePath);
}
}
}
else
value = default(V);
} result.Add(new KeyValuePair<K, V>(key, value));
});
}
return result;
} public override V Get<K, V>(string Category, K Key)
{
var buffer = Get<K, V>(Category, new K[] { Key });
var result = buffer.FirstOrDefault();
return result.Value;
} public override void Recycle<K>(string Category, long Count)
{
if (Count < maxCount)
return; switch (recycleAlgorithm)
{
case RecycleAlgorithms.MRU:
lock (syncRoot)
{
var recycledFileCount = 0; if (isMemory)
{
//find out expired items
var memExpired = memoryExpiration.Where(e => e.Value != null && (DateTime)e.Value < DateTime.Now);
memExpired.ForEach(u =>
{
memoryData.Remove(u.Key);
memoryExpiration.Remove(u.Key);
usageStat.Remove(u.Key);
});
}
else
{
if (expiredFiles.Count == 0)
{
Directory.GetFiles(dataPath, "*." + ExpirationFileExtension).ForEach(f =>
{
var buffer = File.ReadAllText(f);
var expirationDate = Convert.ToDateTime(buffer);
expiredFiles[Path.GetFileNameWithoutExtension(f)] = new KeyValuePair<DateTime, string>(expirationDate, f);
});
}
//find out expired items
var fileExpired = expiredFiles.Where(e => e.Value.Key < DateTime.Now);
fileExpired.ForEach(u =>
{
var dataFile = Path.ChangeExtension(u.Value.Value, DataFileExtension);
File.Delete(dataFile);
File.Delete(u.Value.Value);
usageStat.Remove(u.Key);
recycledFileCount++;
});
} //find out least used items
var leastUsed = usageStat.OrderByDescending(s => s.Value.Value).Skip(maxCount - threshold);
leastUsed.ForEach(u =>
{
if (isMemory)
{
memoryData.Remove(u.Key);
memoryExpiration.Remove(u.Key);
}
else
{
var dataFile = Path.ChangeExtension(u.Value.Key, DataFileExtension);
if (File.Exists(dataFile))
{
recycledFileCount++;
File.Delete(dataFile);
}
if (File.Exists(u.Value.Key))
File.Delete(u.Value.Key);
}
usageStat.Remove(u.Key);
}); if (!isMemory)
{
var statFilePath = dataPath + statFile;
var count = 0;
if (File.Exists(statFilePath))
{
var buffer = File.ReadAllText(statFilePath);
count = Convert.ToInt32(buffer);
}
count = count - recycledFileCount;
if (count < 0)
count = 0;
File.WriteAllText(statFilePath, count.ToString());
}
}
break;
default:
break;
}
}
}
}
Extensions.cs
扩展函数
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO; namespace Com.SuperCache.Engine
{
public static class Extensions
{
public static void ForEach<T>(this IEnumerable<T> source, Action<T> action)
{
if (source != null)
{
foreach (var item in source)
{
action(item);
}
}
} public static string NormalizeFileName(this string FileName)
{
var result = FileName;
Path.GetInvalidFileNameChars().ForEach(c =>
{
result = result.Replace(c.ToString(), string.Empty);
});
return result;
}
}
}
新建
构造CacheEngine需要传递缓存保存到哪个文件夹。
基于内存
如果你不喜欢基于磁盘的缓存，可以使用基于内存，构造函数传递空字符串便可。
增加/更新
同一个方法：Add。用户可以指定类型（Category），譬如User，Employee等。键（Key）支持泛型，值（Data）是object。有一个overload是过期日期（ExpirationDate），默认当前时间30分钟后
获取
Get方法需要指定类型（Category）和键（Key）。
例子
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Diagnostics;
using Com.SuperCache.Engine; namespace Com.SuperCache.Test
{
public class Foo
{
public string Name { get; set; }
public int Age { get; set; }
public double? Some { get; set; }
public DateTime? Birthday { get; set; }
} class Program
{
static void Main(string[] args)
{
//TestAddGet(); //Thread.Sleep(4000);
//TestExpiration(); TestDefaultDiskPerformance();
TestDefaultMemoryPerformance();
TestRawDiskPerformance();
TestRawMemoryPerformance(); //TestConcurrent(); Console.Read();
} private static void TestConcurrent()
{
var w = new Stopwatch();
w.Start();
Parallel.For(1, 3, (a) =>
{
var employees = Enumerable.Range((a - 1) * 1000, a * 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen"));
var engine = new CacheEngine(@"..\..\data");
engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1));
});
w.Stop();
Console.WriteLine("add:" + w.Elapsed); var engine2 = new CacheEngine(@"..\..\data");
var o = engine2.Get<int, string>("Employee", 1005);
Console.WriteLine(o);
} private static void TestDefaultDiskPerformance()
{
TestPerformance(CacheProviders.Default, @"..\..\data");
} private static void TestDefaultMemoryPerformance()
{
TestPerformance(CacheProviders.Default, string.Empty);
} private static void TestRawDiskPerformance()
{
TestPerformance(CacheProviders.Raw, @"..\..\data");
} private static void TestRawMemoryPerformance()
{
TestPerformance(CacheProviders.Raw, string.Empty);
} private static void TestPerformance(CacheProviders Provider, string DataPath)
{
Console.WriteLine("Performance Test: " + Provider.ToString() + ", " + (string.IsNullOrEmpty(DataPath) ? "Memory" : DataPath));
var engine = new CacheEngine(Provider, DataPath, RecycleAlgorithms.MRU, 900, 100, RecycleModes.Passive);
var w = new Stopwatch(); w.Start();
var employees = Enumerable.Range(0, 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen"));
engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1));
w.Stop();
Console.WriteLine("add:" + w.Elapsed); /*w.Restart();
employees.ForEach(key =>
{
var o1 = engine.Get<int, string>("Employee", key.Key);
});
w.Stop();
Console.WriteLine("individual get:" + w.Elapsed);*/ w.Restart();
var keys = employees.Select(i => i.Key);
var o = engine.Get<int, string>("Employee", keys);
w.Stop();
Console.WriteLine("get:" + w.Elapsed);
Console.WriteLine();
} private static void TestExpiration()
{
var engine = new CacheEngine(@"..\..\data");
var o = engine.Get<string, Foo>("User", "wchen");
Console.WriteLine(o != null ? o.Name : "wchen does not exist or expired");
} private static void TestAddGet()
{
var engine = new CacheEngine(@"..\..\data");
var f = new Foo { Name = "Wilson Chen", Age = 30, Birthday = DateTime.Now, Some = 123.456 };
engine.Add("User", "wchen", f, DateTime.Now.AddSeconds(5)); var o = engine.Get<string, Foo>("User", "wchen");
Console.WriteLine(o.Name); var o4 = engine.Get<string, Foo>("User", "foo");
Console.WriteLine(o4 != null ? o4.Name : "foo does not exist"); var o3 = engine.Get<string, string>("PlainText", "A");
Console.WriteLine(o3 ?? "A does not exist");
}
}
}
性能
通过上述性能测试例子，你会发现STSdb的磁盘存取速度要比一个记录对应一个物理文件快。想了解更多，请访问官方网站。
测试条件：1000条记录，7200RPM磁盘，i7。
| 引擎 | 介质 | 写入 | 读取 |
| STSdb | 磁盘 | 0.4s | 0.07s |
| 内存 | 0.06s | 0.02s | |
| Raw | 磁盘 | 1.0s | 0.56s |
| 内存 | 0.01s | 0.002s |
说明
项目中引用了System.Management是因为STSdb支持内存数据库，需要判断最大物理内存。如果不喜欢，大家可以移除引用，并且去掉STSdb4.Database.STSdb.FromMemory方法便可。
下载
点击这里下载