一、碰到的问题
在服务器的文件系统上有一个业务生成的BigTable.json文件，其可能包含的JSON字符串很大，同时里边的集合会包含很多的记录；我们使用以下的代码来反序列化，虽然使用了异步的ReadAllTextAsync来读取文件，但是还是需要将整个的文件内容都读取到内存中，这样会极大的占用服务器内存，同时分配太多对象或分配非常大的对象会导致垃圾收集减慢甚至停止应用程序；
string jsonStr = File.ReadAllTextAsync("json.txt").Result;
BigTable table = JsonConvert.DeserializeObject<BigTable>(jsonStr);
二、JsonTextReader的简单用法
为了尽量减少内存使用和分配对象的数量，Json.NET提供了了解JsonTextReader，支持直接对流进行序列化和反序列化。每次读取或写入JSON片段，而不是将整个JSON字符串加载到内存中，这在处理大于85kb的JSON文档时尤其重要，以避免JSON字符串最终出现在大对象堆中.
string json = @"{
'CPU': 'Intel',
'PSU': '500W',
'Drives': [
'DVD read/writer'
/*(broken)*/,
'500 gigabyte hard drive',
'200 gigabyte hard drive'
]
}";
JsonTextReader reader = new JsonTextReader(new StringReader(json));
while (reader.Read())
{
if (reader.Value != null)
{
Console.WriteLine("Token: {0}, Value: {1}", reader.TokenType, reader.Value);
}
else
{
Console.WriteLine("Token: {0}", reader.TokenType);
}
}
Token: StartObject
Token: PropertyName, Value: CPU
Token: String, Value: Intel
Token: PropertyName, Value: PSU
Token: String, Value: 500W
Token: PropertyName, Value: Drives
Token: StartArray
Token: String, Value: DVD read/writer
Token: Comment, Value: (broken)
Token: String, Value: 500 gigabyte hard drive
Token: String, Value: 200 gigabyte hard drive
Token: EndArray
Token: EndObject
三、简单了解JsonTextReader的处理过程
JsonTextReader内部会通过State枚举记录其现在读取JSON字符串所在的位；
/// <summary>
/// Specifies the state of the reader.
/// </summary>
protected internal enum State
{
/// <summary>
/// A <see cref="JsonReader"/> read method has not been called.
/// </summary>
Start,
/// <summary>
/// The end of the file has been reached successfully.
/// </summary>
Complete,
/// <summary>
/// Reader is at a property.
/// </summary>
Property,
/// <summary>
/// Reader is at the start of an object.
/// </summary>
ObjectStart,
/// <summary>
/// Reader is in an object.
/// </summary>
Object,
/// <summary>
/// Reader is at the start of an array.
/// </summary>
ArrayStart,
/// <summary>
/// Reader is in an array.
/// </summary>
Array,
/// <summary>
/// The <see cref="JsonReader.Close()"/> method has been called.
/// </summary>
Closed,
/// <summary>
/// Reader has just read a value.
/// </summary>
PostValue,
/// <summary>
/// Reader is at the start of a constructor.
/// </summary>
ConstructorStart,
/// <summary>
/// Reader is in a constructor.
/// </summary>
Constructor,
/// <summary>
/// An error occurred that prevents the read operation from continuing.
/// </summary>
Error,
/// <summary>
/// The end of the file has been reached successfully.
/// </summary>
Finished
}
JsonTextReader在Read方法内部会根据当前_currentState来决定下一步处理的逻辑；在开始的时候_currentState=State.Start，所以会调用ParseValue方法；
public override bool Read()
{
EnsureBuffer();
MiscellaneousUtils.Assert(_chars != null);
while (true)
{
switch (_currentState)
{
case State.Start:
case State.Property:
case State.Array:
case State.ArrayStart:
case State.Constructor:
case State.ConstructorStart:
return ParseValue();
case State.Object:
case State.ObjectStart:
return ParseObject();
case State.PostValue:
// returns true if it hits
// end of object or array
if (ParsePostValue(false))
{
return true;
}
break;
case State.Finished:
if (EnsureChars(0, false))
{
EatWhitespace();
if (_isEndOfFile)
{
SetToken(JsonToken.None);
return false;
}
if (_chars[_charPos] == '/')
{
ParseComment(true);
return true;
}
throw JsonReaderException.Create(this, "Additional text encountered after finished reading JSON content: {0}.".FormatWith(CultureInfo.InvariantCulture, _chars[_charPos]));
}
SetToken(JsonToken.None);
return false;
default:
throw JsonReaderException.Create(this, "Unexpected state: {0}.".FormatWith(CultureInfo.InvariantCulture, CurrentState));
}
}
}
JsonTextReader的ParseValue方法会根据当前读取的字符决定下一步的处理逻辑；由于_chars数组默认初始化的时候第一个字符是\0,并且_charsUsed和_charPos都为0，所以会调用ReadData方法；
private bool ParseValue()
{
MiscellaneousUtils.Assert(_chars != null);
while (true)
{
char currentChar = _chars[_charPos];
switch (currentChar)
{
case '\0':
if (_charsUsed == _charPos)
{
if (ReadData(false) == 0)
{
return false;
}
}
else
{
_charPos++;
}
break;
case '"':
case '\'':
ParseString(currentChar, ReadType.Read);
return true;
case 't':
ParseTrue();
return true;
case 'f':
ParseFalse();
return true;
case 'n':
if (EnsureChars(1, true))
{
char next = _chars[_charPos + 1];
if (next == 'u')
{
ParseNull();
}
else if (next == 'e')
{
ParseConstructor();
}
else
{
throw CreateUnexpectedCharacterException(_chars[_charPos]);
}
}
else
{
_charPos++;
throw CreateUnexpectedEndException();
}
return true;
case 'N':
ParseNumberNaN(ReadType.Read);
return true;
case 'I':
ParseNumberPositiveInfinity(ReadType.Read);
return true;
case '-':
if (EnsureChars(1, true) && _chars[_charPos + 1] == 'I')
{
ParseNumberNegativeInfinity(ReadType.Read);
}
else
{
ParseNumber(ReadType.Read);
}
return true;
case '/':
ParseComment(true);
return true;
case 'u':
ParseUndefined();
return true;
case '{':
_charPos++;
SetToken(JsonToken.StartObject);
return true;
case '[':
_charPos++;
SetToken(JsonToken.StartArray);
return true;
case ']':
_charPos++;
SetToken(JsonToken.EndArray);
return true;
case ',':
// don't increment position, the next call to read will handle comma
// this is done to handle multiple empty comma values
SetToken(JsonToken.Undefined);
return true;
case ')':
_charPos++;
SetToken(JsonToken.EndConstructor);
return true;
case StringUtils.CarriageReturn:
ProcessCarriageReturn(false);
break;
case StringUtils.LineFeed:
ProcessLineFeed();
break;
case ' ':
case StringUtils.Tab:
// eat
_charPos++;
break;
default:
if (char.IsWhiteSpace(currentChar))
{
// eat
_charPos++;
break;
}
if (char.IsNumber(currentChar) || currentChar == '-' || currentChar == '.')
{
ParseNumber(ReadType.Read);
return true;
}
throw CreateUnexpectedCharacterException(currentChar);
}
}
}
在JsonTextReader内部会通过_chars来读取少量的字符；
private int ReadData(bool append)
{
return ReadData(append, 0);
}
private int ReadData(bool append, int charsRequired)
{
if (_isEndOfFile)
{
return 0;
}
PrepareBufferForReadData(append, charsRequired);
MiscellaneousUtils.Assert(_chars != null);
int attemptCharReadCount = _chars.Length - _charsUsed - 1;
int charsRead = _reader.Read(_chars, _charsUsed, attemptCharReadCount);
_charsUsed += charsRead;
if (charsRead == 0)
{
_isEndOfFile = true;
}
_chars[_charsUsed] = '\0';
return charsRead;
}
四、使用JsonTextReader优化反序列化
通过以上分析，我们可以直接使用二进制的文件流来读取文件，并将它传递给JsonTextReader，这样就可以实现小片段的读取并序列化；
using (FileStream s = File.Open("json.txt", FileMode.Open))
using (StreamReader sr = new StreamReader(s))
using (JsonReader reader = new JsonTextReader(sr))
{
JsonSerializer serializer = new JsonSerializer();
// read the json from a stream
// json size doesn't matter because only a small piece is read at a time
BigTable table = serializer.Deserialize<BigTable>(reader);
}
using (StreamReader sr = File.OpenText("json.txt"))
using (JsonReader reader = new JsonTextReader(sr))
{
JsonSerializer serializer = new JsonSerializer();
// read the json from a stream
// json size doesn't matter because only a small piece is read at a time
BigTable table = serializer.Deserialize<BigTable>(reader);
}