JSON continuation tests

[devsko]

See #42158
comment and comment and comment

Test only

Test several scenarios in System.Text.Json where deserialization has to continue after the next chunk of data is available.

[devsko]

@layomia This is not yet ready for review. I hope to get done what I have in mind tomorrow.

[devsko]

Is there anything else about continuation / chunked buffer?

[ahsonkhan]

Is there anything else about continuation / chunked buffer?

Since you asked, how about continuation where the token being split isn't null but other types, like whitespace (\r\n), or true/false boolean, or some large string token, or a number?

In the Utf8JsonReader specific tests, a lot of those scenarios are covered, by building ROSequence with segments splits across a whole variety of locations within the JSON, and with partial data/state to test re-entrancy. But most of those are for relatively small payloads (due to test time), and the async Deserializer API for streams could benefit from that type of extensive coverage too :)

runtime/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs

Lines 474 to 643 in e0e1919

	// TestCaseType is only used to give the json strings a descriptive name.
	[Theory]
	// Skipping large JSON since slicing them (O(n^2)) is too slow.
	[MemberData(nameof(SmallTestCases))]
	publicstaticvoidTestJsonReaderUtf8SegmentSizeOne(boolcompactData,TestCaseTypetype,stringjsonString)
	{
	ReadPartialSegmentSizeOne(compactData,type,jsonString);
	}
	// TestCaseType is only used to give the json strings a descriptive name.
	[Theory]
	[MemberData(nameof(LargeTestCases))]
	publicstaticvoidTestJsonReaderLargeUtf8SegmentSizeOne(boolcompactData,TestCaseTypetype,stringjsonString)
	{
	// Skipping really large JSON on Browser to prevent OOM
	if(PlatformDetection.IsBrowser&&(type==TestCaseType.Json40KB||type==TestCaseType.Json400KB||type==TestCaseType.ProjectLockJson))
	{
	return;
	}
	ReadFullySegmentSizeOne(compactData,type,jsonString);
	}
	// TestCaseType is only used to give the json strings a descriptive name.
	[Theory]
	[OuterLoop]
	[MemberData(nameof(LargeTestCases))]
	publicstaticvoidTestJsonReaderLargestUtf8SegmentSizeOne(boolcompactData,TestCaseTypetype,stringjsonString)
	{
	// Skipping really large JSON since slicing them (O(n^2)) is too slow.
	if(type==TestCaseType.Json40KB||type==TestCaseType.Json400KB||type==TestCaseType.ProjectLockJson)
	{
	return;
	}
	ReadPartialSegmentSizeOne(compactData,type,jsonString);
	}
	privatestaticvoidReadPartialSegmentSizeOne(boolcompactData,TestCaseTypetype,stringjsonString)
	{
	// Remove all formatting/indendation
	if(compactData)
	{
	jsonString=JsonTestHelper.GetCompactString(jsonString);
	}
	byte[]dataUtf8=Encoding.UTF8.GetBytes(jsonString);
	Streamstream=newMemoryStream(dataUtf8);
	TextReaderreader=newStreamReader(stream,Encoding.UTF8,false,1024,true);
	stringexpectedStr=JsonTestHelper.NewtonsoftReturnStringHelper(reader);
	ReadOnlySequence<byte>sequence=JsonTestHelper.GetSequence(dataUtf8,1);
	for(intj=0;j<dataUtf8.Length;j++)
	{
	varutf8JsonReader=newUtf8JsonReader(sequence.Slice(0,j),isFinalBlock:false,default);
	byte[]resultSequence=JsonTestHelper.ReaderLoop(dataUtf8.Length,outintlength,refutf8JsonReader);
	stringactualStrSequence=Encoding.UTF8.GetString(resultSequence,0,length);
	longconsumed=utf8JsonReader.BytesConsumed;
	utf8JsonReader=newUtf8JsonReader(sequence.Slice(consumed),isFinalBlock:true,utf8JsonReader.CurrentState);
	resultSequence=JsonTestHelper.ReaderLoop(dataUtf8.Length,outlength,refutf8JsonReader);
	actualStrSequence+=Encoding.UTF8.GetString(resultSequence,0,length);
	stringmessage=$"Expected consumed: {dataUtf8.Length-consumed}, Actual consumed: {utf8JsonReader.BytesConsumed}, Index: {j}";
	Assert.True(dataUtf8.Length-consumed==utf8JsonReader.BytesConsumed,message);
	Assert.Equal(expectedStr,actualStrSequence);
	}
	}
	privatestaticvoidReadFullySegmentSizeOne(boolcompactData,TestCaseTypetype,stringjsonString)
	{
	// Remove all formatting/indendation
	if(compactData)
	{
	jsonString=JsonTestHelper.GetCompactString(jsonString);
	}
	byte[]dataUtf8=Encoding.UTF8.GetBytes(jsonString);
	Streamstream=newMemoryStream(dataUtf8);
	TextReaderreader=newStreamReader(stream,Encoding.UTF8,false,1024,true);
	stringexpectedStr=JsonTestHelper.NewtonsoftReturnStringHelper(reader);
	ReadOnlySequence<byte>sequence=JsonTestHelper.GetSequence(dataUtf8,1);
	varutf8JsonReader=newUtf8JsonReader(sequence,isFinalBlock:true,default);
	byte[]resultSequence=JsonTestHelper.ReaderLoop(dataUtf8.Length,outintlength,refutf8JsonReader);
	stringactualStrSequence=Encoding.UTF8.GetString(resultSequence,0,length);
	Assert.Equal(expectedStr,actualStrSequence);
	}
	[Theory]
	[MemberData(nameof(SmallTestCases))]
	publicstaticvoidTestPartialJsonReaderMultiSegment(boolcompactData,TestCaseTypetype,stringjsonString)
	{
	_=type;
	// Remove all formatting/indendation
	if(compactData)
	{
	jsonString=JsonTestHelper.GetCompactString(jsonString);
	}
	byte[]dataUtf8=Encoding.UTF8.GetBytes(jsonString);
	ReadOnlyMemory<byte>dataMemory=dataUtf8;
	List<ReadOnlySequence<byte>>sequences=JsonTestHelper.GetSequences(dataMemory);
	for(inti=0;i<sequences.Count;i++)
	{
	ReadOnlySequence<byte>sequence=sequences[i];
	varjson=newUtf8JsonReader(sequence,isFinalBlock:true,default);
	while(json.Read())
	;
	Assert.Equal(sequence.Length,json.BytesConsumed);
	Assert.True(sequence.Slice(json.Position).IsEmpty);
	}
	for(inti=0;i<sequences.Count;i++)
	{
	ReadOnlySequence<byte>sequence=sequences[i];
	varjson=newUtf8JsonReader(sequence);
	while(json.Read())
	;
	Assert.Equal(sequence.Length,json.BytesConsumed);
	Assert.True(sequence.Slice(json.Position).IsEmpty);
	}
	}
	[Theory]
	[OuterLoop]
	[MemberData(nameof(SmallTestCases))]
	publicstaticvoidTestPartialJsonReaderSlicesMultiSegment(boolcompactData,TestCaseTypetype,stringjsonString)
	{
	_=type;
	// Remove all formatting/indendation
	if(compactData)
	{
	jsonString=JsonTestHelper.GetCompactString(jsonString);
	}
	byte[]dataUtf8=Encoding.UTF8.GetBytes(jsonString);
	ReadOnlyMemory<byte>dataMemory=dataUtf8;
	List<ReadOnlySequence<byte>>sequences=JsonTestHelper.GetSequences(dataMemory);
	for(inti=0;i<sequences.Count;i++)
	{
	ReadOnlySequence<byte>sequence=sequences[i];
	for(intj=0;j<dataUtf8.Length;j++)
	{
	varjson=newUtf8JsonReader(sequence.Slice(0,j),isFinalBlock:false,default);
	while(json.Read())
	;
	longconsumed=json.BytesConsumed;
	JsonReaderStatejsonState=json.CurrentState;
	byte[]consumedArray=sequence.Slice(0,consumed).ToArray();
	Assert.Equal(consumedArray,sequence.Slice(0,json.Position).ToArray());
	json=newUtf8JsonReader(sequence.Slice(consumed),isFinalBlock:true,jsonState);
	while(json.Read())
	;
	Assert.Equal(dataUtf8.Length-consumed,json.BytesConsumed);
	}
	}
	}

[devsko]

how about continuation where the token being split isn't null but other types, like whitespace (\r\n), or true/false boolean, or some large string token, or a number?

All tests here split the payload once on every single character. Thus all tokens are tested how they work when split into 2 chunks including all mentioned examples except whitespaces. I will add them by enabling WriteIndented

[devsko]

I'd say that's it. Thanks for your suggestions, help and reviews. Really appreciated. Feel free to change whatever you want - or wait 2 weeks. See you - peace

[stephentoub]

@devsko, thanks for your efforts here. Are you still working on this?

[layomia]

LGTM - @devsko we can merge this once conflicts and #42393 (comment) are resolved.

[layomia]

I pushed a commit to finish this PR.