The reason you are getting the stackoverflow exception is that Json.NET is a recursive, single-pass tree or graph serializer that, when PreserveReferencesHandling.Objects is enabled, always serializes the first occurrence of each object. You have constructed your 15,000 element Chacha [] array so that the first entry is the head of a linked list containing all the other items linked sequentially. Json.NET will try to serialize that to nested JSON objects 15,000 levels deep via 15,000 levels of recursion, overflowing the stack in the process.
Thus what you need to do is write the entire table of linkages only at the head of the list, as a JSON array. Unfortunately, however, Json.NET is also a contract-based serializer which means it will try to write the same properties whenever it encounters an object of a given type, no matter what the nesting depth is. Thus adding a Chacha[] NextChachaList property to your Chacha object doesn't help since it will get written at each level. Instead it will be necessary to create a fairly complex custom JsonConverter that tracks the serialization depth in a thread-safe manner and only writes the linkage list only at the top level. The following does the trick:
class ChachaConverter : LinkedListItemConverter<Chacha>
{
protected override bool IsNextItemProperty(JsonProperty member)
{
return member.UnderlyingName == "NextChacha"; // Use nameof(Chacha.NextChacha) in latest c#
}
}
public abstract class LinkedListItemConverter<T> : JsonConverter where T : class
{
const string refProperty = "$ref";
const string idProperty = "$id";
const string NextItemListProperty = "nextItemList";
[ThreadStatic]
static int level;
// Increments the nesting level in a thread-safe manner.
int Level { get { return level; } set { level = value; } }
public override bool CanConvert(Type objectType)
{
return typeof(T).IsAssignableFrom(objectType);
}
protected abstract bool IsNextItemProperty(JsonProperty member);
List<T> GetNextItemList(object value, JsonObjectContract contract)
{
var property = contract.Properties.Where(p => IsNextItemProperty(p)).Single();
List<T> list = null;
for (var item = (T)property.ValueProvider.GetValue(value); item != null; item = (T)property.ValueProvider.GetValue(item))
{
if (list == null)
list = new List<T>();
list.Add(item);
}
return list;
}
void SetNextItemLinks(object value, List<T> list, JsonObjectContract contract)
{
var property = contract.Properties.Where(p => IsNextItemProperty(p)).Single();
if (list == null || list.Count == 0)
return;
var previous = value;
foreach (var next in list)
{
if (next == null)
continue;
property.ValueProvider.SetValue(previous, next);
previous = next;
}
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
using (new PushValue<int>(Level + 1, () => Level, (old) => Level = old))
{
writer.WriteStartObject();
if (serializer.ReferenceResolver.IsReferenced(serializer, value))
{
writer.WritePropertyName(refProperty);
writer.WriteValue(serializer.ReferenceResolver.GetReference(serializer, value));
}
else
{
writer.WritePropertyName(idProperty);
writer.WriteValue(serializer.ReferenceResolver.GetReference(serializer, value));
var contract = (JsonObjectContract)serializer.ContractResolver.ResolveContract(value.GetType());
// Write the data properties (if any).
foreach (var property in contract.Properties
.Where(p => p.Readable && !p.Ignored && (p.ShouldSerialize == null || p.ShouldSerialize(value))))
{
if (IsNextItemProperty(property))
continue;
var propertyValue = property.ValueProvider.GetValue(value);
if (propertyValue == null && serializer.NullValueHandling == NullValueHandling.Ignore)
continue;
writer.WritePropertyName(property.PropertyName);
serializer.Serialize(writer, propertyValue);
}
if (Level == 1)
{
// Write the NextItemList ONLY AT THE TOP LEVEL
var nextItems = GetNextItemList(value, contract);
if (nextItems != null)
{
writer.WritePropertyName(NextItemListProperty);
serializer.Serialize(writer, nextItems);
}
}
}
writer.WriteEndObject();
}
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null)
return null;
var jObject = JObject.Load(reader);
// Detach and process $ref
var refValue = (string)jObject[refProperty].RemoveFromLowestPossibleParent();
if (refValue != null)
{
var reference = serializer.ReferenceResolver.ResolveReference(serializer, refValue);
if (reference != null)
return reference;
}
// Construct the value
var contract = (JsonObjectContract)serializer.ContractResolver.ResolveContract(existingValue == null ? typeof(T) : existingValue.GetType());
T value = (existingValue as T ?? (T)contract.DefaultCreator());
// Detach and process $id
var idValue = (string)jObject[idProperty].RemoveFromLowestPossibleParent();
if (idValue != null)
{
serializer.ReferenceResolver.AddReference(serializer, idValue, value);
}
// Detach the (possibly large) list of next items.
var nextItemList = jObject[NextItemListProperty].RemoveFromLowestPossibleParent();
// populate the data properties (if any)
serializer.Populate(jObject.CreateReader(), value);
// Set the next item references
if (nextItemList != null)
{
var list = nextItemList.ToObject<List<T>>(serializer);
SetNextItemLinks(value, list, contract);
}
return value;
}
}
public struct PushValue<T> : IDisposable
{
Action<T> setValue;
T oldValue;
public PushValue(T value, Func<T> getValue, Action<T> setValue)
{
if (getValue == null || setValue == null)
throw new ArgumentNullException();
this.setValue = setValue;
this.oldValue = getValue();
setValue(value);
}
#region IDisposable Members
// By using a disposable struct we avoid the overhead of allocating and freeing an instance of a finalizable class.
public void Dispose()
{
if (setValue != null)
setValue(oldValue);
}
#endregion
}
public static class JsonExtensions
{
public static JToken RemoveFromLowestPossibleParent(this JToken node)
{
if (node == null)
return null;
var contained = node.AncestorsAndSelf().Where(t => t.Parent is JContainer && t.Parent.Type != JTokenType.Property).FirstOrDefault();
if (contained != null)
contained.Remove();
// Also detach the node from its immediate containing property -- Remove() does not do this even though it seems like it should
if (node.Parent is JProperty)
((JProperty)node.Parent).Value = null;
return node;
}
}
Then, given the slightly modified class Chacha:
class Chacha
{
public Chacha NextChacha { get; set; }
public long Data { get; set; }
}
The following JSON is generated for an array of 3 items:
{
"$type": "Question41828014.Chacha[], Tile",
"$values": [
{
"$id": "1",
"Data": 0,
"nextItemList": {
"$type": "System.Collections.Generic.List`1[[Question41828014.Chacha, Tile]], mscorlib",
"$values": [
{
"$id": "2",
"Data": 1
},
{
"$id": "3",
"Data": 2
}
]
}
},
{
"$ref": "2"
},
{
"$ref": "3"
}
]
}
Notice that the JSON depth is now strictly limited. Example fiddle.
Be aware that, once you specify a custom converter for your type, it needs to do everything manually. If your type Chacha is polymorphic and you need to read and write "$type" properties, you will need to add that logic to the converter yourself.
By the way, I recommend TypeNameHandling.Objects instead of TypeNameHandling.All. Object types may reasonably be specified in the JSON (as long as the types are properly sanitized) but collection types should be specified in the code. Doing so makes it possible to switch from an array to a List<T> without having to postread legacy JSON files.
