Deserializing array of [price, quantity]

Question

Here is the endpoint I'm trying to deserialize. The issue is within the bids, asks and changes. How do I handle it? The API says it's array of [price, quantity].

Snippet

[JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
public record Depth(
    [property: JsonPropertyName("type")] DepthType Type,
    [property: JsonPropertyName("pair")] string Pair,
    [property: JsonPropertyName("sequence")] long Sequence,
    [property: JsonPropertyName("asks")] List<List<string>> Asks,
    //[property: JsonPropertyName("bids")] IEnumerable<Level> Bids,
    [property: JsonPropertyName("prev_sequence")] long PreviousSequence
    //[property: JsonPropertyName("changes")] IEnumerable<Changes> Changes
);

[JsonConverter(typeof(JsonStringEnumMemberConverter))]
public enum DepthType
{
    [EnumMember(Value = "snapshot")]
    Snapshot,

    [EnumMember(Value = "update")]
    Update
}

public record Level(decimal Price, decimal Quantity);

public record Changes(string Side, decimal Price, decimal Quantity);

Response

{
    "channel":"depth",
    "timestamp":1587929552250,
    "module":"spot",
    "data":{
        "type":"snapshot",
        "pair":"BTC-USDT",
        "sequence":9,
        "bids":[
            [
                "0.08000000",
                "0.10000000"
            ]
        ],
        "asks":[
            [
                "0.09000000",
                "0.20000000"
            ]
        ]
    }
}

{
    "channel":"depth",
    "timestamp":1587930311331,
    "module":"spot",
    "data":{
        "type":"update",
        "pair":"BTC-USDT",
        "sequence":10,
        "prev_sequence":9,
        "changes":[
            [
                "sell",
                "0.08500000",
                "0.10000000"
            ]
        ]
    }
}

Attempt

[AttributeUsage(AttributeTargets.Property)]
public sealed class JsonPlainArrayIndexAttribute : Attribute
{
    public JsonPlainArrayIndexAttribute(int index)
    {
        Index = index;
    }

    public int Index { get; }
}

public sealed class JsonPlainArrayConverter<T> : JsonConverter<T> where T : new()
{
    public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
    {
        Debug.Assert(typeof(T) == typeToConvert);

        var props = typeToConvert.GetProperties();
        var linq = from prop in props
            let attr = prop.GetCustomAttributes(typeof(JsonPlainArrayIndexAttribute), true)
            where prop.CanWrite && attr.Length is 1
            orderby ((JsonPlainArrayIndexAttribute)attr[0]).Index
            select prop;

        var arr = JsonSerializer.Deserialize<IEnumerable<JsonElement>>(ref reader, options);
        if (arr is null)
        {
            return default;
        }

        var result = new T();
        foreach (var (prop, value) in linq.Zip(arr))
        {
            prop.SetValue(result, value.Deserialize(prop.PropertyType, options));
        }

        return result;
    }

    public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options)
    {
        var type = typeof(T);
        var props = type.GetProperties();
        var linq = from prop in props
            let attr = prop.GetCustomAttributes(typeof(JsonPlainArrayIndexAttribute), true)
            where prop.CanRead && attr.Length is 1
            orderby ((JsonPlainArrayIndexAttribute)attr[0]).Index
            select prop.GetValue(value);
        JsonSerializer.Serialize<IEnumerable<object>>(writer, linq, options);
    }
}

[JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
public record Depth(
    [property: JsonPropertyName("type")] DepthType Type,
    [property: JsonPropertyName("pair")] string Pair,
    [property: JsonPropertyName("sequence")] long Sequence,
    [property: JsonPropertyName("asks")] BookLevel Asks,
    [property: JsonPropertyName("bids")] BookLevel Bids,
    [property: JsonPropertyName("prev_sequence")] long PreviousSequence
    //[property: JsonPropertyName("changes")] IEnumerable<Changes> Changes
);

[JsonConverter(typeof(JsonStringEnumMemberConverter))]
public enum DepthType
{
    [EnumMember(Value = "snapshot")]
    Snapshot,

    [EnumMember(Value = "update")]
    Update
}

[JsonConverter(typeof(JsonPlainArrayConverter<BookLevel>))]
[JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
public record BookLevel
{
    [JsonPlainArrayIndex(0)]
    public string Price { get; init; }

    [JsonPlainArrayIndex(1)]
    public string Quantity { get; init; }
}

public record Changes(string Side, decimal Price, decimal Quantity);

You will need to use a custom `JsonConverter` and `JsonConverter` to map arrays of values onto the `Level` and `Changes` data models. To do so see [this answer](https://stackoverflow.com/a/71649515) by [yueyinqiu](https://stackoverflow.com/users/15283686/yueyinqiu) to [System.Text.Json fails to deserialize List>](https://stackoverflow.com/q/71649101). In fact this looks to be a duplicate, agree? — dbc, May 16 '22 at 13:50
@dbc, there is an issue. Trying to do `new BookLevel(arr[0].GetDecimal(), arr[1].GetDecimal())`, but it is a `string`. I usually solve this by adding `[JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]`, but how do I do that with that class? — nop, May 16 '22 at 14:01
Well if the `JsonElement` value inside `JsonConverter.Read()` might be number or a string you can use one of the methods from [System.Text.Json.JsonElement ToObject workaround](https://stackoverflow.com/q/58138793/3744182) to deserialize it to a `decimal`. Be sure to pass in `JsonNumberHandling.AllowReadingFromString`. (In .NET 6 you don't even need a workaround, you can use `JsonSerializer.Deserialize(this JsonElement element, JsonSerializerOptions? options = null);`) — dbc, May 16 '22 at 14:04
@dbc, it doesn't work in this case because of the string to decimal. `The JSON value could not be converted to System.String. Path: $ | LineNumber: 0 | BytePositionInLine`. I added the code I tested with to the question — nop, May 16 '22 at 14:24
The converter works as-is. You have the following problems: `Asks` and `Bids` must be declared as collections, e.g. `List`. 2) You need to set `JsonSerializerOptions.NumberHandling = JsonNumberHandling.AllowReadingFromString` when deserializing, and ... — dbc, May 16 '22 at 16:34
... 3) You set `[JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]` and [JsonConverter(typeof(JsonPlainArrayConverter))] on `public record BookLevel`. Turns out if you do both, System.Text.Json crashes out. See https://dotnetfiddle.net/sFWIJg — dbc, May 16 '22 at 16:35

dbc · Accepted Answer · 2022-05-16T17:38:43.380

The converter from this answer by yueyinqiu to System.Text.Json fails to deserialize List<List<object>> basically works as-is to bind your BookLevel model to an array of JSON values corresponding to its properties. However, you have two problems:

Since "bids" and "asks" are jagged 2d arrays in the JSON, you must declare the corresponding properties to be collections, e.g.:

public record Depth(
    [property: JsonPropertyName("asks")] List<BookLevel> Asks,
    [property: JsonPropertyName("bids")] List<BookLevel> Bids,
);

You have applied both [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString) and [JsonConverter(typeof(JsonPlainArrayConverter<BookLevel>))] to BookLevel:

[JsonConverter(typeof(JsonPlainArrayConverter<BookLevel>))]
[JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)]
public record BookLevel
{
    [JsonPlainArrayIndex(0)]
    public string Price { get; init; }

    [JsonPlainArrayIndex(1)]
    public string Quantity { get; init; }
}

Turns out, when you do that, System.Text.Json will fail with a mysterious internal System.NullReferenceException:

System.NullReferenceException: Object reference not set to an instance of an object.
   at System.Text.Json.ThrowHelper.ThrowInvalidOperationException_NumberHandlingOnPropertyInvalid(JsonPropertyInfo jsonPropertyInfo)
   at System.Text.Json.Serialization.Metadata.JsonPropertyInfo.DetermineNumberHandlingForTypeInfo(Nullable`1 numberHandling)
   at System.Text.Json.Serialization.Metadata.JsonPropertyInfo.GetPolicies(Nullable`1 ignoreCondition, Nullable`1 declaringTypeNumberHandling)
   at System.Text.Json.Serialization.Metadata.JsonPropertyInfo`1.Initialize(Type parentClassType, Type declaredPropertyType, Type runtimePropertyType, ConverterStrategy runtimeClassType, MemberInfo memberInfo, Boolean isVirtual, JsonConverter converter, Nullable`1 ignoreCondition, Nullable`1 parentTypeNumberHandling, JsonSerializerOptions options)
   at System.Text.Json.Serialization.Metadata.JsonTypeInfo.CreateProperty(Type declaredPropertyType, Type runtimePropertyType, MemberInfo memberInfo, Type parentClassType, Boolean isVirtual, JsonConverter converter, JsonSerializerOptions options, Nullable`1 parentTypeNumberHandling, Nullable`1 ignoreCondition)
   at System.Text.Json.Serialization.Metadata.JsonTypeInfo..ctor(Type type, JsonConverter converter, Type runtimeType, JsonSerializerOptions options)
   at System.Text.Json.JsonSerializerOptions.<InitializeForReflectionSerializer>g__CreateJsonTypeInfo|112_0(Type type, JsonSerializerOptions options)
   at System.Text.Json.JsonSerializerOptions.GetClassFromContextOrCreate(Type type)
   at System.Text.Json.JsonSerializerOptions.GetOrAddClass(Type type)
   at System.Text.Json.Serialization.Metadata.JsonTypeInfo.get_ElementTypeInfo()
   at System.Text.Json.Serialization.JsonCollectionConverter`2.OnTryRead(Utf8JsonReader& reader, Type typeToConvert, JsonSerializerOptions options, ReadStack& state, TCollection& value)
   at System.Text.Json.Serialization.JsonConverter`1.TryRead(Utf8JsonReader& reader, Type typeToConvert, JsonSerializerOptions options, ReadStack& state, T& value)
   at System.Text.Json.Serialization.JsonConverter`1.TryReadAsObject(Utf8JsonReader& reader, JsonSerializerOptions options, ReadStack& state, Object& value)
   at System.Text.Json.Serialization.Converters.LargeObjectWithParameterizedConstructorConverter`1.ReadAndCacheConstructorArgument(ReadStack& state, Utf8JsonReader& reader, JsonParameterInfo jsonParameterInfo)
   at System.Text.Json.Serialization.Converters.ObjectWithParameterizedConstructorConverter`1.OnTryRead(Utf8JsonReader& reader, Type typeToConvert, JsonSerializerOptions options, ReadStack& state, T& value)
   at System.Text.Json.Serialization.JsonConverter`1.TryRead(Utf8JsonReader& reader, Type typeToConvert, JsonSerializerOptions options, ReadStack& state, T& value)
   at System.Text.Json.Serialization.JsonConverter`1.ReadCore(Utf8JsonReader& reader, JsonSerializerOptions options, ReadStack& state)
   at System.Text.Json.JsonSerializer.ReadFromSpan[TValue](ReadOnlySpan`1 utf8Json, JsonTypeInfo jsonTypeInfo, Nullable`1 actualByteCount)
   at System.Text.Json.JsonSerializer.ReadFromSpan[TValue](ReadOnlySpan`1 json, JsonTypeInfo jsonTypeInfo)
   at System.Text.Json.JsonSerializer.Deserialize[TValue](String json, JsonSerializerOptions options)

This looks to be a bug in System.Text.Json itself. You could report it if you want. Even if the two attributes are not intended to work together, System.Text.Json should not throw a NullReferenceException.

Demo fiddle here: https://dotnetfiddle.net/sFWIJg.

As a workaround to issue #2, you could unseal JsonPlainArrayConverter<T> to allow subclasses to use customized options when deserializing specific properties, like so:

public sealed class JsonNumberHandlingPlainArrayConverter<T> : JsonPlainArrayConverter<T> where T : new()
{
    protected override JsonSerializerOptions CustomizePropertyOptions(PropertyInfo info, JsonSerializerOptions options) =>
        new JsonSerializerOptions(options)
        {
            NumberHandling = JsonNumberHandling.AllowReadingFromString,
        };
}

// JsonPlainArrayIndexAttribute and  JsonPlainArrayConverter<T> adapted from this answer https://stackoverflow.com/a/71649515/3744182
// By https://stackoverflow.com/users/15283686/yueyinqiu
// To https://stackoverflow.com/questions/71649101/system-text-json-fails-to-deserialize-listlistobject

[AttributeUsage(AttributeTargets.Property, Inherited = true, AllowMultiple = false)]
public sealed class JsonPlainArrayIndexAttribute : Attribute
{
    public int Index { get; }
    public JsonPlainArrayIndexAttribute(int index) => this.Index = index;
}

public class JsonPlainArrayConverter<T> : JsonConverter<T> where T : new()
{
    protected virtual JsonSerializerOptions CustomizePropertyOptions(PropertyInfo info, JsonSerializerOptions options) => options;
    
    public override T? Read(
        ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
    {
        Debug.Assert(typeof(T) == typeToConvert);

        var props = typeToConvert.GetProperties();
        var linq = from prop in props
                   let attr = prop.GetCustomAttributes(typeof(JsonPlainArrayIndexAttribute), true)
                   where prop.CanWrite && attr.Length is 1
                   orderby ((JsonPlainArrayIndexAttribute)attr[0]).Index
                   select prop;

        var arr = JsonSerializer.Deserialize<IEnumerable<JsonElement>>(ref reader, options);
        if (arr is null)
            return default;

        var result = new T();
        foreach (var (prop, value) in linq.Zip(arr))
        {
            prop.SetValue(result, value.Deserialize(prop.PropertyType, CustomizePropertyOptions(prop, options)));
        }

        return result;
    }

    public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options)
    {
        var type = typeof(T);
        var props = type.GetProperties();
        var linq = from prop in props
                   let attr = prop.GetCustomAttributes(typeof(JsonPlainArrayIndexAttribute), true)
                   where prop.CanRead && attr.Length is 1
                   orderby ((JsonPlainArrayIndexAttribute)attr[0]).Index
                   select prop.GetValue(value);
        JsonSerializer.Serialize<IEnumerable<object>>(writer, linq, options);
    }
}

And then modify BookLevel as follows:

[JsonConverter(typeof(JsonNumberHandlingPlainArrayConverter<BookLevel>))]
public record BookLevel
{
    [JsonPlainArrayIndex(0)]
    public decimal Price { get; init; }

    [JsonPlainArrayIndex(1)]
    public decimal Quantity { get; init; }
}

Demo fiddle here: https://dotnetfiddle.net/AwtFni

Actually, `Price` and `Quantity` should be decimal and when I change it, it returns a JsonException. May you fix that? — nop, May 16 '22 at 17:09

Deserializing array of [price, quantity]

Snippet

Response

Attempt

1 Answers1