Newtonsoft Ignore Property when deserializing in custom JsonConverter

Question

This is my Custom JsonConverter

public class TransactionJsonConverter : JsonConverter
{
    private Dictionary<string, object> _objectReferenceHolder = new Dictionary<string, object>();
    private TransactionCore _core;

    public TransactionJsonConverter(TransactionCore core)
    {
        _core = core;
    }

    public override bool CanConvert(Type objectType)
    {
        if (objectType == typeof(Transaction)) return true;

        return false;
    }

    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
    {
        JObject jObject = JObject.Load(reader);

        if (objectType == typeof(Transaction))
        {
            // process id must not be set --> will be set during population
            Transaction transaction = new Transaction(_core, null);

            _objectReferenceHolder.Add("Transaction", transaction);
            serializer.Populate(jObject.CreateReader(), transaction);
            transaction.Properties.ProcessConfiguration = _core.ConfigProvider.GetDocProcess(transaction.ProcessId);

            return transaction;
        }
        throw new NotImplementedException();
    }

    public override bool CanWrite => false;

    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
    {
        throw new NotImplementedException();
    }
}

The Object Transaction has a field Rows (Row[]) and Row has 2 Properties like that:

private int? _baseLoyaltyPoints = null;
private int? _loyaltyPoints = null;

[JsonIgnore]
public override int BaseLoyaltyPoints
{
    get
    {
        if (_baseLoyaltyPoints.HasValue)
            return _baseLoyaltyPoints.Value;

        return calcBasePoints();
    }
    internal set
    {
        if (value != 0)
            _baseLoyaltyPoints = value;
    }
}

[JsonIgnore]
public override int LoyaltyPoints
{
    get
    {
        if (_loyaltyPoints.HasValue)
            return _loyaltyPoints.Value;
        else return BaseLoyaltyPoints;
    }
    internal set
    {
        if (value != 0)
            _loyaltyPoints = value;
    }
}

As you can see I already tried to add the JsonIgnore attribute, but Custom JsonConverters do not have accsess to attributes. I am currently struggling to implement a solution so the JsonConverter would dezerialize the private variables but not the Properties

The end goal is that the private Variable does not get set when dezerialising the Property but does get set if it already had a value