private void CollectionErrors()
    {
        Errors.Clear();
        if (string.IsNullOrWhiteSpace(NumberProp))
        {
            Errors.Add(nameof(NumberProp), "This value cannot be empty");
        }
        else if (decimal.TryParse(NumberProp, out _))
        {
            Errors.Add(nameof(NumberProp), "NumberProp is not a decimal number!");
        }
    }
And it is better not to clear all errors completely, but to clear only the error of the checked property:
public Dictionary<string, string> Errors { get; } = new Dictionary<string, string>();
public string this[string propertyName]
{
    get
    {
        CollectionErrors(propertyName);
        return Errors.ContainsKey(propertyName) ? Errors[propertyName] : string.Empty;
    }
}
private void CollectionErrors(string propertyName)
{
    Errors.Remove(propertyName);
    if (propertyName == nameof(NumberProp))
    {
        if (string.IsNullOrWhiteSpace(NumberProp))
        {
            Errors.Add(nameof(NumberProp), "This value cannot be empty");
        }
        else if (decimal.TryParse(NumberProp, out _))
        {
            Errors.Add(nameof(NumberProp), "NumberProp is not a decimal number!");
        }
    }
}
public string Error => string.Join(Environment.NewLine, Errors.Select(pair => $"[{pair.Key}]=\"{pair.Value}\""));
By convention (contract) INotifyPropertyChanged intreface, the PropertyChanged event should only be raised WHEN the property CHANGES.
Therefore, the correct implementation should be like this:
    private double _number;
    public double Number
    {
        get { return _number; }
        private set
        {
            if (Equals(_number, value))
                return;
            _number = value;
            OnPropertyChanged(nameof(Number));
        }
    }
    private string _numberProp;
    public string NumberProp
    {
        get { return _numberProp; }
        set
        {
            if (Equals(_numberProp, value))
                return;
            _numberProp = value;
            OnPropertyChanged(nameof(NumberProp));
            if (double.TryParse(value, out double number))
                Number = number;
        }
    }
A complete example of a class implementation with two bound properties.
NumberString - Used to bind to a TextBox.
Number - Contains the value of the NumberString property converted to double.
Error checking is implemented for both properties.
Also notice the [CallerMemberName] attribute.
It allows you not to specify the property name in the methods.
public class Data : INotifyPropertyChanged, IDataErrorInfo
{
    public event PropertyChangedEventHandler PropertyChanged;
    private void OnPropertyChanged([CallerMemberName] string propertyName = null)
        => PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
    private double _number;
    public double Number
    {
        get { return _number; }
        private set
        {
            if (Equals(_number, value))
                return;
            _number = value;
            CollectionErrors();
            OnPropertyChanged();
        }
    }
    private string _numberProp;
    public string NumberProp
    {
        get { return _numberProp; }
        set
        {
            if (Equals(_numberProp, value))
                return;
            _numberProp = value;
            CollectionErrors();
            OnPropertyChanged();
            if (double.TryParse(value, out double number))
                Number = number;
        }
    }
    public Dictionary<string, string> Errors { get; } = new Dictionary<string, string>();
    public string this[string propertyName] => Errors[propertyName];
    private void CollectionErrors([CallerMemberName] string propertyName = null)
    {
        if (propertyName == null)
            return;
        Errors.Remove(propertyName);
        if (propertyName == nameof(NumberProp))
        {
            if (string.IsNullOrWhiteSpace(NumberProp))
            {
                Errors[nameof(NumberProp)] = "This value cannot be empty";
            }
            else if (decimal.TryParse(NumberProp, out _))
            {
                Errors[nameof(NumberProp)] = "NumberProp is not a decimal number!";
            }
        }
        else if (propertyName == nameof(Number))
        {
            if (Number < 0 || Number > 1000)
                Errors[nameof(Number)] = "The number must be in the range 0 ... 1000!";
        }
    }
    public string Error => string.Join(Environment.NewLine, Errors.Select(pair => $"[{pair.Key}]=\"{pair.Value}\""));
}