I got a look at the source code of the library, here is what the Value getter does:
get
{
    string str = this.FormulaA1;
    if (!XLHelper.IsNullOrWhiteSpace(str))
    {
        string str2;
        string sName;
        if (str[0] == '{')
        {
            str = str.Substring(1, str.Length - 2);
        }
        if (str.Contains<char>('!'))
        {
            sName = str.Substring(0, str.IndexOf('!'));
            if (sName[0] == '\'')
            {
                sName = sName.Substring(1, sName.Length - 2);
            }
            str2 = str.Substring(str.IndexOf('!') + 1);
        }
        else
        {
            sName = this.Worksheet.Name;
            str2 = str;
        }
        if (this._worksheet.Workbook.WorksheetsInternal.Any<XLWorksheet>(w => (string.Compare(w.Name, sName, true) == 0)) && XLHelper.IsValidA1Address(str2))
        {
            return this._worksheet.Workbook.Worksheet(sName).Cell(str2).Value;
        }
        object obj2 = this.Worksheet.Evaluate(str);
        IEnumerable enumerable = obj2 as IEnumerable;
        if ((enumerable != null) && !(obj2 is string))
        {
            using (IEnumerator enumerator = enumerable.GetEnumerator())
            {
                while (enumerator.MoveNext())
                {
                    return enumerator.Current;
                }
            }
        }
        return obj2;
    }
    string s = this.HasRichText ? this._richText.ToString() : this._cellValue;
    if (this._dataType == XLCellValues.Boolean)
    {
        return (s != "0");
    }
    if (this._dataType == XLCellValues.DateTime)
    {
        return DateTime.FromOADate(double.Parse(s));
    }
    if (this._dataType == XLCellValues.Number)
    {
        return double.Parse(s);
    }
    if (this._dataType == XLCellValues.TimeSpan)
    {
        return TimeSpan.Parse(s);
    }
    return s;
}
I noticed, that when it reach string s = this.HasRichText ? this._richText.ToString() : this._cellValue;, if you have never called cell.RichText (so neither viewing it with the debugger) you get the correct value, because this.HasRichText is false because the class has still not assigned the correct value to it.
When it(this.HasRichText) is true, you get this._richText.ToString(), that is the formatted number.
So, if you access the Value property before accessing RichText you should get the correct value, anyway you can get _cellValue using reflection, then convert it to double like this:
var theRealDoubleValue = Convert.ToDouble(yourCell.GetType().GetField("_cellValue", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance).GetValue(yourCell));