I actually have the following solution. Anyone who is interested can copy/paste the following code to check for any cell selected and see if I have missed out on any edge case.
The thinking is like this: even if the user choose to use parenthesis, we don't want anything immediately before the "(". 
Similarly, we want to get rid of any simple arithmetic operators (+,-,-,*,/,^).  I put down both subtraction and negate here since for some input language (e.g. Chinese), the two symble could be different? Not sure about that but don't want to risk it...
Lastly, we don't want a range of cells (i.e. no ":" or ",").
Sub test()
    Dim rng As range
    Set rng = Selection
    MsgBox (referenceOnly(rng))
End Sub
Then we have the function that test if it is only a reference
Function referenceOnly(rng As range) As Boolean
    ' Three situations tht it might has formula
    ' 1: has anything before "(" except for "="
    ' 2: has any of the simple specialsmetic operators
    ' 3: has ":" which refers to a range of cells
    referenceOnly = True
    Dim str As String
    If rng.HasFormula Then
        str = rng.Formula
    Else
        referenceOnly = False
        Exit Function
    End If
    ' start of checks
    Dim i As Integer
    ' start pos of "("
    Dim startPos As Integer
    ' check 1
    startPos = InStr(2, str, "(")
    If startPos > 0 Then
        If startPos <> 2 Then
            referenceOnly = False
            Exit Function
        End If
    End If
    ' referenceOnly 2 and 3
    Dim specials(1 To 6) As String
    specials(1) = chr(43)   '+
    specials(2) = chr(45)   '-
    specials(3) = chr(46)   '-
    specials(3) = chr(42)   '*
    specials(4) = chr(47)   '/
    specials(5) = chr(94)   '^
    specials(6) = chr(58)   ':
    For i = 2 To Len(str)
        If IsInArray(Mid(str, i, 1), specials) Then
            referenceOnly = False
            Exit Function
        End If
    Next i
End Function
Helper function to see if some element is in the array. Copied the code from somewhere :)
Private Function IsInArray(valToBeFound As Variant, arr As Variant) As Boolean
'INPUT: Pass the function a value to search for and an array of values of any data type.
'OUTPUT: True if is in array, false otherwise
    Dim element As Variant
    On Error GoTo IsInArrayError: 'array is empty
        For Each element In arr
            If element = valToBeFound Then
                IsInArray = True
                Exit Function
            End If
        Next element
    Exit Function
IsInArrayError:
    On Error GoTo 0
    IsInArray = False
End Function
Just found out that someone down-voted this thread. Not sure what is that about lol