This is a tested method
public static String getExtension(String fileName) {
    char ch;
    int len;
    if(fileName==null || 
            (len = fileName.length())==0 || 
            (ch = fileName.charAt(len-1))=='/' || ch=='\\' || //in the case of a directory
             ch=='.' ) //in the case of . or ..
        return "";
    int dotInd = fileName.lastIndexOf('.'),
        sepInd = Math.max(fileName.lastIndexOf('/'), fileName.lastIndexOf('\\'));
    if( dotInd<=sepInd )
        return "";
    else
        return fileName.substring(dotInd+1).toLowerCase();
}
And test case:
@Test
public void testGetExtension() {
    assertEquals("", getExtension("C"));
    assertEquals("ext", getExtension("C.ext"));
    assertEquals("ext", getExtension("A/B/C.ext"));
    assertEquals("", getExtension("A/B/C.ext/"));
    assertEquals("", getExtension("A/B/C.ext/.."));
    assertEquals("bin", getExtension("A/B/C.bin"));
    assertEquals("hidden", getExtension(".hidden"));
    assertEquals("dsstore", getExtension("/user/home/.dsstore"));
    assertEquals("", getExtension(".strange."));
    assertEquals("3", getExtension("1.2.3"));
    assertEquals("exe", getExtension("C:\\Program Files (x86)\\java\\bin\\javaw.exe"));
}