Here is a code I've just write that does the job.
Read the comments for further details!
Note: I tested this code fragment under Google Chrome!
function compareTypes(a,b){
    if (a instanceof b) {
        if(b instanceof a) {
            /* same types */
            return 0; 
        }
        else {
            /* first is son */
            return -1;
        }
    }
    if (b instanceof a) {
        /* second is son */
        return 1; 
    }
    return 0; /* not related to each other */
}
function closestInstanceOf(obj, possibleTypesAsAdditionalArguments) {
    /*
    * How to call this function:
    * The first argument should be the object to check.
    * All other arguments are possible types.
    */
    if (arguments.length === 0)
        return; /* undefined */
    if (arguments.length === 1)
        return Object; /* If no possible type was passed, we returned Object as the base type */
    /* converts the arguments to Javascript array */
    var types = Array.prototype.slice.call(arguments, 0);
    var obj = types[0]; /* object to be checked */
    types.splice(0, 1); /* possible types */
    /* Sorts the types array from the more specific to the more generic type */
    types.sort(compareTypes);
    /* find the first match */
    for (var i = 0; i < types.length; i++) {
        var type = types[i];
        if (obj instanceof type) {
            return type;
        }
    }
    return Object;
}
// examples
// 1. Sorting types
console.info([Array, Number, Object, String, Boolean].sort(compareTypes));
// results:
//[function, function, function, function, function]
//  0: function Array() { [native code] }
//  1: function Number() { [native code] }
//  2: function String() { [native code] }
//  3: function Boolean() { [native code] }
//  4: function Object() { [native code] }
// 2. Find specific type
console.info(closestInstanceOf([], Array, Number, Object, String, Boolean));
// results:
// function Array() { [native code] } 
// 3. Find specific type
console.info(closestInstanceOf(new String("Hello World"), Array, Number, Object, String, Boolean));
// results:
// function String() { [native code] }
// 4. Find specific type
console.info(closestInstanceOf(new Number(19), Array, Number, Object, String, Boolean));
// results:
// function Number() { [native code] }
// 5. Find specific type
console.info(closestInstanceOf("Hello world", Array, Number, Object, String, Boolean));
// results:
// function Object() { [native code] } 
// Note: String iterals derived from object!
// 6. Find specific type
console.info(closestInstanceOf(18, Array, Number, Object, String, Boolean));
// results:
// function Object() { [native code] } 
// The same applies for numbers
// 7. Find specific type
console.info(closestInstanceOf(false, Array, Number, Object, String, Boolean));
// results:
// function Object() { [native code] } 
// And booleans...
// 8. Find specific type
console.info(closestInstanceOf(new Boolean(false), Array, Number, Object, String, Boolean));
// results:
// function Object() { [native code] } 
// Again, using new Boolean does the job!
// 9. Find specific type
console.info(closestInstanceOf(function () { /* this is custom class */ }, Array, Number, Object, String, Boolean));
// results:
// function Object() { [native code] } 
// This code works for custom class as well, try it!