Inspired by kvb's answer using NumericLiterals, I was driven to develop an approach which would allow us to force "sane" type signatures without having to add extensive type annotations.
First we define some helper functions and wrapper type for language primitives:
let inline zero_of (target:'a) : 'a = LanguagePrimitives.GenericZero<'a>
let inline one_of (target:'a) : 'a = LanguagePrimitives.GenericOne<'a>
let inline two_of (target:'a) : 'a = one_of(target) + one_of(target)
let inline three_of (target:'a) : 'a = two_of(target) + one_of(target)
let inline negone_of (target:'a) : 'a = zero_of(target) - one_of(target)
let inline any_of (target:'a) (x:int) : 'a =
    let one:'a = one_of target
    let zero:'a = zero_of target
    let xu = if x > 0 then 1 else -1
    let gu:'a = if x > 0 then one else zero-one
    let rec get i g = 
        if i = x then g
        else get (i+xu) (g+gu)
    get 0 zero 
type G<'a> = {
    negone:'a
    zero:'a
    one:'a
    two:'a
    three:'a
    any: int -> 'a
}    
let inline G_of (target:'a) : (G<'a>) = {
    zero = zero_of target
    one = one_of target
    two = two_of target
    three = three_of target
    negone = negone_of target
    any = any_of target
}
Then we have:
let inline factorizeG n = 
    let g = G_of n
    let rec factorize n j flist =  
        if n = g.one then flist 
        elif n % j = g.zero then factorize (n/j) j (j::flist) 
        else factorize n (j + g.one) (flist) 
    factorize n g.two []
[Edit: due to an apparent bug with F# 2.0 / .NET 2.0, factorizen, factorizeL, and factorizeI below run significantly slower than factorizeG when compiled in Release-mode but otherwise run slightly faster as expected -- see F# performance question: what is the compiler doing?]
Or we can take it a few step further (inspired by Expert F#, p.110):
let inline factorize (g:G<'a>) n =   //'
    let rec factorize n j flist =  
        if n = g.one then flist 
        elif n % j = g.zero then factorize (n/j) j (j::flist) 
        else factorize n (j + g.one) (flist) 
    factorize n g.two []
//identical to our earlier factorizeG
let inline factorizeG n = factorize (G_of n) n
let gn = G_of 1  //int32
let gL = G_of 1L //int64
let gI = G_of 1I //bigint
//allow us to limit to only integral numeric types
//and to reap performance gain by using pre-computed instances of G
let factorizen = factorize gn
let factorizeL = factorize gL
let factorizeI = factorize gI
Also, here is an extended version of kvb's NumericLiteralG which allows us to use "2G", "-8G", etc.  Though I couldn't figure out how to implement a memoization strategy (though that should be doable for G.any).
module NumericLiteralG = 
    let inline FromZero() = LanguagePrimitives.GenericZero
    let inline FromOne() = LanguagePrimitives.GenericOne
    let inline FromInt32(n:int):'a =
        let one:'a = FromOne()
        let zero:'a = FromZero()
        let nu = if n > 0 then 1 else -1
        let gu:'a = if n > 0 then one else zero-one
        let rec get i g = 
            if i = n then g
            else get (i+nu) (g+gu)
        get 0 zero