You can change the legend key generating function. This still requires a bit of manual intervention, but arguably less than using the grobs.
library(ggplot2)
library(grid)
data(iris)
iris$abbrev = substr( iris$Species, 1, 2 )
oldK <- GeomText$draw_key # to save for later
# define new key
# if you manually add colours then add vector of colours 
# instead of `scales::hue_pal()(length(var))`
GeomText$draw_key <- function (data, params, size, 
                               var=unique(iris$abbrev), 
                               cols=scales::hue_pal()(length(var))) {
    # sort as ggplot sorts these alphanumerically / or levels of factor
    txt <- if(is.factor(var)) levels(var) else sort(var)
    txt <- txt[match(data$colour, cols)]
    textGrob(txt, 0.5, 0.5,  
             just="center", 
             gp = gpar(col = alpha(data$colour, data$alpha), 
                       fontfamily = data$family, 
                       fontface = data$fontface, 
                       fontsize = data$size * .pt))
}
ggplot(data=iris, aes(x=Sepal.Length, y=Sepal.Width, 
                      shape=Species, colour=Species)) +  
  geom_text(aes(label = abbrev))
# reset key
GeomText$draw_key <- oldK
