I am reiterating what @Danilo Kobold said. 
I had to make sure that users can enter only numbers (0-9) [Without 'e' or '.' or '-'] and a limit of 4 values only. 
app.directive("limitTo", [function() {
    return {
        restrict: "A",
        link: function(scope, elem, attrs) {
            var limit = parseInt(attrs.limitTo);
            var exclude = /Backspace|Enter/;
            angular.element(elem).on("keydown", function(e) {
                if (event.keyCode > 47 && event.keyCode < 58) {
                    if (this.value.length == limit) e.preventDefault();
                } else if (!exclude.test(event.key)) {
                    e.preventDefault();
                }
            });
        }
    }
}]);
If you want to use only limit then use 
app.directive("limitTo", [function() {
    return {
        restrict: "A",
        link: function(scope, elem, attrs) {
            var limit = parseInt(attrs.limitTo);
            var exclude = /Backspace|Enter/;
            angular.element(elem).on("keydown", function(e) {
                if (!exclude.test(event.key)) {
                    if (this.value.length == limit) e.preventDefault();
                }
            });
        }
    }
}]);
You can add more keys if you want to the exclude variable, like this:
var exclude = /Backspace|Enter|Tab|Delete|Del|ArrowUp|Up|ArrowDown|Down|ArrowLeft|Left|ArrowRight|Right/;
Got idea from this post
Hope I helped someone.