Below is a simple directive, which allows you only to enter numbers inside a input field.
When linked to an input field:
<input type="text" ng-model="test" valid-number>
It bind a keydown event on the input field, which checks what key has been pressed. Each key on your keyboard has an global number. For example: enter has the numer 13 assigned.
A complete list of key codes
When you press a key on your keyboard, and its matches the key code listed inside var keyCode 
var keyCode = [8, 9, 37, 39, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 110];
then we prevent the action from executing.
(function () {
  angular.module("app", [])
  
  .directive("validNumber", function () {
        return {
            restrict: "EA",
            require: "?ngModel",
            link: function (scope, element, attrs, ngModel) {
                if (!ngModel) return;
                var keyCode = [8, 9, 37, 39, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 110];
                element.bind("keydown", function (event) {
                    if ($.inArray(event.which, keyCode) === -1) {
                        scope.$apply(function () {
                            scope.$eval(attrs.validNumber);
                            event.preventDefault();
                        });
                        event.preventDefault();
                    }
                });
            }
        };
    });
})();
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.2.23/angular.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<div ng-app="app">
  <input type="text" ng-model="test" valid-number="" name="test">
</div>
 
 
UPDATE
Based on your comments on other posted awnsers. You can just use .replace() function to replace everything except numbers.
Even when you copy and paste a valid number with invalid characters, it will replace the invalid characters.
I allowed users to write an dot ., because 5.2 is a valid number. But if you dont want this, you can replace the regex with
value.replace(/[^0-9]/g, "");
DEMO
(function () {
  angular.module("app", [])
  
  .directive("validNumber", function () {
        return {
            restrict: "EA",
            require: "?ngModel",
            link: function (scope, element, attrs, ngModel) {
                if (!ngModel) return;
                element.bind("keypress keyup", function (event) {
                    var value = element.val();
                    value = value.replace(/[^0-9.]/g, "");
                    
                    element.val(value);
                });
            }
        };
    });
})();
<script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.2.23/angular.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<div ng-app="app">
  <input type="text" ng-model="test" valid-number="" name="test">
</div>