I had the exact same problem. I tried "everything" to make it both user friendly and to not accept invalid values. Finally I gave up on apparently easy solutions, like ng-pattern, and with help of a friend @Teemu Turkia, we came up with integers-only directive.
It uses type="text", supports both min and max, do not accept chars beyond numbers and - (as a first character in case minimum is negative) to be typed.
Also, ng-model is never assigned with invalid value such as empty string or NaN, only values between given range or null. 
I know, at first it looks rather intimidating ;)
HTML 
// note: uses underscore.js
<body>
  <form name="form">
    <header>DD / MM / YYYY</header>
    <section>
      <input type="text" 
             name="day" 
             ng-model="day" 
             min="1" 
             max="31" 
             integers-only>
      <input type="text" 
             name="month" 
             ng-model="month" 
             min="1" 
             max="12" 
             integers-only>
      <input type="text" 
             name="year" 
             ng-model="year" 
             min="1900" 
             max="2016" 
             integers-only> 
    </section>
    <section>
      <span ng-show="form.day.$invalid">Invalid day</span>
      <span ng-show="form.month.$invalid">Invalid month</span>
      <span ng-show="form.year.$invalid">Invalid year</span>
    </section>
  </form> 
</body>
JavaScript
/**
 * numeric input
 * <input type="text" name="name" ng-model="model" min="0" max="100" integers-only>
 */
angular.module('app', [])
.directive('integersOnly', function() {
  return {
    restrict: 'A',
    require: 'ngModel',
    scope: {
        min: '=',
        max: '='
    },
    link: function(scope, element, attrs, modelCtrl) {
      function isInvalid(value) {
        return (value === null || typeof value === 'undefined' || !value.length);
      }
      function replace(value) {
        if (isInvalid(value)) {
          return null;
        }
        var newValue = [];
        var chrs = value.split('');
        var allowedChars = ['0','1','2','3','4','5','6','7','8','9','-'];
        for (var index = 0; index < chrs.length; index++) {
          if (_.contains(allowedChars, chrs[index])) {
            if (index > 0 && chrs[index] === '-') {
              break;
            }
            newValue.push(chrs[index]);
          }
        }
        return newValue.join('') || null;
      }
      modelCtrl.$parsers.push(function(value) {
        var originalValue = value;
        value = replace(value);
        if (value !== originalValue) {
          modelCtrl.$setViewValue(value);
          modelCtrl.$render();
        }
        return value && isFinite(value) ? parseInt(value) : value;
      });
      modelCtrl.$formatters.push(function(value) {
        if (value === null || typeof value === 'undefined') {
          return null;
        }
        return parseInt(value);
      });
      modelCtrl.$validators.min = function(modelValue) {
        if (scope.min !== null && modelValue !== null && modelValue < scope.min) { return false; }
        return true;
      };
      modelCtrl.$validators.max = function(modelValue) {
        if (scope.max !== null && modelValue !== null && modelValue > scope.max) { return false; }
        return true;
      };
      modelCtrl.$validators.hasOnlyChar = function(modelValue) {
        if (!isInvalid(modelValue) && modelValue === '-') { return false; }
        return true;
      };
    }
  };
});
Result

Related plunker here http://plnkr.co/edit/mIiKuw