Here is a simple standalone Winforms custom control, derived from the standard TextBox, that allows only System.Int32 input (it could be easily adapted for other types such as System.Int64, etc.). It supports copy/paste operations and negative numbers:
public class Int32TextBox : TextBox
{
    protected override void OnKeyPress(KeyPressEventArgs e)
    {
        base.OnKeyPress(e);
        NumberFormatInfo fi = CultureInfo.CurrentCulture.NumberFormat;
        string c = e.KeyChar.ToString();
        if (char.IsDigit(c, 0))
            return;
        if ((SelectionStart == 0) && (c.Equals(fi.NegativeSign)))
            return;
        // copy/paste
        if ((((int)e.KeyChar == 22) || ((int)e.KeyChar == 3))
            && ((ModifierKeys & Keys.Control) == Keys.Control))
            return;
        if (e.KeyChar == '\b')
            return;
        e.Handled = true;
    }
    protected override void WndProc(ref System.Windows.Forms.Message m)
    {
        const int WM_PASTE = 0x0302;
        if (m.Msg == WM_PASTE)
        {
            string text = Clipboard.GetText();
            if (string.IsNullOrEmpty(text))
                return;
            if ((text.IndexOf('+') >= 0) && (SelectionStart != 0))
                return;
            int i;
            if (!int.TryParse(text, out i)) // change this for other integer types
                return;
            if ((i < 0) && (SelectionStart != 0))
                return;
        }
        base.WndProc(ref m);
    }
Update 2017: My first answer has some issues:
- you can type something that's longer than an integer of a given type (for example 2147483648 is greater than Int32.MaxValue);
 
- more generally, there's no real validation of the result of what has been typed;
 
- it only handles int32, you'll have to write specific TextBox derivated control for each type (Int64, etc.)
 
So I came up with another version that's more generic, that still supports copy/paste, + and - sign, etc.
public class ValidatingTextBox : TextBox
{
    private string _validText;
    private int _selectionStart;
    private int _selectionEnd;
    private bool _dontProcessMessages;
    public event EventHandler<TextValidatingEventArgs> TextValidating;
    protected virtual void OnTextValidating(object sender, TextValidatingEventArgs e) => TextValidating?.Invoke(sender, e);
    protected override void WndProc(ref Message m)
    {
        base.WndProc(ref m);
        if (_dontProcessMessages)
            return;
        const int WM_KEYDOWN = 0x100;
        const int WM_ENTERIDLE = 0x121;
        const int VK_DELETE = 0x2e;
        bool delete = m.Msg == WM_KEYDOWN && (int)m.WParam == VK_DELETE;
        if ((m.Msg == WM_KEYDOWN && !delete) || m.Msg == WM_ENTERIDLE)
        {
            DontProcessMessage(() =>
            {
                _validText = Text;
                _selectionStart = SelectionStart;
                _selectionEnd = SelectionLength;
            });
        }
        const int WM_CHAR = 0x102;
        const int WM_PASTE = 0x302;
        if (m.Msg == WM_CHAR || m.Msg == WM_PASTE || delete)
        {
            string newText = null;
            DontProcessMessage(() =>
            {
                newText = Text;
            });
            var e = new TextValidatingEventArgs(newText);
            OnTextValidating(this, e);
            if (e.Cancel)
            {
                DontProcessMessage(() =>
                {
                    Text = _validText;
                    SelectionStart = _selectionStart;
                    SelectionLength = _selectionEnd;
                });
            }
        }
    }
    private void DontProcessMessage(Action action)
    {
        _dontProcessMessages = true;
        try
        {
            action();
        }
        finally
        {
            _dontProcessMessages = false;
        }
    }
}
public class TextValidatingEventArgs : CancelEventArgs
{
    public TextValidatingEventArgs(string newText) => NewText = newText;
    public string NewText { get; }
}
For Int32, you can either derive from it, like this:
public class Int32TextBox : ValidatingTextBox
{
    protected override void OnTextValidating(object sender, TextValidatingEventArgs e)
    {
        e.Cancel = !int.TryParse(e.NewText, out int i);
    }
}
or w/o derivation, use the new TextValidating event like this:
var vtb = new ValidatingTextBox();
...
vtb.TextValidating += (sender, e) => e.Cancel = !int.TryParse(e.NewText, out int i);
but what's nice is it works with any string, and any validation routine.