I think you can validate with using .reject(&:marked_for_destruction?).length.
How about this?
class User < ActiveRecord::Base
  has_many :groups do
    def length
      reject(&:marked_for_destruction?).length
    end
  end
  accepts_nested_attributes_for :groups, allow_destroy: true
  validates :groups, length: { maximum: 5 }
end
Or this.
class User < ActiveRecord::Base
  has_many :groups
  accepts_nested_attributes_for :groups, allow_destroy: true
  GROUPS_MAX_LENGTH = 5
  validate length_of_groups
  def length_of_groups
    groups_length = 0
    if groups.exists?
      groups_length = groups.reject(&:marked_for_destruction?).length
    end
    errors.add(:groups, 'too many') if groups_length > GROUPS_MAX_LENGTH
  end
end
Then, you can command.
@user.assign_attributes(params[:user])
@user.valid?
Thank you for reading.
References:
http://homeonrails.com/2012/10/validating-nested-associations-in-rails/
http://qiita.com/asukiaaa/items/4797ce44c3ba7bd7a51f