I finally did it by my own. As I spent a significant amount time on this and saw that others where also looking for a solution, here is how I did it.
At first, I suggest you read https://docs.zendframework.com/tutorials/in-depth-guide/models-and-servicemanager/ if you're not familiar with Dependency Injection and Factories (this was my case).
module.config.php
// In module/YourModule/config/module.config.php:
namespace YourAppNamespace;
use Zend\ServiceManager\Factory\InvokableFactory;
return [
    'controllers' => [
        'factories' => [
            Controller\YourController::class => Factory\YourControllerFactory::class,
        ],
    ],
    'service_manager' => [ /** Your Service Manager Config **/ ]        
    'router' => [ /** Your Router Config */ ]
    'view_manager' => [ /** Your ViewManager Config */ ],
];
YourControllerFactory.php
// In module/YourModule/src/Controller/YourControllerFactory.php:
namespace YourAppNamespace\Factory;
use YourAppNamespace\Controller\YourController;
use Interop\Container\ContainerInterface;
use Zend\ServiceManager\Factory\FactoryInterface;
class YourControllerFactory implements FactoryInterface
{
    /**
     * @param ContainerInterface $container
     * @param string             $requestedName
     * @param null|array         $options
     *
     * @return YourController
     */
    public function __invoke(ContainerInterface $container, $requestedName, array $options = null)
    {
        $controllerPluginManager = $container;
        $serviceManager          = $controllerPluginManager->get('ServiceManager');
        // Requires zf-campus/zf-oauth2
        $server   = $serviceManager->get('ZF\OAuth2\Service\OAuth2Server');
        $provider = $serviceManager->get('ZF\OAuth2\Provider\UserId');
        return new YourController($server, $provider);
    }
}
YourController.php
// In module/YourModule/src/Controller/YourController.php:
namespace YourAppNamespace\Controller;
use ZF\OAuth2\Controller\AuthController;
use OAuth2\Request as OAuth2Request;
use ZF\OAuth2\Provider\UserId\UserIdProviderInterface;
class YourController extends AuthController
{
    public function __construct($serverFactory, UserIdProviderInterface $userIdProvider)
    {
        parent::__construct($serverFactory, $userIdProvider);
    }
    public function indexAction()
    {
        $server = call_user_func($this->serverFactory, "oauth");
        if (!$server->verifyResourceRequest(OAuth2Request::createFromGlobals())) {
            // Failure
            $response = $server->getResponse();
            return $this->getApiProblemResponse($response);
        }
        // Success
        echo json_encode(array('success' => true, 'message' => 'It works!'));
    }
}
Hope it helps!