I have used this approach in a solution using asp core 1.1. First define a custom scheme:
public static class Authentication
{
    public const string Scheme = "Custom";
}
You then have to inherit AuthenticationHandler<TOptions>. Here is where the logic for validating the header value will go:
public class MyAuthenticationHandler : AuthenticationHandler<MyOptions>
{
    protected override Task<AuthenticateResult> HandleAuthenticateAsync()
    {
        var authorizationHeader = Context.Request.Headers["Authorization"];
        if (!authorizationHeader.Any())
            return Task.FromResult(AuthenticateResult.Skip());
        var value = authorizationHeader.ToString();
        if (string.IsNullOrWhiteSpace(value))
            return Task.FromResult(AuthenticateResult.Skip());
        // place logic here to validate the header value (decrypt, call db etc)
        var claims = new[]
        {
            new Claim(System.Security.Claims.ClaimTypes.Name, "Bob")
        };
        // create a new claims identity and return an AuthenticationTicket 
        // with the correct scheme
        var claimsIdentity = new ClaimsIdentity(claims, Authentication.Scheme);
        var ticket = new AuthenticationTicket(new ClaimsPrincipal(claimsIdentity), new AuthenticationProperties(), Authentication.Scheme);
        return Task.FromResult(AuthenticateResult.Success(ticket));
    }
}
In order to inherit AuthenticationHandler you must create an options class where you set the AuthenticationScheme-property to the scheme you are using:
public class MyOptions : AuthenticationOptions
{
    AuthenticationScheme = Authentication.Scheme;
}
After this you have to inherit AuthenticationMiddleware<TOptions>. This will create the handler you implemented in the previous step:
public class MyAuthenticationMiddleware : AuthenticationMiddleware<MyOptions>
{
    public MyAuthenticationMiddleware(RequestDelegate next, IOptions<MyOptions> options, ILoggerFactory loggerFactory, UrlEncoder encoder) : base(next, options, loggerFactory, encoder)
    {
    }
    protected override AuthenticationHandler<MyOptions> CreateHandler()
    {
        return new MyAuthenticationHandler();
    }
}
In order to easily plug in your middleware you can define these extension methods:
public static IApplicationBuilder UseMyAuthentication(this IApplicationBuilder app, IConfigurationSection config)
{
    return app.UseMyAuthentication(options => {});
}
private static IApplicationBuilder UseMyAuthentication(this IApplicationBuilder app, Action<MyOptions> configure)
{
    var options = new MyOptions();
    configure?.Invoke(options);
    return app.UseMiddleware<MyAuthenticationMiddleware>(new OptionsWrapper<MyOptions>(options));
}
Then in your Startup class you can finally add your middleware:
public void Configure(IApplicationBuilder app, IHostingEnvironment env)
{
    app.UseMyAuthentication(Configuration.GetSection("MyAuthenticationOptions"));
    // other stuff
    app.UseMvc();
}
Then add the AuthorizeAttribute on your actions specifying the scheme you just created:
[Authorize(ActiveAuthenticationSchemes = Authentication.Scheme)]
public IActionResult Get()
{
    // stuff ...
}
There are a lot of steps but hopefully this will get you going!