Server:
Try FirebaseAdmin for server side. Very straighforward with this package.
https://github.com/firebase/firebase-admin-dotnet
Follow these setup instructions:
https://firebase.google.com/docs/admin/setup#c
For the Xamarin app:
I decided I didn't want to use the CrossGeeks plugin and it was pretty straightforward.
For Android:
Install the relevant Xamarin.Firebase packages and create your own Firebase Messaging Class in Android project inheriting the package FirebaseMessagingService.
[Service]
[IntentFilter(new[] { "com.google.firebase.INSTANCE_ID_EVENT" })]
[IntentFilter(new[] { "com.google.firebase.MESSAGING_EVENT" })]
class PushNotificationFirebaseMessagingService : FirebaseMessagingService
{
    private static string foregroundChannelId = "9001";
    public override void OnNewToken(string refreshedToken)
    {
        base.OnNewToken(refreshedToken);
        SendRegistrationToServer(refreshedToken);
    }
    private void SendRegistrationToServer(string token)
    {
        //Your code here to register device token on server
    }
    public override void OnMessageReceived(RemoteMessage message)
    {
        SendNotification(message);
        base.OnMessageReceived(message);
    }
    private void SendNotification(RemoteMessage message)
    {
        try
        {
            var notificationManager = GetSystemService(Context.NotificationService) as NotificationManager;
            var notificationChannel = new NotificationChannel(foregroundChannelId, "messaging_channel", NotificationImportance.High);
            var audioAttributes = new AudioAttributes.Builder()
                .SetContentType(AudioContentType.Sonification)
                .SetUsage(AudioUsageKind.Notification).Build();
            var notificationUri = RingtoneManager.GetDefaultUri(RingtoneType.Notification);
            notificationChannel.EnableLights(true);
            notificationChannel.EnableVibration(true);
            notificationChannel.SetSound(notificationUri, audioAttributes);
            notificationManager.CreateNotificationChannel(notificationChannel);
            var remoteNotification = message.GetNotification();
            var builder = new Notification.Builder(this, foregroundChannelId)
                .SetContentTitle(remoteNotification.Title)
                .SetContentText(remoteNotification.Body)
                .SetSmallIcon(Resource.Mipmap.icon);
            var notification = builder.Build();
            notificationManager.Notify(0, notification);
        }
        catch (Exception ex)
        {
        }
    }
}
Add the following to the AndroidManifest.xml in the Application tag.
<receiver android:name="com.google.firebase.iid.FirebaseInstanceIdInternalReceiver" android:exported="false" />
<receiver android:name="com.google.firebase.iid.FirebaseInstanceIdReceiver" android:exported="true" android:permission="com.google.android.c2dm.permission.SEND">
    <intent-filter>
        <action android:name="com.google.android.c2dm.intent.RECEIVE" />
        <action android:name="com.google.android.c2dm.intent.REGISTRATION" />
        <category android:name="${applicationId}" />
    </intent-filter>
</receiver>