My complete answer can be found here as a gist
I found the answer to determining the drive letter from the serial # from this question/answer
How to get the drive letter of USB device using WMI
And I modified Phil Minor's code to make it reactive:
   public class UsbDetector : IUsbDetector
    {
        private const string Query = "SELECT * FROM {0} WITHIN 2 WHERE TargetInstance ISA 'Win32_USBHub'";
        private const string CreationEvent = "__InstanceCreationEvent";
        private const string DeletionEvent = "__InstanceDeletionEvent";
        private const int ReplayNumber = 1;
        private readonly Subject<USBDeviceInfo> adds = new Subject<USBDeviceInfo>();
        private readonly Subject<USBDeviceInfo> removes = new Subject<USBDeviceInfo>();
        public UsbDetector()
        {
            var bgwDriveDetector = new BackgroundWorker();
            bgwDriveDetector.DoWork += DoWork;
            bgwDriveDetector.RunWorkerAsync();
        }
        public IObservable<USBDeviceInfo> Adds => adds.AsObservable();
        public IObservable<USBDeviceInfo> Removes => removes.AsObservable();
        private void DoWork(object sender, DoWorkEventArgs e)
        {
            SubscribeToEvent(CreationEvent, adds);
            SubscribeToEvent(DeletionEvent, removes);
        }
        private static void SubscribeToEvent(string eventType, IObserver<USBDeviceInfo> observer)
        {
            WqlEventQuery wqlEventQuery = new WqlEventQuery(string.Format(Query, eventType));
            ManagementEventWatcher insertWatcher = new ManagementEventWatcher(wqlEventQuery);
            var observable = Observable.FromEventPattern<EventArrivedEventHandler, EventArrivedEventArgs>(
                h => insertWatcher.EventArrived += h,
                h => insertWatcher.EventArrived -= h).Replay(ReplayNumber);
            observable.Connect();
            observable.Select(a => a.EventArgs).Select(MapEventArgs).Subscribe(observer);
            insertWatcher.Start();
        }
        private static USBDeviceInfo MapEventArgs(EventArrivedEventArgs e)
        {
            ManagementBaseObject instance = (ManagementBaseObject)e.NewEvent["TargetInstance"];
            string deviceId = (string)instance.GetPropertyValue("DeviceID");
            string serialNr = deviceId.Substring(deviceId.LastIndexOf('\\')).Replace("\\", "");
            char driveLetter = GetDriveLetter(serialNr).First();
            return new USBDeviceInfo(deviceId, serialNr, driveLetter);
        }