My problem was having a view that a user could draw in but the scroll view was hijacking the touch and scrolling instead of drawing. This is my working solution:
In my drawing view I have:
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
    // Disable scroll on touch begin
    setContainerScrollingEnabled(false)
    super.touchesBegan(touches, with: event)
}
override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) {
    // Reenable scroll on touch end
    setContainerScrollingEnabled(true)
    super.touchesEnded(touches, with: event)
}
private func setContainerScrollingEnabled(_ enabled: Bool) {
    // loop up superviews to find the scrollview
    func scrollingSuperview(of view: UIView) -> UIScrollView? {
        guard let superview = view.superview else { return nil }
        if let scrolling = view.superview as? UIScrollView { return scrolling }
        return scrollingSuperview(of: superview)
    }
    // and switch on/off scrolling
    let scrollView = scrollingSuperview(of: self)
    scrollView?.isScrollEnabled = enabled
}
and in the view controller containing the scroll view I have:
// Not sure what exactly this does, but I think it allows the drawing view to have precedence. This was the missing piece of the puzzle.
scrollView.delaysContentTouches = false