I looked around documentation and other sources about this issue and found nothing. There are only a few solutions, based on using UIKit and UIViewControllerRepresentable. I tried to combine solutions from this question and I saved swipe back gesture even while replacing back button with other view. The code is still dirty a little, but I think that is the start point to go further (totally hide navigation bar, for example). So, here is how ContentView looks like:
import SwiftUI
struct ContentView: View {
    var body: some View {
        SwipeBackNavController {
            SwipeBackNavigationLink(destination: DetailViewWithCustomBackButton()) {
                Text("Main view")
            }
            .navigationBarTitle("Standard SwiftUI nav view")
        }
        .edgesIgnoringSafeArea(.top)
    }
}
// MARK: detail view with custom back button
struct DetailViewWithCustomBackButton: View {
    @Environment(\.presentationMode) var presentationMode
    var body: some View {
        Text("detail")
            .navigationBarItems(leading: Button(action: {
                self.dismissView()
            }) {
                HStack {
                    Image(systemName: "return")
                    Text("Back")
                }
            })
        .navigationBarTitle("Detailed view")
    }
    private func dismissView() {
        presentationMode.wrappedValue.dismiss()
    }
}
Here is realization of SwipeBackNavController and SwipeBackNavigationLink which mimic NavigationView and NavigationLink. They are just wrappers for SwipeNavigationController's work. The last one is a subclass of UINavigationController, which can be customized for your needs:
import UIKit
import SwiftUI
struct SwipeBackNavController<Content: View>: UIViewControllerRepresentable {
    let content: Content
    public init(@ViewBuilder content: @escaping () -> Content) {
        self.content = content()
    }
    func makeUIViewController(context: Context) -> SwipeNavigationController {
        let hostingController = UIHostingController(rootView: content)
        let swipeBackNavController = SwipeNavigationController(rootViewController: hostingController)
        return swipeBackNavController
    }
    func updateUIViewController(_ pageViewController: SwipeNavigationController, context: Context) {
    }
}
struct SwipeBackNavigationLink<Destination: View, Label:View>: View {
    var destination: Destination
    var label: () -> Label
    public init(destination: Destination, @ViewBuilder label: @escaping () -> Label) {
        self.destination = destination
        self.label = label
    }
    var body: some View {
        Button(action: {
            guard let window = UIApplication.shared.windows.first else { return }
            guard let swipeBackNavController = window.rootViewController?.children.first as? SwipeNavigationController else { return }
            swipeBackNavController.pushSwipeBackView(DetailViewWithCustomBackButton())
        }, label: label)
    }
}
final class SwipeNavigationController: UINavigationController {
    // MARK: - Lifecycle
    override init(rootViewController: UIViewController) {
        super.init(rootViewController: rootViewController)
    }
    override init(nibName nibNameOrNil: String?, bundle nibBundleOrNil: Bundle?) {
        super.init(nibName: nibNameOrNil, bundle: nibBundleOrNil)
        delegate = self
    }
    required init?(coder aDecoder: NSCoder) {
        super.init(coder: aDecoder)
        delegate = self
    }
    override func viewDidLoad() {
        super.viewDidLoad()
        // This needs to be in here, not in init
        interactivePopGestureRecognizer?.delegate = self
    }
    deinit {
        delegate = nil
        interactivePopGestureRecognizer?.delegate = nil
    }
    // MARK: - Overrides
    override func pushViewController(_ viewController: UIViewController, animated: Bool) {
        duringPushAnimation = true
        setNavigationBarHidden(true, animated: false)
        super.pushViewController(viewController, animated: animated)
    }
    var duringPushAnimation = false
    // MARK: - Custom Functions
    func pushSwipeBackView<Content>(_ content: Content) where Content: View {
        let hostingController = SwipeBackHostingController(rootView: content)
        self.delegate = hostingController
        self.pushViewController(hostingController, animated: true)
    }
}
// MARK: - UINavigationControllerDelegate
extension SwipeNavigationController: UINavigationControllerDelegate {
    func navigationController(_ navigationController: UINavigationController, didShow viewController: UIViewController, animated: Bool) {
        guard let swipeNavigationController = navigationController as? SwipeNavigationController else { return }
        swipeNavigationController.duringPushAnimation = false
    }
}
// MARK: - UIGestureRecognizerDelegate
extension SwipeNavigationController: UIGestureRecognizerDelegate {
    func gestureRecognizerShouldBegin(_ gestureRecognizer: UIGestureRecognizer) -> Bool {
        guard gestureRecognizer == interactivePopGestureRecognizer else {
            return true // default value
        }
        // Disable pop gesture in two situations:
        // 1) when the pop animation is in progress
        // 2) when user swipes quickly a couple of times and animations don't have time to be performed
        let result = viewControllers.count > 1 && duringPushAnimation == false
        return result
    }
}
// MARK: Hosting controller
class SwipeBackHostingController<Content: View>: UIHostingController<Content>, UINavigationControllerDelegate {
    func navigationController(_ navigationController: UINavigationController, didShow viewController: UIViewController, animated: Bool) {
        guard let swipeNavigationController = navigationController as? SwipeNavigationController else { return }
        swipeNavigationController.duringPushAnimation = false
    }
    override func viewWillDisappear(_ animated: Bool) {
        super.viewWillDisappear(animated)
        guard let swipeNavigationController = navigationController as? SwipeNavigationController else { return }
        swipeNavigationController.delegate = nil
    }
}
This realization provides to save custom back button and swipe back gesture for now. I still don't like some moments, like how SwipeBackNavigationLink pushes view, so later I'll try to continue research.