Yeah... something really weird going on there.
My guess would be that UIKit has some sort of algorithm that is trying to apply "readability" to the buttons. Clearly, though, that's not the goal here.
After some quick testing, one way to get around this is to add the button as a subview to a clear "container" view, and then add that container view to the stack view.
Some sample code:
class SampleViewController: UIViewController {
    
    let filterStackViewA: UIStackView = {
        let v = UIStackView()
        v.spacing = 8
        v.distribution = .fill
        v.alignment = .center
        return v
    }()
    
    let filterStackViewB: UIStackView = {
        let v = UIStackView()
        v.spacing = 8
        v.distribution = .fill
        v.alignment = .center
        return v
    }()
    
    let scrollViewA: UIScrollView = {
        let v = UIScrollView()
        v.backgroundColor = .systemYellow
        return v
    }()
    
    let scrollViewB: UIScrollView = {
        let v = UIScrollView()
        v.backgroundColor = .systemOrange
        return v
    }()
    
    override func viewDidLoad() {
        super.viewDidLoad()
        
        view.backgroundColor = .darkGray
        let addBtn = UIButton()
        addBtn.setTitle("Add Button", for: [])
        addBtn.setTitleColor(.white, for: .normal)
        addBtn.setTitleColor(.lightGray, for: .highlighted)
        addBtn.backgroundColor = .systemGreen
        addBtn.layer.cornerRadius = 8
        let labelA: UILabel = {
            let v = UILabel()
            v.textColor = .white
            v.font = .systemFont(ofSize: 15.0, weight: .light)
            v.text = "Buttons added to \"Container\" views..."
            return v
        }()
        
        let labelB: UILabel = {
            let v = UILabel()
            v.textColor = .white
            v.font = .systemFont(ofSize: 15.0, weight: .light)
            v.text = "Buttons added directly to stack view..."
            return v
        }()
        
        [addBtn, scrollViewA, scrollViewB, filterStackViewA, filterStackViewB, labelA, labelB].forEach { v in
            v.translatesAutoresizingMaskIntoConstraints = false
        }
        view.addSubview(addBtn)
        
        view.addSubview(labelA)
        
        view.addSubview(scrollViewA)
        scrollViewA.addSubview(filterStackViewA)
        view.addSubview(labelB)
        
        view.addSubview(scrollViewB)
        scrollViewB.addSubview(filterStackViewB)
        
        let g = view.safeAreaLayoutGuide
        
        let cgA = scrollViewA.contentLayoutGuide
        let fgA = scrollViewA.frameLayoutGuide
        
        let cgB = scrollViewB.contentLayoutGuide
        let fgB = scrollViewB.frameLayoutGuide
        
        NSLayoutConstraint.activate([
            
            addBtn.topAnchor.constraint(equalTo: g.topAnchor, constant: 40.0),
            addBtn.widthAnchor.constraint(equalToConstant: 200.0),
            addBtn.centerXAnchor.constraint(equalTo: g.centerXAnchor),
            labelA.topAnchor.constraint(equalTo: addBtn.bottomAnchor, constant: 40.0),
            labelA.leadingAnchor.constraint(equalTo: g.leadingAnchor, constant: 20.0),
            labelA.trailingAnchor.constraint(equalTo: g.trailingAnchor, constant: -20.0),
            scrollViewA.topAnchor.constraint(equalTo: labelA.bottomAnchor, constant: 8.0),
            scrollViewA.leadingAnchor.constraint(equalTo: g.leadingAnchor, constant: 20.0),
            scrollViewA.trailingAnchor.constraint(equalTo: g.trailingAnchor, constant: -20.0),
            scrollViewA.heightAnchor.constraint(equalToConstant: 60.0),
            filterStackViewA.topAnchor.constraint(equalTo: cgA.topAnchor, constant: 4.0),
            filterStackViewA.leadingAnchor.constraint(equalTo: cgA.leadingAnchor, constant: 8.0),
            filterStackViewA.trailingAnchor.constraint(equalTo: cgA.trailingAnchor, constant: -8.0),
            filterStackViewA.bottomAnchor.constraint(equalTo: cgA.bottomAnchor, constant: -4.0),
            filterStackViewA.heightAnchor.constraint(equalTo: fgA.heightAnchor, constant: -8.0),
            
            labelB.topAnchor.constraint(equalTo: filterStackViewA.bottomAnchor, constant: 20.0),
            labelB.leadingAnchor.constraint(equalTo: g.leadingAnchor, constant: 20.0),
            labelB.trailingAnchor.constraint(equalTo: g.trailingAnchor, constant: -20.0),
            
            scrollViewB.topAnchor.constraint(equalTo: labelB.bottomAnchor, constant: 8.0),
            scrollViewB.leadingAnchor.constraint(equalTo: g.leadingAnchor, constant: 20.0),
            scrollViewB.trailingAnchor.constraint(equalTo: g.trailingAnchor, constant: -20.0),
            scrollViewB.heightAnchor.constraint(equalToConstant: 60.0),
            
            filterStackViewB.topAnchor.constraint(equalTo: cgB.topAnchor, constant: 4.0),
            filterStackViewB.leadingAnchor.constraint(equalTo: cgB.leadingAnchor, constant: 8.0),
            filterStackViewB.trailingAnchor.constraint(equalTo: cgB.trailingAnchor, constant: -8.0),
            filterStackViewB.bottomAnchor.constraint(equalTo: cgB.bottomAnchor, constant: -4.0),
            
            filterStackViewB.heightAnchor.constraint(equalTo: fgB.heightAnchor, constant: -8.0),
            
        ])
        
        addBtn.addTarget(self, action: #selector(addButton), for: .touchUpInside)
    }
    
    @objc func addButton() {
        // unwrap optional system image
        guard let img = UIImage(systemName: "xmark.circle", withConfiguration: UIImage.SymbolConfiguration(scale: .unspecified))
        else { return }
        
        var newButtonConfiguration = UIButton.Configuration.filled()
        newButtonConfiguration.title = randomString(length: Int.random(in: 5..<15))
        newButtonConfiguration.image = img
        // let's add a new button to a "container" view
        let newButtonA = UIButton(configuration: newButtonConfiguration)
        newButtonA.translatesAutoresizingMaskIntoConstraints = false
        let v = UIView()
        v.addSubview(newButtonA)
        NSLayoutConstraint.activate([
            newButtonA.topAnchor.constraint(equalTo: v.topAnchor),
            newButtonA.leadingAnchor.constraint(equalTo: v.leadingAnchor),
            newButtonA.trailingAnchor.constraint(equalTo: v.trailingAnchor),
            newButtonA.bottomAnchor.constraint(equalTo: v.bottomAnchor),
        ])
        
        // add the container view to the stack view
        filterStackViewA.addArrangedSubview(v)
        
        // let's add a new button directly to the stack view
        let newButtonB = UIButton(configuration: newButtonConfiguration)
        filterStackViewB.addArrangedSubview(newButtonB)
        newButtonA.addTarget(self, action: #selector(removeMe(_:)), for: .touchUpInside)
        newButtonB.addTarget(self, action: #selector(removeMe(_:)), for: .touchUpInside)
        // let's make sure the new button is visible
        //  note: this is just for example...
        //  if we rapidly tap and add buttons, this can easily "miss" the last button
        //  so don't expect this to be "production" code
        DispatchQueue.main.async {
            let sz = self.scrollViewA.contentSize
            let rA = CGRect(x: sz.width - 1.0, y: 0.0, width: 1.0, height: 1.0)
            self.scrollViewA.scrollRectToVisible(rA, animated: true)
            let szB = self.scrollViewB.contentSize
            let rB = CGRect(x: szB.width - 1.0, y: 0.0, width: 1.0, height: 1.0)
            self.scrollViewB.scrollRectToVisible(rB, animated: true)
        }
        
    }
    
    @objc func removeMe(_ sender: UIButton) {
        // get a reference to the tapped button's superview
        guard let sv = sender.superview else { return }
        
        var v: UIView!
        if sv is UIStackView {
            // we tapped a button added directly to the stack view
            v = sender
        } else {
            // we tapped a button that's in a "container" view
            v = sv
        }
        guard let st = v.superview as? UIStackView,
              let idx = st.arrangedSubviews.firstIndex(of: v),
              filterStackViewA.arrangedSubviews.count > idx,
              filterStackViewB.arrangedSubviews.count > idx
        else {
            print("something's not setup right, so return")
            return
        }
        
        let bA = filterStackViewA.arrangedSubviews[idx]
        let bB = filterStackViewB.arrangedSubviews[idx]
        // let's animate the buttons away
        bA.isHidden = true
        bB.isHidden = true
        UIView.animate(withDuration: 0.5, animations: {
            self.view.layoutIfNeeded()
        }, completion: { _ in
            bA.removeFromSuperview()
            bB.removeFromSuperview()
        })
    }
    
    func randomString(length: Int) -> String {
        let letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
        return String((0..<length).map{ _ in letters.randomElement()! })
    }
}
It will look like this when running:

Each tap of "Add Button" will add a new button to each scrolling-stackView, with the top set using buttons in container views. Tapping any of those buttons will remove it from its stack view, along with its "twin" from the other one.