React web3 contract events removing listener is not possible

Question

I am trying to create a custom hook which listens to my web3 contract events and unbinds when I change my account or network. Nonetheless it executes twice when I switch my account back and forth. Except this it works fine but for any reasons it does not remove the listener.

I have also tried removing the listener before I add it but still it will only be added and so on execute more then once.

const EVENT_OPTIONS = {
    fromBlock: "latest",
};

export const useEvent = (event, handler) => {
    useEffect(() => {
        if (!event) return;
        event(EVENT_OPTIONS).addListener("data", handler);

        return () => {
            //This gets executed but the event will not be removed
            // (Executed when the metamask network changes)
            event(EVENT_OPTIONS).removeListener("data", handler);
        }
    }, [event])
}

// ----- Example for useEvent:

export const useOnNewRound = (callback) => {
    const contract = useContractWSS(); // returns websocket contract
    
    const sortData = useCallback((data) => {
        // do something with data...
        callback(data)
    }, [])
     
    useEvent(contract?.events.NewRound, sortData); //Pass event and handler function
}