I'm guessing you've seen answers such as this one, since you mention not removing event listeners on child elements. Of course doing a deep clone won't work, but what can work is to do a shallow clone, and then transfer the child nodes to the clone. It's a little more work, but it will do what you want.
The main work is in the setTimeout, all the rest is really just setup.
Here I'm attaching two click event listeners: one to the button and one to the container div. before the 5 second setTimeout, clicking the button will fire both listeners (so you'll see console logs for both BUTTON and DIV). After the timer, you'll still see the BUTTON event.
var rem = document.getElementById('rem');
var btn = document.getElementById('btn');
function onClick(e) {
  var t = e.currentTarget
  console.log(t.tagName);
}
rem.addEventListener('click', onClick);
btn.addEventListener('click', onClick);
setTimeout(() => {
  console.log('removing events from div');
  // shallow clone
  var rem2 = rem.cloneNode(false);
  // make and extract range to doc fragment 
  // for all contents of original div
  var range = document.createRange(); 
  range.setStartBefore(rem.firstChild);
  range.setEndAfter(rem.lastChild);
  var documentFragment = range.extractContents();
  // append doc fragment to clone
  // insert clone and remove old div
  rem2.appendChild(documentFragment);
  document.body.insertBefore(rem2, rem);
  rem.remove();
}, 5000);
<div id='rem'>
  <h3>Heading</h3>
  <button id='btn'>click me</button>
  <p>paragraph <b>bold</b> stuff</p>
</div>