How to get a canvas relative mouse position of a CSS 3D transformed canvas?

Question

Just for fun I'm trying to draw on 3D transformed canvases. I wrote some code and it kind of works

const m4 = twgl.m4;

[...document.querySelectorAll('canvas')].forEach((canvas) => {
  const ctx = canvas.getContext('2d');
  let count = 0;

  canvas.addEventListener('mousemove', (e) => {
    const pos = getElementRelativeMousePosition(e, canvas);
    ctx.fillStyle = hsl((count++ % 10) / 10, 1, 0.5);
    ctx.fillRect(pos.x - 1, pos.y - 1, 3, 3);
  });
});

function getElementRelativeMousePosition(e, elem) {
  const pos = convertPointFromPageToNode(elem, e.pageX, e.pageY); 
  
  return {
    x: pos[0],
    y: pos[1],
  };
}

function hsl(h, s, l) {
  return `hsl(${h * 360 | 0},${s * 100 | 0}%,${l * 100 | 0}%)`;
}

function convertPointFromPageToNode(elem, pageX, pageY) {
  const mat = m4.inverse(getTransformationMatrix(elem));
  return m4.transformPoint(mat, [pageX, pageY, 0]);
};

function getTransformationMatrix(elem) {
  let matrix = m4.identity();
  let currentElem = elem;

  while (currentElem !== undefined && 
         currentElem !== currentElem.ownerDocument.documentElement) {
    const style = window.getComputedStyle(currentElem);
    const localMatrix = parseMatrix(style.transform);
    matrix = m4.multiply(localMatrix, matrix);
    currentElem = currentElem.parentElement;
  }

  const w = elem.offsetWidth;
  const h = elem.offsetHeight;
  let i = 4;
  let left = +Infinity;
  let top = +Infinity;
  for (let i = 0; i < 4; ++i) {
    const p = m4.transformPoint(matrix, [w * (i & 1), h * ((i & 2) >> 1), 0]);
    left = Math.min(p[0], left);
    top = Math.min(p[1], top);
  }
  const rect = elem.getBoundingClientRect()
  document.querySelector('p').textContent =
    `${w}x${h}`;
  matrix =  m4.multiply(m4.translation([
     window.pageXOffset + rect.left - left, 
     window.pageYOffset + rect.top - top,
     0]), matrix);
  return matrix;
}


function parseMatrix(str) {
  if (str.startsWith('matrix3d(')) {
    return str.substring(9, str.length - 1).split(',').map(v => parseFloat(v.trim()));
  } else if (str.startsWith('matrix(')) {
    const m = str.substring(7, str.length - 1).split(',').map(v => parseFloat(v.trim()));
    return [
      m[0], m[1], 0, 0,
      m[2], m[3], 0, 0,
      0, 0, 1, 0,
      m[4], m[5], 0, 1,
    ]
  } else if (str == 'none') {
    return m4.identity();
  }
  throw new Error('unknown format');
}

canvas { 
  display: block;
  background: yellow;
  transform: scale(0.75);
}
#c1 {
  margin: 20px;
  background: red;
  transform: translateX(-50px);
  display: inline-block;
}
#c2 {
  margin: 20px;
  background: green;
  transform: rotate(45deg);
  display: inline-block;
}
#c3 {
  margin: 20px;
  background: blue;
  display: inline-block;
}

#c4 {
  position: absolute;
  top: 0;
  background: cyan;
  transform: translateX(-250px) rotate(55deg);
  display: inline-block;
}
#c5 {
  background: magenta;
  transform: translate(50px);
  display: inline-block;
}
#c6 {
  background: pink;
  transform: rotate(45deg);
  display: inline-block;
}

<p>
foo
</p>
<div id="c1">
  <div id="c2">
    <div id="c3">
      <canvas></canvas>
    </div>
  </div>
</div>
<div id="c4">
  <div id="c5">
    <div id="c6">
      <canvas></canvas>
    </div>
  </div>
</div>
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>

The code above works. Move the mouse over either of the yellow canvas elements and you'll see it draws correctly.

But, as soon as I add some 3D transform it fails.

Change the CSS for '#c6' to

    #c6 {
      background: pink;
      transform: rotate(45deg) rotateX(45deg);  /* changed */
      display: inline-block;
    }

and now when I draw over the right yellow canvas things are off.

const m4 = twgl.m4;

[...document.querySelectorAll('canvas')].forEach((canvas) => {
  const ctx = canvas.getContext('2d');
  let count = 0;

  canvas.addEventListener('mousemove', (e) => {
    const pos = getElementRelativeMousePosition(e, canvas);
    ctx.fillStyle = hsl((count++ % 10) / 10, 1, 0.5);
    ctx.fillRect(pos.x - 1, pos.y - 1, 3, 3);
  });
});

function getElementRelativeMousePosition(e, elem) {
  const pos = convertPointFromPageToNode(elem, e.pageX, e.pageY); 
  
  return {
    x: pos[0],
    y: pos[1],
  };
}

function hsl(h, s, l) {
  return `hsl(${h * 360 | 0},${s * 100 | 0}%,${l * 100 | 0}%)`;
}

function convertPointFromPageToNode(elem, pageX, pageY) {
  const mat = m4.inverse(getTransformationMatrix(elem));
  return m4.transformPoint(mat, [pageX, pageY, 0]);
};

function getTransformationMatrix(elem) {
  let matrix = m4.identity();
  let currentElem = elem;

  while (currentElem !== undefined && 
         currentElem !== currentElem.ownerDocument.documentElement) {
    const style = window.getComputedStyle(currentElem);
    const localMatrix = parseMatrix(style.transform);
    matrix = m4.multiply(localMatrix, matrix);
    currentElem = currentElem.parentElement;
  }

  const w = elem.offsetWidth;
  const h = elem.offsetHeight;
  let i = 4;
  let left = +Infinity;
  let top = +Infinity;
  for (let i = 0; i < 4; ++i) {
    const p = m4.transformPoint(matrix, [w * (i & 1), h * ((i & 2) >> 1), 0]);
    left = Math.min(p[0], left);
    top = Math.min(p[1], top);
  }
  const rect = elem.getBoundingClientRect()
  document.querySelector('p').textContent =
    `${w}x${h}`;
  matrix =  m4.multiply(m4.translation([
     window.pageXOffset + rect.left - left, 
     window.pageYOffset + rect.top - top,
     0]), matrix);
  return matrix;
}


function parseMatrix(str) {
  if (str.startsWith('matrix3d(')) {
    return str.substring(9, str.length - 1).split(',').map(v => parseFloat(v.trim()));
  } else if (str.startsWith('matrix(')) {
    const m = str.substring(7, str.length - 1).split(',').map(v => parseFloat(v.trim()));
    return [
      m[0], m[1], 0, 0,
      m[2], m[3], 0, 0,
      0, 0, 1, 0,
      m[4], m[5], 0, 1,
    ]
  } else if (str == 'none') {
    return m4.identity();
  }
  throw new Error('unknown format');
}

canvas { 
  display: block;
  background: yellow;
  transform: scale(0.75);
}
#c1 {
  margin: 20px;
  background: red;
  transform: translateX(-50px);
  display: inline-block;
}
#c2 {
  margin: 20px;
  background: green;
  transform: rotate(45deg);
  display: inline-block;
}
#c3 {
  margin: 20px;
  background: blue;
  display: inline-block;
}

#c4 {
  position: absolute;
  top: 0;
  background: cyan;
  transform: translateX(-250px) rotate(55deg);
  display: inline-block;
}
#c5 {
  background: magenta;
  transform: translate(50px);
  display: inline-block;
}
#c6 {
  background: pink;
  transform: rotate(45deg) rotateX(45deg);
  display: inline-block;
}

<p>
foo
</p>
<div id="c1">
  <div id="c2">
    <div id="c3">
      <canvas></canvas>
    </div>
  </div>
</div>
<div id="c4">
  <div id="c5">
    <div id="c6">
      <canvas></canvas>
    </div>
  </div>
</div>
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>

Any ideas what I'm doing wrong?

Answer 1

Note: That's just a complementary answer on what OP already did find by themselves.

You can actually make this all work by using the MouseEvent constructor.

You can pass the clientX and clientY properties of this Event inside the constructor (or pageX & pageY if you prefer), then dispatching this composed event to your target will set its offsetX and offsetY properties relative to the target.

Since dispatchEvent does fire the Event synchronously, we can even make a converter:

const init_pos = { x: 50, y: 50};
const relative_pos = {};
const canvas = document.querySelector('canvas');

canvas.addEventListener('mousemove', e => {
  relative_pos.x = e.offsetX;
  relative_pos.y = e.offsetY;
}, {once: true});

canvas.dispatchEvent(new MouseEvent('mousemove', {
  clientX: init_pos.x,
  clientY: init_pos.y
}));
// synchronously log
console.log(relative_pos);

canvas { 
  display: block;
  background: yellow;
  transform: scale(0.75);
}
#c4 {
  position: absolute;
  top: 0;
  background: cyan;
  transform: translateX(-250px) rotate(55deg);
  display: inline-block;
}
#c5 {
  background: magenta;
  transform: translate(50px);
  display: inline-block;
}
#c6 {
  background: pink;
  transform: rotate(45deg);
  display: inline-block;
}

<div id="c4">
  <div id="c5">
    <div id="c6">
      <canvas></canvas>
    </div>
  </div>
</div>

Now, given the example in your own answer, you may want to actually hold a single object that will keep the global Event's position, and to get the relative positions of your canvas at every frame in a requestAnimationFrame loop.
However, this setup will obviously traverse your canvases, if you want only the visible face to handle the events, then you'd have to check which one matches document.elementFromPoint(x, y), which itself needs your elements to react to pointer-events.

// will hold our last event's position
const pos = {
  x: 0,
  y: 0
};
const canvases = document.querySelectorAll('canvas');
// A single global "real" MouseEvent handler
document.body.onmousemove = (e) => {
  pos.x = e.clientX;
  pos.y = e.clientY;
};

canvases.forEach(canvas => {
  const ctx = canvas.getContext('2d');
  let count = 0;
  canvas.addEventListener('mousemove', draw);
  function draw(e) {
    // do not fire on real Events
    if (e.cancelable) return;
    
    const x = e.offsetX * canvas.width / canvas.clientWidth;
    const y = e.offsetY * canvas.height / canvas.clientHeight;
    
    if (x < 0 || x > canvas.width || y < 0 || y > canvas.height) {
      return;
    }
    
    ctx.fillStyle = hsl((count++ % 10) / 10, 1, 0.5);
    ctx.fillRect(x - 1, y - 1, 3, 3);
  }
});

anim();

function anim() {
  requestAnimationFrame(anim);

  // in case we want to paint only on the front element
  const front_elem = single_face.checked && document.elementFromPoint(pos.x, pos.y);

  // at every frame
  canvases.forEach(c => {
    if (!front_elem || c === front_elem) {
      // force a composed event (synchronously, so we are still in rAF callback)
      c.dispatchEvent(
        new MouseEvent('mousemove', {
          clientX: pos.x,
          clientY: pos.y
        })
      );
    }
  });
}

function hsl(h, s, l) {
  return `hsl(${h * 360 | 0},${s * 100 | 0}%,${l * 100 | 0}%)`;
}

.scene {
  width: 200px;
  height: 200px;
  perspective: 600px;
}

.cube {
  width: 100%;
  height: 100%;
  position: relative;
  transform-style: preserve-3d;
  animation-duration: 16s;
  animation-name: rotate;
  animation-iteration-count: infinite;
  animation-timing-function: linear;
  pointer-events: none; /* no need for mouse events */
}

#single_face:checked+.scene .cube {
  pointer-events: all; /* except if we want to find out who is the front one */
}
label,#single_face {float: right}
@keyframes rotate {
  from {
    transform: translateZ(-100px) rotateX( 0deg) rotateY( 0deg);
  }
  to {
    transform: translateZ(-100px) rotateX(360deg) rotateY(720deg);
  }
}

.cube__face {
  position: absolute;
  width: 200px;
  height: 200px;
  display: block;
}

.cube__face--front {
  background: rgba(255, 0, 0, 0.2);
  transform: rotateY( 0deg) translateZ(100px);
}

.cube__face--right {
  background: rgba(0, 255, 0, 0.2);
  transform: rotateY( 90deg) translateZ(100px);
}

.cube__face--back {
  background: rgba(0, 0, 255, 0.2);
  transform: rotateY(180deg) translateZ(100px);
}

.cube__face--left {
  background: rgba(255, 255, 0, 0.2);
  transform: rotateY(-90deg) translateZ(100px);
}

.cube__face--top {
  background: rgba(0, 255, 255, 0.2);
  transform: rotateX( 90deg) translateZ(100px);
}

.cube__face--bottom {
  background: rgba(255, 0, 255, 0.2);
  transform: rotateX(-90deg) translateZ(100px);
}

<label>Draw on a single face</label><input type="checkbox" id="single_face">
<div class="scene">
  <div class="cube">
    <canvas class="cube__face cube__face--front"></canvas>
    <canvas class="cube__face cube__face--back"></canvas>
    <canvas class="cube__face cube__face--right"></canvas>
    <canvas class="cube__face cube__face--left"></canvas>
    <canvas class="cube__face cube__face--top"></canvas>
    <canvas class="cube__face cube__face--bottom"></canvas>
  </div>
</div>
<pre id="debug"></pre>

Answer 2

Sigh ... not the definitive answer yet but apparently event.offsetX and event.offsetY are supposed to be this value even though according to MDN they are not standard yet.

Testing it seems to work in both Chrome and Firefox. Safari is off though in some tests. Also unfortunately offsetX and offsetY do not exist on touch events. They do exist on pointer events but pointer events are not supported by Safari as of 2019/05

[...document.querySelectorAll('canvas')].forEach((canvas) => {
  const ctx = canvas.getContext('2d');
  let count = 0;

  canvas.addEventListener('mousemove', (e) => {
    const pos = {
      x: e.offsetX * ctx.canvas.width / ctx.canvas.clientWidth,
      y: e.offsetY * ctx.canvas.height / ctx.canvas.clientHeight,
    };
    ctx.fillStyle = hsl((count++ % 10) / 10, 1, 0.5);
    ctx.fillRect(pos.x - 1, pos.y - 1, 3, 3);
  });
});

function hsl(h, s, l) {
  return `hsl(${h * 360 | 0},${s * 100 | 0}%,${l * 100 | 0}%)`;
}

canvas { 
  display: block;
  background: yellow;
  transform: scale(0.75);
}
#c1 {
  margin: 20px;
  background: red;
  transform: translateX(-50px);
  display: inline-block;
}
#c2 {
  margin: 20px;
  background: green;
  transform: rotate(45deg);
  display: inline-block;
}
#c3 {
  margin: 20px;
  background: blue;
  display: inline-block;
}

#c4 {
  position: absolute;
  top: 0;
  background: cyan;
  transform: translateX(-250px) rotate(55deg);
  display: inline-block;
}
#c5 {
  background: magenta;
  transform: translate(50px);
  display: inline-block;
}
#c6 {
  background: pink;
  transform: rotate(45deg) rotateX(45deg);  /* changed */
  display: inline-block;
}

<p>
foo
</p>
<div id="c1">
  <div id="c2">
    <div id="c3">
      <canvas></canvas>
    </div>
  </div>
</div>
<div id="c4">
  <div id="c5">
    <div id="c6">
      <canvas></canvas>
    </div>
  </div>
</div>

Unfortunately we still have the issue that sometimes we want a canvas relative position outside of an event. In the example below we'd like to keep drawing under the mouse pointer even when the pointer is not moving.

[...document.querySelectorAll('canvas')].forEach((canvas) => {
  const ctx = canvas.getContext('2d');
  ctx.canvas.width  = ctx.canvas.clientWidth;
  ctx.canvas.height = ctx.canvas.clientHeight;
  let count = 0;

  function draw(e, radius = 1) {
    const pos = {
      x: e.offsetX * ctx.canvas.width / ctx.canvas.clientWidth,
      y: e.offsetY * ctx.canvas.height / ctx.canvas.clientHeight,
    };
    document.querySelector('#debug').textContent = count;
    ctx.beginPath();
    ctx.arc(pos.x, pos.y, radius, 0, Math.PI * 2);
    ctx.fillStyle = hsl((count++ % 100) / 100, 1, 0.5);
    ctx.fill();
  }

  function preventDefault(e) {
    e.preventDefault();
  }

  if (window.PointerEvent) {
    canvas.addEventListener('pointermove', (e) => {
      draw(e, Math.max(Math.max(e.width, e.height) / 2, 1));
    });
    canvas.addEventListener('touchstart', preventDefault, {passive: false});
    canvas.addEventListener('touchmove', preventDefault, {passive: false});
  } else {
    canvas.addEventListener('mousemove', draw);
    canvas.addEventListener('mousedown', preventDefault);
  }
});

function hsl(h, s, l) {
  return `hsl(${h * 360 | 0},${s * 100 | 0}%,${l * 100 | 0}%)`;
}

.scene {
  width: 200px;
  height: 200px;
  perspective: 600px;
}

.cube {
  width: 100%;
  height: 100%;
  position: relative;
  transform-style: preserve-3d;
  animation-duration: 16s;
  animation-name: rotate;
  animation-iteration-count: infinite;
  animation-timing-function: linear;
}

@keyframes rotate {
  from { transform: translateZ(-100px) rotateX(  0deg) rotateY(  0deg); }
  to   { transform: translateZ(-100px) rotateX(360deg) rotateY(720deg); }
}

.cube__face {
  position: absolute;
  width: 200px;
  height: 200px;
  display: block;
}

.cube__face--front  { background: rgba(255, 0, 0, 0.2); transform: rotateY(  0deg) translateZ(100px); }
.cube__face--right  { background: rgba(0, 255, 0, 0.2); transform: rotateY( 90deg) translateZ(100px); }
.cube__face--back   { background: rgba(0, 0, 255, 0.2); transform: rotateY(180deg) translateZ(100px); }
.cube__face--left   { background: rgba(255, 255, 0, 0.2); transform: rotateY(-90deg) translateZ(100px); }
.cube__face--top    { background: rgba(0, 255, 255, 0.2); transform: rotateX( 90deg) translateZ(100px); }
.cube__face--bottom { background: rgba(255, 0, 255, 0.2); transform: rotateX(-90deg) translateZ(100px); }

<div class="scene">
  <div class="cube">
    <canvas class="cube__face cube__face--front"></canvas>
    <canvas class="cube__face cube__face--back"></canvas>
    <canvas class="cube__face cube__face--right"></canvas>
    <canvas class="cube__face cube__face--left"></canvas>
    <canvas class="cube__face cube__face--top"></canvas>
    <canvas class="cube__face cube__face--bottom"></canvas>
  </div>
</div>
<pre id="debug"></pre>