The x and y properties exist on the calling context, the num object - either reference num.x, or when calling the function, use fn.call(num) so that the this inside the function refers to the num object:
var num = {
  x: 12,
  y: 3,
  calculate: function(operation) {
    var fn;
    switch (operation) {
      case '+':
        fn = function() {
          return this.x + this.y
        };
        break;
      case '-':
        fn = function() {
          return this.x - this.y
        };
        break;
      default:
        fn = function() {};
    }
    return fn.call(num);
  }
}
risult = num.calculate('+');
console.log(risult);
 
 
You could also use arrow functions, so that the this is inherited from the outer scope instead, eg fn = () => this.x + this.y:
var num = {
  x: 12,
  y: 3,
  calculate: function(operation) {
    var fn;
    switch (operation) {
      case '+':
        fn = () => this.x + this.y
        break;
      case '-':
        fn = () => this.x - this.y;
        break;
      default:
        fn = function() {};
    }
    return fn();
  }
}
risult = num.calculate('+');
console.log(risult);
 
 
But switch is quite verbose and can be pretty error-prone. How about using an object indexed by operation instead (and use correct spelling to prevent bugs, use result, not risult):
const fns = {
  '+': () => num.x + num.y,
  '-': () => num.x - num.y,
};
var num = {
  x: 12,
  y: 3,
  calculate: operation => {
    const fn = fns[operation] || (() => null);
    return fn();
  }
};
const result = num.calculate('+');
console.log(result);