I was building a game when I came across this problem.
Here's a typescript class that might be able to help you. It uses a tree to do its magic.
You should be able to easily modify this to use arrays instead of the x and y parameters
// uses an internal tree structure to simulate set-like behaviour
export default class CoordinateSet {
  tree: Record<number, Record<number, boolean>> = {}
  add(x: number, y: number) {
    this.tree[x] ||= {}
    this.tree[x][y] = true;
  }
  remove(x: number, y: number) {
    // if the coordinate doesn't exist, don't do anything
    if (!this.tree[x] || !this.tree[y]) {
      return;
    }
    // otherwise, delete it
    delete this.tree[x][y];
    // if the branch has no leaves, delete the branch, too
    if (!Object.keys(this.tree[x]).length) {
      delete this.tree[x]
    }
  }
  has(x: number, y: number) {
    return !!this.tree[x]?.[y];
  }
}
And tests, which will also show you how it works:
import CoordinateSet from "./CoordinateSet";
describe("CoordinateSet", () => {
  it("Can add a coordinate", () => {
    const cs = new CoordinateSet();
    expect(cs.has(1,1)).toBeFalsy();
    cs.add(1, 1);
    expect(cs.has(1,1)).toBeTruthy();
  });
  it("Can remove a coordinate", () => {
    const cs = new CoordinateSet();
    cs.add(1, 1);
    expect(cs.has(1,1)).toBeTruthy();
    cs.remove(1,1);
    expect(cs.has(1,1)).toBeFalsy();
  })
})