Writing a generic utility that will remove duplicates from any array in javascript

Question

i'm working on writing a generic utility that will remove duplicates from an array in javascript. I have the following code where what's desired actually works. I'd like to then turn it into a utility. The 1st block of code works, the 2nd is the utility, and the 3rd is how I am trying to apply the utility (doesn't work - the duplicates from the array still appear). Any insight would be super appreciated.

//Code without utility - works//
var productArr = [];
for (var p in theData) {
  var theProduct = theData[p];
  var theProductSelect = productArr.filter(function(value, index, self) {
    return self.indexOf(value) === index;
  });
  productArr.push(theProduct.name);
}

//Here's the utility I am trying to write
publicMethods.deDuplicate = function(array, key, type) {
  var providedArray = [];
  var uniqueArray = providedArray.filter(function(value, index, self) {
    return self.indexOf(value) === index;
    return providedArray;
  });
};

//Code with Utility - duplicates still appear
var productArr = [];
for (var p in theData) {
  var theProduct = theData[p];
  productArr.push(theProduct.name);
  publicMethods.deDuplicate(productArr)
}

[Don't use `for…in` enumerations on arrays!](https://stackoverflow.com/q/500504/1048572) — Bergi, Jan 17 '18 at 15:20
`return self.indexOf(value) === index;` and `return providedArray;`: one of these two shouldn't be there. — Federico klez Culloca, Jan 17 '18 at 15:21
`publicMethods.deDuplicate` will never work as you are trying to apply filter to an empty array `providedArray` and then returning that empty array. Please debug the code for it. Did you mean to use filter on passed `array` param actually? — palaѕн, Jan 17 '18 at 15:21
Also, you're calling `deDuplicate` with just one argument, while it takes three. Something's not right. — Federico klez Culloca, Jan 17 '18 at 15:22
In your original code, there's only one array. In the attempt with the utility, there's the `productArr` *and* a `providedArray` for *each* call of your utility. — Bergi, Jan 17 '18 at 15:23

score 2 · Accepted Answer · answered Jan 17 '18 at 15:22

Two alternatives:

let arr = [1, 2, 3, 4, 1, 2, 3];

const getUnique = (arr) => {
    return Array.from(new Set(arr));
}

const getUniqueAlt = (arr) => {
    return arr.reduce((acc, cur) => {
    if(acc.indexOf(cur) == -1) {
        acc.push(cur);
    }

    return acc;
  }, [])
}

console.log(getUnique(arr))
console.log(getUniqueAlt(arr));

https://jsfiddle.net/z1w775y1/

Won't work so well with an array of objects

zer00ne · Answer 2 · 2018-01-17T15:57:00.003

1

Use Set() used with the spread operator: ... turns a Set into an array and a Set must have only unique values. @Bergi pointed out an alternative statement using Set() that's easier to read.

Array.from(new Set(array))

Demo

var arr = [2, "88c", 22, "v21", 3, 865, "a", "a1", 5521, "v21", 5678, "5678", 22];


function uq(array) {
  return [...new Set(array)];
} 
    
console.log('uq(arr)= '+uq(arr));

function unique(array) {
  return Array.from(new Set(array));
}

console.log('unique(arr)= '+unique(arr));

edited Jan 17 '18 at 15:57

answered Jan 17 '18 at 15:39

zer00ne

41,936
6
41
68

2

Using `Array.from(new Set(array))` does just the same but might make the intention more clear – Bergi Jan 17 '18 at 15:43

zfrisch · Answer 3 · 2018-01-17T16:59:38.507

Sorry for the delay in response, this took me a little bit.

This will remove duplicates from an array with elements of any type. This includes elements that are Arrays and Objects. Please let me know if you run into any trouble using it. I did as much testing with it as I could.

The concept really isn't that difficult. With recursion it's often more about not crossing the wires while implementing that concept. We recursively test if any elements are an array or object, then we recursively stringify them. Afterwards we create a set object, fill it with the stringified items, turn that into an array, and then reinflate the stringified elements to objects or arrays.

function uniqueArr(original) {
  let returnable = json_recurse(original);
  function json_recurse(arr) {
    return arr.map((item, ind) => {
      if (Array.isArray(item)) {
        return JSON.stringify(json_recurse(item));
      }
      if (typeof item === "object") {
        return JSON.stringify(item);
      } else return item;
    });
  }
  function inflate_recurse(arr) {
    return arr.map(item => {
      if (Array.isArray(item)) return inflate_recurse(item);
      try {
        var obj = JSON.parse(item);
      } catch (e) {
        return item;
      }
      if(obj) {
      return obj;
      }
      else return item;
    })
  }
  return inflate_recurse(Array.from(new Set(returnable)));
}

function uniqueArr(original) {
  let returnable = json_recurse(original);
  function json_recurse(arr) {
    return arr.map((item, ind) => {
      if (Array.isArray(item)) {
        return JSON.stringify(json_recurse(item));
      }
      if (typeof item === "object") {
        return JSON.stringify(item);
      } else return item;
    });
  }
  function inflate_recurse(arr) {
    return arr.map(item => {
      if (Array.isArray(item)) return inflate_recurse(item);
      try {
        var obj = JSON.parse(item);
      } catch (e) {
        return item;
      }
      if(obj) {
      return obj;
      }
      else return item;
    })
  }
  return inflate_recurse(Array.from(new Set(returnable)));
}

let arr = [
   ["whatever", 5], 
   ["whatever", 5], 
  {
    id: 1
  }, 
  ["whatever", 5, 7], 
  3, 5, 9, 
  "hello", {
    id: 2
  }, 
  {
    id: 3
  }, 
  {
    id: 1
  }, 
  3, 5, 9, 
  "hello", 
  {
    id: 2
  }, 
  7, 35, 50, {
    id: 3
  },
  50
];
console.log(uniqueArr(arr));

thanks @zfrisch - this is really useful. Appreciate all the help — erics15, Jan 17 '18 at 17:14

Writing a generic utility that will remove duplicates from any array in javascript

3 Answers3

Demo