javascript - Remove exact duplicate row from array

Question

I have a following list in Javascript.

var list = [{
    ID: "2566",
    NAME: "ENTERPRISE EXPENSE",
    EXECUTE: "LOAD_DIMENTION",
    TYPE: "PROCEDURE"
  },
  {
    ID: "1234",
    NAME: "LOAD EXPENSE FACT",
    EXECUTE: "LOAD_EXPENSE_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RID005",
    NAME: "CALCULATE FAST GROUP",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  }
];

This list have many duplicate rows. I want to remove all the duplicate rows out of it.

The object can have more 10 properties also, so checking each property is not the solution. Plus whatever way I got online are either for 1 dimensional array or to remove duplicates by specific property. How can I maintain a list containing all unique rows?

What I could think of, is comparing JSON stringify version of the object. Like so : JSON.stringify(obj1) === JSON.stringify(obj2). That would be fast enough I guess, but the order of the elements properties would matter. Is this an issue ? — Scalpweb
– Scalpweb, Commented Feb 25, 2017 at 11:42
No order is not an issue. But even if I compare each row with all the others won't the process be too slow? With increase of list size the complexity will increase. — Harshita Sethi
– Harshita Sethi, Commented Feb 25, 2017 at 11:46
The answer from "Kind user" is exactly what I had in mind. The JSON.stringify process is the quickest way of comparing 2 objects in JS. I'd say you'll have to make some benchmark to check if it's fast enough but it should be fine. — Scalpweb
– Scalpweb, Commented Feb 25, 2017 at 12:17

Community · Accepted Answer · 2017-05-23 12:00:10Z

From the MDN article on JSON.stringify:

Properties of non-array objects are not guaranteed to be stringified in any particular order. Do not rely on ordering of properties within the same object within the stringification.

Instead of comparing the JSON stringification of each object directly, one should first use Object.entries to get an array of [key, value] pairs and sort it by the string value of key. This sorted entries array can then be safely stringified and compared regardless of the JSON.stringify implementation.

Note that my example below uses the latest ES6 features, including Set and Object.entries. To transpile this code to ES5 and run it, please visit https://babeljs.io/repl/.

function deepUnique (array) {
  return [...new Set(array.map(e => JSON.stringify(Object.entries(e).sort(compareEntriesByKey))))].map(e => objectFromEntries(JSON.parse(e)))
}

function compareEntriesByKey (a, b) {
  return a[0] < b[0] ? -1 : a[0] === b[0] ? 0 : 1
}

function objectFromEntries (entries) {
  let result = {}
  entries.forEach(e => result[e[0]] = e[1])
  return result
}

var list = [{
    ID: "2566",
    NAME: "ENTERPRISE EXPENSE",
    EXECUTE: "LOAD_DIMENTION",
    TYPE: "PROCEDURE"
  },
  {
    ID: "1234",
    NAME: "LOAD EXPENSE FACT",
    EXECUTE: "LOAD_EXPENSE_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RID005",
    NAME: "CALCULATE FAST GROUP",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  }
];

console.log(deepUnique(list))

Credit to Kind user for creating an answer using both JSON and Set before I posted mine.

kind user · Accepted Answer · 2017-02-25 11:45:30Z

2

This will remove exact duplicates from your array of objects.

var list = [{ ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" },
     { ID: "1234", NAME: "LOAD EXPENSE FACT", EXECUTE: "LOAD_EXPENSE_FACT", TYPE: "STORED PROCEDURE" },
     { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" },
     {NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required"},
     {NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate"}, 
     { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" },
     { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     {ID: "RID005",NAME: "CALCULATE FAST GROUP",EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     {NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required"}, 
     {NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate"}];
     
var filtered = [...new Set(list.map(v => JSON.stringify(v)))];

console.log(filtered.map(v => JSON.parse(v)));

answered Feb 25, 2017 at 11:45

kind user

42k8 gold badges69 silver badges78 bronze badges

1 Comment

Harshita Sethi Over a year ago

Yeah its working fine. Thanks for the solution. But if the order of the object property is changed then its causing issue.

dodov · Accepted Answer · 2017-02-25 12:00:36Z

Stringifies all elements, while checking if similar ones exist and then parses them. JSON.stringify() and JSON.parse() are not incredibly fast, though. Keep that in mind, if your objects are quite large.

var list = [{
    ID: "2566",
    NAME: "ENTERPRISE EXPENSE",
    EXECUTE: "LOAD_DIMENTION",
    TYPE: "PROCEDURE"
  },
  {
    ID: "1234",
    NAME: "LOAD EXPENSE FACT",
    EXECUTE: "LOAD_EXPENSE_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RID005",
    NAME: "CALCULATE FAST GROUP",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  }
];

function removeDuplicates(list) {
  return list.reduce(function(memo, obj) {
    var string = JSON.stringify(obj);

    if (memo.indexOf(string) === -1) {
      memo.push(string);
    }

    return memo;
  }, []).map(function(stringified) {
    return JSON.parse(stringified);
  });
}

list = removeDuplicates(list);
console.log(list);

Redu · Accepted Answer · 2017-02-25 12:40:08Z

The following snippet will remove the duplicates by constructing a hash map with JSON stringified keys and the corresponding object items as values. By not stringifying the values, we keep their references. So no JSON.parse() operation is used here.

function removeDupes(a){
  var hash = a.reduce(function(h,o){
                        var os = JSON.stringify(o);
                        return h[os] ? h : (h[os] = o, h);
  },{});
  return Object.keys(hash)
               .map(k => hash[k]);
}

var list = [{     ID: "2566",
                NAME: "ENTERPRISE EXPENSE",
             EXECUTE: "LOAD_DIMENTION",
                TYPE: "PROCEDURE"
            },
            {     ID: "1234",
                NAME: "LOAD EXPENSE FACT",
             EXECUTE: "LOAD_EXPENSE_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RID005",
                NAME: "CALCULATE FAST GROUP",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            }
           ],
     res = removeDupes(list);
console.log(res);

However if you would like to remove all occurrences of a duplicated item then you may still do the job in a very similar fashion like;

function removeDupes(a){
  var hash = a.reduce(function(h,o){
                        var os = JSON.stringify(o);
                        return h[os] ? (h[os] = false,h) : (h[os] = o, h);
                      },{});
  return Object.keys(hash)
               .reduce((r,k) => hash[k] === false ? r : r.concat(hash[k]),[]);
}

var list = [{     ID: "2566",
                NAME: "ENTERPRISE EXPENSE",
             EXECUTE: "LOAD_DIMENTION",
                TYPE: "PROCEDURE"
            },
            {     ID: "1234",
                NAME: "LOAD EXPENSE FACT",
             EXECUTE: "LOAD_EXPENSE_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RID005",
                NAME: "CALCULATE FAST GROUP",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            }
           ],
     res = removeDupes(list);
console.log(res);

I know the likelihood of a collision occurring with "don't count this one" is low, but please, this is a perfect example of when exploiting the unique identity of objects is in order. var IGNORE = {} . . . if (hash[k] === IGNORE) { ... }
@gyre Yes sure you are right but it was just a comment-ish piece of code not meant for production.
I get your point, but you'd be surprised how much code from SO makes it into someone's production one way or another...
@gyre Wow i haven't thought about it. Let me correct then... Thanks.

Nina Scholz · Accepted Answer · 2017-02-25 12:46:13Z

You could use a hash table with only the sorted keys as hash and an array for the objects with the same keys.

{
    "EXECUTE|ID|NAME|TYPE": [
        { ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" },
        { ID: "1234", /* ... */ },
        { ID: "RI0031", /* ... */ },
        { ID: "RI04", /* ... */ },
        { ID: "RID005", /* ... */ }
    ],
    "EXECUTE|NAME|TYPE": [
        { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" },
        { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" },
        { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }
    ]
}

For filtering, you could first check the keys, joined with pipe, in the hash table and then if found, iterate the array of the hash table and check all properties. If not equal, then push the actual object to the hash table, and return true.

var list = [{ ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" }, { ID: "1234", NAME: "LOAD EXPENSE FACT", EXECUTE: "LOAD_EXPENSE_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RID005", NAME: "CALCULATE FAST GROUP", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }],
    result = list.filter(function (hash) {
        return function (o) {
            var keys = Object.keys(o).sort(),
                key = keys.join('|');

            if (!hash[key]) {
                hash[key] = [o];
                return true;
            }
            if (!hash[key].some(function (p) { return keys.every(function (k) { return o[k] === p[k]; }); })) {
                hash[key].push(o);
                return true;
            }
        };
    }(Object.create(null)));

console.log(result);

.as-console-wrapper { max-height: 100% !important; top: 0; }

ES6 with Map

var list = [{ ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" }, { ID: "1234", NAME: "LOAD EXPENSE FACT", EXECUTE: "LOAD_EXPENSE_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RID005", NAME: "CALCULATE FAST GROUP", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }],
    result = list.filter((map => o => {
        var keys = Object.keys(o).sort(),
            key = keys.join('|');

        if (!map.has(key)) {
            map.set(key, [o]);
            return true;
        }
        if (!map.get(key).some(p => keys.every(k => o[k] === p[k]))) {
            map.get(key).push(o);
            return true;
        }
    })(new Map));

console.log(result);

.as-console-wrapper { max-height: 100% !important; top: 0; }

Collectives™ on Stack Overflow

javascript - Remove exact duplicate row from array

5 Answers 5

Comments

1 Comment

Comments

4 Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

5 Answers 5

Comments

1 Comment

Comments

4 Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related