Bluebird with mongoose using Promise.Each

Question

I'm stuck in a function I'm working with ( I can be doing this all wrong ). So a quick explanation, I want to add bulk data in a collection, the collection is called "Sites" the format of the CSV is site,country,type. I'm trying to use promises for this (Bluebird). So consider the code:

 Promise.each(sites, sites => new Promise((resolve, reject) => {
    //console.log(sites);
    let name = tools.extractDomain(req, res, sites[0]);
    let country = sites[1];
    let group = sites[2];
    if (name != "" && country != "" && group != "") {

        Site.findOne({ name: name }, "_id", function(err, duplicate) {
            if (false) {
                console.log("Duplicate site: " + duplicate);

            } else { 
                   //console.log("Adding " + name)
                let site = new Site()
                site.name = name
                site.meta = {}
                site.group = group
                site.country = country
                site.geomix = []
                site.addedBy = req.user._id
                site.addedAt = Date.now()
                site.saveAsync().then(function(response){
                    tools.saveHistory(req, res, response._id, response.name, "Website Meta fetched.");
                    tools.saveHistory(req, res, response._id, response.name, "Link added for the first time."); //Save in history 
                    resolve(site);
                }).catch(function (e){
                    console.log(name);
                    reject();
                });  
            }
         });

    }else{
        console.log('Wrong Format');
    }
}).then((data) => {
      console.log('All websites processed!');
      addedSites.push(data);
}).catch(err => {
      //console.error('Failed');
}));

    res.send({ status: 'ok', message: ''});

I'm making ajax calls so I return a res.send({ status: 'ok', message: ''}), I know that its in the incorrect place and I want to send some data along the res.send. Currently it sends the headers before the code actually finishes. I want to send the headers after all the data is added in Mongo but for every each in this case he resolve() so if I send the headers inside the ".then" of the ".each" I will get headers already sent error.

This might be a bit confusing. I feel I'm not doing this right. I'm going a bit crazy as well as I can't find a proper example that I can understand and implement.

But in the end my main question is: using an Ajax call what's the proper way to add let's say 1000 records in a collection using promises and actually control properly those who fail to add and those who don't?

Right now my code actually works but the logic is wrong for sure.

Thanks.

Answer 1

You can use bulkWrite on your model.

Ref: http://mongoosejs.com/docs/api.html#model_Model.bulkWrite

EDIT:

Sorry I misunderstood you. You need to move res.send({ status: 'ok', message: ''}); to then() and catch() blocks, so you will get something like this:

Promise.each(sites, sites => new Promise((resolve, reject) => {
    // stuff you did before
}).then((data) => {
    console.log('All websites processed!');
    addedSites.push(data);
    res.send({ status: 'ok', message: ''});
}).catch(err => {
    res.send({ status: 'failed', message: err.message});
}));

Answer 2

This is what I came too, if someone can tell me if this is a good arch.

    exports.addBulkSite = function(req, res, next) {
    let siteArray = csv.parse((req.body.sites).trim()),
        addedSites = [],
        failedSites = [],
        duplicated = [],
        sites = siteArray,
        size = sites.length,
        processed = 0,
        meta;
    Promise.each(sites, sites => new Promise((resolve, reject) => {
        let name = tools.extractDomain(req, res, sites[0]),
            country = sites[1],
            group = sites[2];
        if (name != "" && country != "" && group != "") {
            Site.findOneAsync({ name: name }, "_id").then(function(duplicate) {
                duplicated.push(duplicate);
                reject({name:name, message: 'Duplicated', critical:false});         
             }).catch(function(notDuplicated){
                let site = new Site()
                site = { 
                            name: name, 
                            meta: {}, 
                            group: group, 
                            country: country, geomix:{}, 
                            addedBy: req.user._id,
                            addedAt:Date.now()
                        }
                site.saveAsync().then(function(response){
                    tools.saveHistory(req, res, response._id, response.name, "Website Meta fetched.");
                    tools.saveHistory(req, res, response._id, response.name, "Link added for the first time."); //Save in history 
                    resolve(site);
                }).catch(function (e){
                    console.log(e);
                    reject({name:name, message: 'Error saving in the database. Please contact the administrator.', critical: true});
                });  
            });
        }else{
            reject({name:name, message: 'Paramaters are missing', critical:false});
        }
    }).then((data) => {
          processed++;
          addedSites.push(data);
          if(processed==size){
            console.log('out');
            res.send({ status: 'ok', addedSites: addedSites, failedSites: failedSites, duplicated: duplicated});
          }
    }).catch((err) => {
          processed++;
          console.log(err);
          failedSites.push(err);
          if(processed==size){
            console.log('out');
            res.send({ status: 'ok', addedSites: addedSites, failedSites: failedSites, duplicated: duplicated});
          }
    })); 
}