Express and Mongodb insert same data multiple times

Question

I am quite new to Express and Mongodb. The project that I am working on requires me to:

Take an object that contains multiple url
Download the content of the url and save it to a cloud storage
Generate links for each of the file saved
Save these links into Mongodb as individual documents

The incoming object looks something like this:

{
    "id" : 12345678,
    "attachments" : [ 
        {
            "original_url" : "https://example.com/1.png",
        },
        {
            "original_url" : "https://example.com/2.png",
        },
        {
            "original_url" : "https://example.com/3.png",
        }
    ]
}

the end goal is to have 3 separate document like this saved on mongodb:

{
    "id" : 87654321,
    "some_other_data": "etc",
    "new_url" : "https://mycloudstorage.com/name_1.png"
}

I have a simple loop like this:

for(var i = 0; i < original_data.attachments.length; i++){

    var new_url = "https://example.com/" + i + ".png";

    download(original_url, new_url, function(new_url){

        console.log(new_url)

        new_data.new_url = new_url;

        save_new_url_to_mongodb(new_data);

    });
}

and the saving function looks like this:

function save_new_url_to_mongodb (data, cb) {
  getCollection(collection, function (err, collection) {
    if (err) {
      return cb(err);
    }

    collection.insert(data, {w: 1, ordered: false}, function (err, result) {
      if (err) {
        return cb(err);
      }

      var item = fromMongo(result.ops);
      cb(null, item);
    });
  });
}

var download = function(original_url, new_url, callback){
  request.head(original_url, function(err, res, body){
    if(res === undefined){
        console.log(err);
    } else {

        var localUrlStream = request(original_url);
        var file = bucket.file(new_url);
        var remoteWriteStream = file.createWriteStream();
        var stream = localUrlStream.pipe(remoteWriteStream);

        stream.on('error', function (err) {
            next(err);
        });

        stream.on('finish', function(){
            callback(new_url);
        });
    }
  });
};

The downloading part is fine, I get 3 different image files in my cloud storage. The console.log also gives me 3 different new urls.

The problem is that the newly saved mongodb document all have the same new_url. And sometimes if there are more original_url in the original data, some of the new documents would fail to save.

Thanks a lot

@Wasiq Muhammad included and updated the save function as well so that it make more sense. thx — gdzla, Sep 10 '16 at 17:43

score 0 · Answer 1 · answered Sep 10 '16 at 21:23

It's a scoping issue in your assignment of new_url in the for loop. See here: JavaScript closure inside loops – simple practical example

A solution is to use Array.Prototype.forEach which inherently solves the scope issue since each iteration creates a closure for the callback

original_data.attachments.forEach(function(i) {
  var new_url = "https://example.com/" + i + ".png";

  download(original_url, new_url, function(new_url){
    console.log(new_url)
    new_data.new_url = new_url;
    save_new_url_to_mongodb(new_data);
  });
})

Express and Mongodb insert same data multiple times

1 Answers1