Making forEach wait for callback

Question

New to async programming so I just can't figure how to do this:

$results = [];

products.forEach(function (product) {

  // 1. Search ...
  google(keyword, function (err, res) {
    if (err) console.error(err)

    for (var i = 0; i < res.links.length; ++i) {
      var result = res.links[i];
      var obj = {
        title: res.links[i].title,
        href: res.links[i].href,
        description: res.links[i].description
      }
      results.push(obj); // 2. store each result in results Array
    }
  }, processData); // 3. send all results to processData when done

  // 5. NOW, itereate further ...

});

function processData(results) {
  console.log('processing data');
  // 4. save results to DB
}

Since the process requires making HTTP requests, collecting data and then saving to DB which all takes time, so I don't want forEach to advance to the next element until one is done.

Potential duplicate of http://stackoverflow.com/questions/5010288/how-to-make-a-function-wait-until-a-callback-has-been-called-using-node-js or http://stackoverflow.com/questions/18983138/callback-after-all-asynchronous-foreach-callbacks-are-completed ? — qxz, Dec 16 '16 at 07:40

Pankaj Jatav · Answer 1 · 2016-12-16T10:13:06.780

Use async package.

async.eachSeries(docs, function iteratee(product, callback) {
    // 1. Search ...
    google(keyword, function (err, res) {
        if (err) {
           console.error(err)
           callback(results) // this will send a fail callback.
        }
        for (var i = 0; i < res.links.length; ++i) {
          var result = res.links[i];
          var obj = {
            title: res.links[i].title,
            href: res.links[i].href,
            description: res.links[i].description
          }
          results.push(obj); // 2. store each result in results Array
          callback(null, results) // this is a success callback
        }
      }, processData); // 3. send all results to processData when done
});

Note: Callback behaves like return. Once callback meet the value, It won't further proceed. Now it will send request for the next product.

Robba · Accepted Answer · 2016-12-16T08:29:40.543

Since the forEach is synchronous and the request is asynchronous, there is no way to do it exactly as you describe. What you can do however is to create a function that handles one item from the docs array and removes it, then when you're done processing, go to the next:

var results;
var productsToProcess;
MongoClient.connect( 'mongodb://localhost:27017/suppliers', function ( err, db ) {
  assert.equal( null, err );
  var findDocuments = function ( db ) {
    var collection = db.collection( 'products' );
    collection.find( {
      $and: [ {
        "qty": {
          $gt: 0
        }
      }, {
        "costex": {
          $lte: 1000.0
        }
      } ]
    }, {
      "mpn": 1,
      "vendor": 1,
      "_id": 0
    } ).limit( 1 ).toArray( function ( err, products ) {
      assert.equal( err, null );
      productsToProcess = products;
      getSearching();
      db.close();
    } );
  }
  findDocuments( db );
} );

function getSearching() {
  if ( productsToProcess.length === 0 ) return;
  var product = productsToProcess.splice( 0, 1 )[0];
  var keyword = product[ 'vendor' ] + ' "' + product[ 'mpn' ] + '"';
  google( keyword, function ( err, res ) {
    if ( err ) console.error( err )
    for ( var i = 0; i < res.links.length; ++i ) {
      var result = res.links[ i ];
      var obj = {
        title: res.links[ i ].title,
        href: res.links[ i ].href,
        description: res.links[ i ].description
      }
      results.push( obj );
    }
  }, processData );
}

function processData( results ) {
  MongoClient.connect( 'mongodb://localhost:27017/google', function ( err, db ) {
    assert.equal( null, err );
    // insert document to DB
    var insertDocuments = function ( db, callback ) {
      // Get the documents collection
      var collection = db.collection( 'results' );
      // Insert some documents
      collection.insert( results, function ( err, result ) {
        assert.equal( err, null );
        console.log( "Document inserted" );
        callback( result );
        db.close();
      } );
    }
    insertDocuments( db, getSearching );
  } );
}

EDIT

Moved the products from the database to the productsToProcess variable and changed the getSearching() to no longer require a parameter.

Yeah, there were some things not really clear like what the `docs` variable is, where `keyword` comes from and why the `product` parameter is never used. I take it you can make the example work for your situation nonetheless. — Robba, Dec 16 '16 at 07:50
Still confusing as hell, but from what I gather .. its kind of recursive instead of a loop — eozzy, Dec 16 '16 at 07:55
Yes, by making it a recursive function, you can determine manually when the next iteration is started, allowing you to start it when the processing of the previous iteration has completed. In your example, where does `keyword` come from? — Robba, Dec 16 '16 at 08:05
http://pastebin.com/pZg3F5Jy <- The array in my script comes from the DB so do I have to do a DB query for each iteration? — eozzy, Dec 16 '16 at 08:06
Ah, no. I think you should keep track of the products list. Let me update my answer using your code. — Robba, Dec 16 '16 at 08:29
Thanks heaps! Updated the code now (http://pastebin.com/qjtTx1N7) but I don't know why it complains `TypeError: getSearching is not a function` when it clearly is a function and hoisted before calling too. — eozzy, Dec 16 '16 at 08:48
On line 56 you specify `getSearching` as a parameter to the function, but on line 79 it is called without. You can just remove the parameter as the `getSearching` function is now global — Robba, Dec 16 '16 at 08:54

score 0 · Answer 3 · answered Dec 16 '16 at 09:25

You can't wait for asynchronous operations inside Array.prototype.forEach().

Taking into account the library you are using for the requests to Google is not compatible with Promises maybe using Async could be a fast solution in your case (for big projects or Promise compatible libraries I recommend the Promise way).

Async map allows you to use asynchronous operations inside, since it waits for the callback.

In your case it would be something like this I guess:

async.map(products, function(product, callback) {
  var keyword = product['vendor'] + ' "' + product['mpn'] + '"';
  google( keyword, function (err,res) {
    if (err) {
      // if it fails it finish here
      return callback(err);
    }

    // using map here makes it easier to loop through the results
    var results = res.links.map(function(link) {
      return {
        title: link.title,
        href: link.href,
        description: link.description
      };
    });
    callback(null, results);
  });
}, processData);

If you have any question about the above code let me know.

Making forEach wait for callback

3 Answers3