Node.js - dependant API request calls using async.waterfall and merge JSON response with underscore.extend

Question

Okay, there is quite a bit happening here but I tried to isolate the issue as much as possible.

In my node.js Express project, I'm making two API request calls that the second call is dependant on the first. To make this task easier, I'm using waterfall method from async module.

In getVideoDetails function, I put the second API request in loop with the videoId retreived from the first response to get video data.

The problem currently I have is that, var extended only gives me body value while I expect it to be result + body.

I wonder that's because _extend shouldn't be inside the loop. I'm also not clear how I can make the result accessible to call callback(result) outside the loop.

async.waterfall([

    function getVideos (getVideoCallback) {

        ...

    },

    function getVideoDetails (result, getVideoDetailsCallback) {

        var urls = [];

        Object.keys(result.items).forEach(function(item) {
            urls.push ("https://www.googleapis.com/youtube/v3/videos?part=contentDetails&id=" + result.items[item].contentDetails.videoId + "&key=xxx");
        })

        urls.forEach(function(url) {
            request( url, function(err, response, body) {
                if(err) { console.log(err); return; }
                body = JSON.parse(body);    

                var extended = _.extend(result, body);

                getVideoDetailsCallback(null, extended);

            });
        });

    }
], function (err, result) {

    if (err) { console.log(err); return; }

    callback(result);

});

Answer 1

If you want to do these things in order...

1. Generate an array asynchronously
2. Do some asynchronous process for each item in the array
3. Do something asynchronous after you have processed the array

...then this is one way you could do it with the async library.

var request = require("request");
var _ = require("lodash");
var db = require("whatever you are using");

module.exports = function(req, res) {
    var params = req.params;

    async.waterfall([

        function getVideos(next) {
            db.findAll().then(function(rows) {
                next(null, rows);
            });
        },

        function forEachVideoDoSomethingAsync(videos, next) {
            var urls = videos.map(function(obj) {
                obj.url = obj.name + "http://googleapi.com/whatever";
                return obj;
            });

            /** this is the part you are missing **/

            //async.map takes three arguments
            //1. the array you want to work on
            //2. a function that is applied to each item in the array
            //    the iterator function receives two arguments
            //    1. the current item in the array
            //    2. a callback you invoke when you want to move to the next item
            //3. a callback that is executed once the loop has been completed

            async.map(urls, function(currentItem, callback) {
                request(currentItem.url, function(err, response, body) {
                    if (err) 
                        return console.log(error);
                    //you probably have to extend the current item in the array with the response object
                    var json = JSON.parse(body);
                    var extended = _.extend(currentItem, json);
                    //each item you send via the callback will be pushed into the result of async.map
                    callback(extended);
                });
            }, function(err, urls_extended) {
                //urls_extended is now an array of extended items
                //now you have to exit the waterfall
                next(null, urls_extended);
            });

        }

    ], function(err, urls_extended) {
        if (err) 
            return console.log(err);
        //exit the entire process
        res.send(urls_extended);
    })

};