how to make synchronous http calls using promises in node.js

Question

I would like to iterate thru an array of students and make http call for each of them and parse the response and insert into mongodb, so I would like to do this for each student one by one untill all data are inserted then continue with the next one so that it would be better for the CPU and RAM Memory...

So far I am doing this, but this for some reason is not what I wanted...

var startDate = new Date("February 20, 2016 00:00:00");  //Start from February
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;

iterateThruAllStudents(from, to);

function iterateThruAllStudents(from, to) {
    Student.find({status: 'student'})
        .populate('user')
        .exec(function (err, students) {
            if (err) {
                throw err;
            }

            async.eachSeries(students, function iteratee(student, callback) {
                if (student.worksnap.user != null) {
                    var worksnapOptions = {
                        hostname: 'worksnaps.com',
                        path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
                        headers: {
                            'Authorization': 'Basic xxx='
                        },
                        method: 'GET'
                    };

                    promisedRequest(worksnapOptions)
                        .then(function (response) { //callback invoked on deferred.resolve
                            parser.parseString(response, function (err, results) {
                                var json_string = JSON.stringify(results.time_entries);
                                var timeEntries = JSON.parse(json_string);
                                _.forEach(timeEntries, function (timeEntry) {
                                    _.forEach(timeEntry, function (item) {
                                        saveTimeEntry(item);
                                    });
                                });
                                callback(null);
                            });
                        }, function (newsError) { //callback invoked on deferred.reject
                            console.log(newsError);
                        });
                }
            });
        });
}

function saveTimeEntry(item) {
    Student.findOne({
            'worksnap.user.user_id': item.user_id[0]
        })
        .populate('user')
        .exec(function (err, student) {
            if (err) {
                throw err;
            }
            student.timeEntries.push(item);
            student.save(function (err) {
                if (err) {
                    console.log(err);
                } else {
                    console.log(Math.random());
                }
            });

        });
}

function promisedRequest(requestOptions) {
    //create a deferred object from Q
    process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
    var deferred = Q.defer();
    var req = http.request(requestOptions, function (response) {
        //set the response encoding to parse json string
        response.setEncoding('utf8');
        var responseData = '';
        //append data to responseData variable on the 'data' event emission
        response.on('data', function (data) {
            responseData += data;
        });
        //listen to the 'end' event
        response.on('end', function () {
            //resolve the deferred object with the response
            console.log('http call finished');
            deferred.resolve(responseData);
        });
    });

    //listen to the 'error' event
    req.on('error', function (err) {
        //if an error occurs reject the deferred
        deferred.reject(err);
    });
    req.end();
    //we are returning a promise object
    //if we returned the deferred object
    //deferred object reject and resolve could potentially be modified
    //violating the expected behavior of this function
    return deferred.promise;
}

It also seem that the saveEntry() inside .then is being called for all students at once, and that seems problematic.

I am new to Javascript especially when it comes to promises, callbacks... Anyone has any idea of achieving such thing...

Answer 1

Ok I did resolved this problem and if someone in future has this problem here is a solution I came of.

    var startDate = new Date("February 20, 2016 00:00:00");  //Start from February
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;

iterateThruAllStudents(from, to);

function iterateThruAllStudents(from, to) {
    Student.find({status: 'student'})
        .populate('user')
        .exec(function (err, students) {
            if (err) {
                throw err;
            }

            var counter = 1;
            async.eachSeries(students, function iteratee(student, callback) {
                counter++;
                if (student.worksnap.user != null) {
                    console.log('');
                    console.log('--------------');
                    console.log(student.worksnap.user.user_id);
                    var worksnapOptions = {
                        hostname: 'worksnaps.com',
                        path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
                        headers: {
                            'Authorization': 'Basic xxxxx'
                        },
                        method: 'GET'
                    };

                    promisedRequest(worksnapOptions)
                        .then(function (response) { //callback invoked on deferred.resolve
                            parser.parseString(response, function (err, results) {
                                var json_string = JSON.stringify(results.time_entries);
                                var timeEntries = JSON.parse(json_string);
                                var isEmpty = _.isEmpty(timeEntries); // true
                                if (isEmpty) {
                                    callback(null);
                                }
                                saveTimeEntry(timeEntries).then(function (response) {
                                    console.log('all timeEntries for one student finished....Student: ' + student.worksnap.user.user_id + ' Student Counter: ' + counter);
                                    callback(null);
                                });
                            });
                        }, function (newsError) { //callback invoked on deferred.reject
                            console.log(newsError);
                        });
                } else {
                    callback(null);
                }
            });
        });
}

function saveTimeEntry(timeEntries) {
    var deferred = Q.defer();
    _.forEach(timeEntries, function (timeEntry) {
        _.forEach(timeEntry, function (item) {
            Student.findOne({
                    'worksnap.user.user_id': item.user_id[0]
                })
                .populate('user')
                .exec(function (err, student) {
                    if (err) {
                        //throw err;
                        console.log(err);
                    }
                    student.timeEntries.push(item);
                    student.save(function (err) {
                        if (err) {
                            console.log(err);
                            deferred.reject(err);
                        } else {
                            //console.log(Math.random());
                        }
                    });

                });
        });
        deferred.resolve('finished saving timeEntries for one student...');
    });

    return deferred.promise;
}

function promisedRequest(requestOptions) {
    //create a deferred object from Q
    process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
    var deferred = Q.defer();
    var req = http.request(requestOptions, function (response) {
        //set the response encoding to parse json string
        response.setEncoding('utf8');
        var responseData = '';
        //append data to responseData variable on the 'data' event emission
        response.on('data', function (data) {
            responseData += data;
        });
        //listen to the 'end' event
        response.on('end', function () {
            //resolve the deferred object with the response
            console.log('http call finished');
            deferred.resolve(responseData);
        });
    });

    //listen to the 'error' event
    req.on('error', function (err) {
        //if an error occurs reject the deferred
        console.log('inside On error.');
        console.log(err);
        deferred.reject(err);
    });
    req.end();
    //we are returning a promise object
    //if we returned the deferred object
    //deferred object reject and resolve could potentially be modified
    //violating the expected behavior of this function
    return deferred.promise;
}

Answer 2

First off, multiple nested operations are going to be a lot easier to code and handle errors reliably if you use promises for all async operations. That means learning how to use the promises that are built into your database (I assume you're using mongoose) and then wrapping any other async operations to use promises. Here are a couple links on using promises with mongoose:

Switching to use promises in Mongoose

Plugging in your own promise library into Mongoose

So, looking at the Mongoose documentation, you can see that .exec() and .save() already return promises so we can use them directly.

Adding this line of code will tell Mongoose to use Q promises (since that's the promise library you show):

// tell mongoose to use Q promises
mongoose.Promise = require('q').Promise;

Then, you need to promisify some operations that don't use promises such as the parsing step:

function parse(r) {
    var deferred = Q.defer();
    parser.parseString(r, function(err, results) {
        if (err) {
            deferred.reject(err);
        } else {
            deferred.resolve(results);
        }
    });
    return deferred.promise;
}

saveTimeEntry() can easily be written to return a promise, just using the promise support already in your database:

function saveTimeEntry(item) {
    return Student.findOne({'worksnap.user.user_id': item.user_id[0]}).populate('user').exec().then(function(student) {
        student.timeEntries.push(item);
        return student.save();
    });
}

So, now you have all the right pieces to rewrite the main logic using promises. The key thing to remember here is that if you return a promise from a .then() handler, it chains that promise onto the parent promise. We will use that a lot in your processing. Further, a common design pattern for sequencing promises that are iterating through an array is to use array.reduce() like this:

return array.reduce(function(p, item) {
    return p.then(function() {
         return someAsyncPromiseOperation(item);
    });
}, Q());

We will use that structure in a couple of places to rewrite the core logic using promises:

// tell mongoose to use Q promises
mongoose.Promise = require('q').Promise;

iterateThruAllStudents(from, to).then(function() {
    // done successfully here
}, function(err) {
    // error occurred here
});

function iterateThruAllStudents(from, to, callback) {
    return Student.find({status: 'student'}).populate('user').exec().then(function (students) {
        // iterate through the students array sequentially
        students.reduce(function(p, student) {
            return p.then(function() {
                if (student.worksnap.user != null) {
                    var worksnapOptions = {
                        hostname: 'worksnaps.com',
                        path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + 
                              '&from_timestamp=' + from + '&to_timestamp=' + to,
                        headers: {'Authorization': 'Basic xxx='},
                        method: 'GET'
                    };

                    return promisedRequest(worksnapOptions).then(function(response) {
                        return parse(response).then(function(results) {
                            // assuming results.time_entries is an array
                            return results.time_entries.reduce(function(p, item) {
                                return p.then(function() {
                                    return saveTimeEntry(item);
                                });
                            }, Q());
                        });
                    });
                }
            });
        }, Q())
    });
}

function parse(r) {
    var deferred = Q.defer();
    parser.parseString(r, function(err, results) {
        if (err) {
            deferred.reject(err);
        } else {
            deferred.resolve(results);
        }
    });
    return deferred.promise;
}

function saveTimeEntry(item) {
    return Student.findOne({'worksnap.user.user_id': item.user_id[0]}).populate('user').exec().then(function(student) {
        student.timeEntries.push(item);
        return student.save();
    });
}

function promisedRequest(requestOptions) {
    //create a deferred object from Q
    process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
    var deferred = Q.defer();
    var req = http.request(requestOptions, function (response) {
        //set the response encoding to parse json string
        response.setEncoding('utf8');
        var responseData = '';
        //append data to responseData variable on the 'data' event emission
        response.on('data', function (data) {
            responseData += data;
        });
        //listen to the 'end' event
        response.on('end', function () {
            //resolve the deferred object with the response
            console.log('http call finished');
            deferred.resolve(responseData);
        });
    });

    //listen to the 'error' event
    req.on('error', function (err) {
        //if an error occurs reject the deferred
        deferred.reject(err);
    });
    req.end();
    return deferred.promise;
}

One thing this offers that your code does not is that ANY error that occurs will percolate all the way back to the returned promise from iterateThruAllStudents() so no errors are hidden.

---

And, then cleaning it up some to reduce nested indentation and adding a useful utility function, it looks like this:

// tell mongoose to use Q promises
mongoose.Promise = require('q').Promise;

// create utility function for promise sequencing through an array
function sequence(array, iterator) {
    return array.reduce(function(p, item) {
        return p.then(function() {
            return iterator(item);
        });
    }, Q());
}

iterateThruAllStudents(from, to).then(function() {
    // done successfully here
}, function(err) {
    // error occurred here
});

function iterateThruAllStudents(from, to, callback) {
    return Student.find({status: 'student'}).populate('user').exec().then(function (students) {
        // iterate through the students array sequentially
        return sequence(students, function(item) {
            if (student.worksnap.user != null) {
                var worksnapOptions = {
                    hostname: 'worksnaps.com',
                    path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + 
                          '&from_timestamp=' + from + '&to_timestamp=' + to,
                    headers: {'Authorization': 'Basic xxx='},
                    method: 'GET'
                };
                return promisedRequest(worksnapOptions).then(function(response) {
                    return parse(response);
                }).then(function(results) {
                    // assuming results.time_entries is an array
                    return sequence(results.time_entries, saveTimeEntry);
                });
            }
        });
    });
}

function parse(r) {
    var deferred = Q.defer();
    parser.parseString(r, function(err, results) {
        if (err) {
            deferred.reject(err);
        } else {
            deferred.resolve(results);
        }
    });
    return deferred.promise;
}

function saveTimeEntry(item) {
    return Student.findOne({'worksnap.user.user_id': item.user_id[0]}).populate('user').exec().then(function(student) {
        student.timeEntries.push(item);
        return student.save();
    });
}

function promisedRequest(requestOptions) {
    //create a deferred object from Q
    process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
    var deferred = Q.defer();
    var req = http.request(requestOptions, function (response) {
        //set the response encoding to parse json string
        response.setEncoding('utf8');
        var responseData = '';
        //append data to responseData variable on the 'data' event emission
        response.on('data', function (data) {
            responseData += data;
        });
        //listen to the 'end' event
        response.on('end', function () {
            //resolve the deferred object with the response
            console.log('http call finished');
            deferred.resolve(responseData);
        });
    });

    //listen to the 'error' event
    req.on('error', function (err) {
        //if an error occurs reject the deferred
        deferred.reject(err);
    });
    req.end();
    //we are returning a promise object
    //if we returned the deferred object
    //deferred object reject and resolve could potentially be modified
    //violating the expected behavior of this function
    return deferred.promise;
}

Of course, this is a lot of code and I have no way to test it and I've never written Mongoose code before, so there very well could be some goofs in here, but hopefully you can see the general idea and work through any mistakes I might have made.