Nodejs async calling drain callback before complete

Question

I've got a nodejs app which runs about 40 oracle connections/queries. I'm using async to limit the asynchronous calls from the proposed solution here. My issue is the drain callback is running and completing before my function/queries complete. Any help greatly appreciated.

var queue = async.queue(runAsyncCallout, 5); 

        queue.concurrency = 5; 

        db.collection('callout').find({'active':0}).each(function(error, callout) {

             queue.push(callout);

        });

        queue.drain = function() {
            db.collection('callout_lock').update({'context': 'ALL_CALLOUTS'}, {$set: {'lock': '0'}});
            db.collection('callout_lock').update({'context': 'ALL_CALLOUTS'}, {$set: {'date': time_stamp}});
            console.log("All callouts done unlocking");
        };
        queue.concurrency = 5;


function runAsyncCallout(callout, callback) {

    switch(callout.db_connect){//Grab connection profiles

        case "A":
            connAttrs=A;
            break;
        case "B":
            connAttrs=B;
            break;
        case "C":
            connAttrs=C;
            break;
        default:
            connAttrs=D;
    }


    oracledb.getConnection(connAttrs, function (e, connection) {


            connection.execute(callout.query, {}, {
                outFormat: oracledb.OBJECT
            }, function (e, result) {

                //Sparing the details here, run oracle query and save to mongodb

                doRelease(connection);



            });


    });

callback();
}

Welcome to StackOverflow. Is it possible that the call `oracledb.getConnection` is done asynchronously? If so, the `runAsyncCallout` would fire, call `oracledb.getConnection` which would then go do it's thing, but in the meantime `runAsyncCallout` would continue on and possibly return to the caller before the actual queries are executed. — akousmata, Jan 12 '17 at 22:58

score 0 · Accepted Answer · answered Jan 12 '17 at 22:40

I think the short answer is that your callback at the end (passed from the queue) is in the wrong place. Invoking the callback at that time is like telling the queue, "Hey, done with that thing, next!"

You're not really done processing a "callout" until you've committed your work in Oracle and released the connection. Try invoking the callback within the callback passed into connection.release or connection.close (whichever you're using).

Here are some other tips/suggestions:

Switch from one-off connections to connection pools. The driver now supports multiple named pools in an internal pool cache, so it's pretty easy to use.
Switch from multiple round trips to bulk processing. Granted, this one is trickier depending on your experience with JavaScript and PL/SQL but I can provide an example if you like.

Thank you so much, this worked for me. I will explore your two other suggestions :) — Kochaloch, Jan 13 '17 at 16:19

Nodejs async calling drain callback before complete

1 Answers1