How to make node js to wait for function to execute (even if it take time) and then execute the next statement

Question

How to make node js to wait for function to execute (even if it take time) and then execute the next statement . for me problem is for loop prints all urls first it does nt wait for require request function to check each url the http response. It just prints all urls. beacuse of this i am getting the error

console.log(dd[i]['loc'][0]); 
                 ^

TypeError: Cannot read property 'loc' of undefined

my full source code is as below

       var hapi = require('hapi');
    var glue = require('glue');
    var joi = require('joi');
    var handlebars = require('handlebars');
    var strop = require('string');
    var inert = require('inert');
    var async = require('async');
    var poop = require('poop');
    var bell = require('bell');
    var nconf = require('nconf');

    //xml parser starts

    var fs = require('fs'),
        xml2js = require('xml2js');

    var parser = new xml2js.Parser();
    fs.readFile('sitemapscene1.xml', function(err, data) {
    var count = 0;
    var a=0;
    var b=0;
    var c=0;
    var d=0;
    var e=0;
    var f=0;
    var g=0;
    var h=0;
    //parsing inside url starts
    var parseString = require('xml2js').parseString;

        parser.parseString(data, function (err, result) {
            ff = result;
            dd = ff['urlset']['url'];

            for (i =0; i< dd.length; i++){
             console.log(dd[i]['loc'][0]);
    //server reuest and checking
    var request = require('request');
    request(dd[i]['loc'][0], function (error, response, body) {
      if (!error && response.statusCode == 200) {
        console.log(dd[i]['loc'][0]); 
        console.log("This url http response is 200");
        count=count+1;
        console.log(count);
      }
    })



    }
    console.log("Total number of urls in this sitemap");
            console.log(dd.length);
            });

        });


    //xml parser ends

My full error code is 

    Total number of urls in this sitemap
22569

    console.log(dd[i]['loc'][0]); 
                     ^

TypeError: Cannot read property 'loc' of undefined
    at Request._callback 
    at Request.self.callback
    at emitTwo (events.js:87:13)
    at Request.emit (events.js:172:7)
    at Request.<anonymous> 
    at emitOne (events.js:77:13)
    at Request.emit (events.js:169:7)
    at IncomingMessage.<anonymous> 
    at emitNone (events.js:72:20)
    at IncomingMessage.emit (events.js:166:7)

My actual full code

 var hapi = require('hapi');
var glue = require('glue');
var joi = require('joi');
var handlebars = require('handlebars');
var strop = require('string');
var inert = require('inert');
var async = require('async');
var poop = require('poop');
var bell = require('bell');
var nconf = require('nconf');
var async = require('async');

//xml parser starts

var fs = require('fs'),
    xml2js = require('xml2js');

var parser = new xml2js.Parser();
fs.readFile('sitemapscene1.xml', function(err, data) {
var count = 0;
var a=0;
var b=0;
var c=0;
var d=0;
var e=0;
var f=0;
var g=0;
var h=0;
//parsing inside url starts
var parseString = require('xml2js').parseString;

    parser.parseString(data, function (err, result) {
        ff = result;
        dd = ff['urlset']['url'];
        console.log("Total number of urls in this sitemap");
        console.log(dd.length);
        for (i =0; i< dd.length; i++){
         console.log(dd[i]['loc'][0]);

//sending to server to get response
var request = require("request");

request(dd[i]['loc'][0], function(error, response, body) {
    //console.log(response.statuscode);
if (!error && response.statusCode == 200) {

    count=count+1;

   console.log("Number of pages with responese.statuscode 200 is");
    console.log(count); 

    var ddddd = "Number of pages with responese.statuscode 200 is " + count + " " + dd[i]['loc'][0]
// output is wriiten in output.txt file start here

fs = require('fs');
fs.writeFile('output.txt', ddddd, function (err) {
  if (err) return console.log(err);
  console.log('Hello World > output.txt');
});

// output is wriiten in output.txt file ends here

    } else if ( error && response.statusCode == 400 ) {

         a=a+1;
    console.log("Number of pages with responese.statuscode 400 is");
    console.log(a);
    var ddddd = "Number of pages with responese.statuscode 200 is " + count + " " + dd[i]['loc'][0]
// output is wriiten in output.txt file start here

fs = require('fs');
fs.writeFile('output.txt', ddddd, function (err) {
  if (err) return console.log(err);
  console.log('Hello World > output.txt');
});

// output is wriiten in output.txt file ends here

    }else if ( error && response.statusCode == 402 ) {

         b=b+1;
    console.log("Number of pages with responese.statuscode 402 is");
    console.log(b);
    var ddddd = "Number of pages with responese.statuscode 200 is " + count + " " + dd[i]['loc'][0]
// output is wriiten in output.txt file start here

fs = require('fs');
fs.writeFile('output.txt', ddddd, function (err) {
  if (err) return console.log(err);
  console.log('Hello World > output.txt');
});

// output is wriiten in output.txt file ends here

    }else if ( error && response.statusCode == 403 ) {

         c=c+1;
    console.log("Number of pages with responese.statuscode 403 is");
    console.log(c);
    var ddddd = "Number of pages with responese.statuscode 200 is " + count + " " + dd[i]['loc'][0]
// output is wriiten in output.txt file start here

fs = require('fs');
fs.writeFile('output.txt', ddddd, function (err) {
  if (err) return console.log(err);
  console.log('Hello World > output.txt');
});

// output is wriiten in output.txt file ends here

    }else if ( error && response.statusCode == 404 ) {

         d=d+1;
    console.log("Number of pages with responese.statuscode 404 is");
    console.log(d);

    }else if ( error && response.statusCode == 500 ) {

         e=e+1;
    console.log("Number of pages with responese.statuscode 500 is");
    console.log(e);
    var ddddd = "Number of pages with responese.statuscode 200 is " + count + " " + dd[i]['loc'][0]
// output is wriiten in output.txt file start here

fs = require('fs');
fs.writeFile('output.txt', ddddd, function (err) {
  if (err) return console.log(err);
  console.log('Hello World > output.txt');
});

// output is wriiten in output.txt file ends here

    }else if ( error && response.statusCode == 502 ) {

         f=f+1;
    console.log("Number of pages with responese.statuscode 502 is");
    console.log(f);
    var ddddd = "Number of pages with responese.statuscode 200 is " + count + " " + dd[i]['loc'][0]
// output is wriiten in output.txt file start here

fs = require('fs');
fs.writeFile('output.txt', ddddd, function (err) {
  if (err) return console.log(err);
  console.log('Hello World > output.txt');
});

// output is wriiten in output.txt file ends here

    }else if ( error && response.statusCode == 504 ) {

         g=g+1;
    console.log("Number of pages with responese.statuscode 504 is");
    console.log(g);
    var ddddd = "Number of pages with responese.statuscode 200 is " + count + " " + dd[i]['loc'][0]
// output is wriiten in output.txt file start here

fs = require('fs');
fs.writeFile('output.txt', ddddd, function (err) {
  if (err) return console.log(err);
  console.log('Hello World > output.txt');
});

// output is wriiten in output.txt file ends here

    } else   {

         h=h+1;
    console.log("Number of pages with error is");
    console.log(h);
    var ddddd = "Number of pages with responese.statuscode 200 is " + count + " " + dd[i]['loc'][0]
// output is wriiten in output.txt file start here

fs = require('fs');
fs.writeFile('output.txt', ddddd, function (err) {
  if (err) return console.log(err);
  console.log('Hello World > output.txt');
});

// output is wriiten in output.txt file ends here

    }



});

        }
        //console.log("Number of urls,in this site map");
        //console.log(dd.length); //number of urls
    });
});

//xml parser ends

This is the code which crash the server, i think i made mistake in code than server run out of memory

    var hapi = require('hapi');
var glue = require('glue');
var joi = require('joi');
var handlebars = require('handlebars');
var strop = require('string');
var inert = require('inert');
var async = require('async');
var poop = require('poop');
var bell = require('bell');
var nconf = require('nconf');
var async = require('async');

//xml parser starts

var fs = require('fs'),
    xml2js = require('xml2js');

var parser = new xml2js.Parser();
fs.readFile('sitemapscene1.xml', function(err, data) {
var count = 0;
var a=0;
var b=0;
var c=0;
var d=0;
var e=0;
var f=0;
var g=0;
var h=0;
//parsing inside url starts
var parseString = require('xml2js').parseString;

    parser.parseString(data, function (err, result) {
        ff = result;
        dd = ff['urlset']['url'];

        for (i =0; i< dd.length; i++){
         console.log(dd[i]['loc'][0]);
//server reuest and checking
var asyncRequests = [];
var request = require('request');

dd.forEach(function(ddElement){
    asyncRequests.push(function(callback){
        request(ddElement.loc[0], function(error, response, body) {
            if (!error && response.statusCode == 200) {
                return callback(null, 1);
            }

        callback();
        });
    });
});

async.parallel(asyncRequests, function afterStatement(err, results){
    var count = 0;

    for (var i = 0; i < results.length; i ++){
                if (results[i]){
            count += results[i];
        }
    }
});



}
console.log("Total number of urls in this sitemap");
        console.log(dd.length);
        });

    });


//xml parser ends

NodeJS is non-blocking... don't think is a good idea to try 'block' it. http://stackoverflow.com/a/14797359/4838205 — Augusto Goncalves, Jul 21 '16 at 13:19

F. Kauder · Answer 1 · 2016-07-21T14:10:12.560

1

You require async, so why don't you use it ?

var asyncRequests = [];
var request = require('request');

dd.forEach(function(ddElement){
    asyncRequests.push(function(callback){
        request(ddElement.loc[0], function(error, response, body) {
            if (!error && response.statusCode == 200) {
                return callback(null, 1);
            }

        callback();
        });
    });
});

async.parallel(asyncRequests, function afterStatement(err, results){
    var count = 0;

    for (var i = 0; i < results.length; i ++){
        // I don't remember if something is send with callback(), so check data
        if (results[i]){
            count += results[i];
        }
    }
});

And please, next time, send readable and indented code.

EDIT : If you want to count get a object with results associated to the route (I consider ddElement.loc[0] is the route) :

var asyncRequests = [];
var request = require('request');

dd.forEach(function(ddElement){
    asyncRequests.push(function(callback){
        request(ddElement.loc[0], function(error, response, body) {
            callback(error, {
                statusCode : response.statusCode,
                route : ddElement.loc[0]
            });
        });
    });
});

async.parallel(asyncRequests, function afterStatement(err, results){
    if (err){
        console.error(err);
    }

    console.log(results);
});

edited Jul 21 '16 at 14:10

answered Jul 21 '16 at 13:36

F. Kauder

879
5
9

Actually I am trying to parse an xml sitemap file. Take url alone and checking for http response and print which url gets what http response. and count also count total urls in sitemap and number of urls with 200 response,400 response,404 response, 500 response... – sudan kanakavel Jul 21 '16 at 14:02
Thank you for your code,but still i am getting error – sudan kanakavel Jul 21 '16 at 14:09
I add a way to get an object with routes and results. You can make stats with it. And what is your error ? – F. Kauder Jul 21 '16 at 14:11
My error is <--- Last few GCs ---> 24607 ms: Scavenge 1404.5 (1457.3) -> 1404.5 (1457.3) MB, 53.6 / 0 ms (+ 6.7 ms in 1 steps since last GC) [allocation failure] [incremental marking delaying mark-sweep]. 25745 ms: Mark-sweep 1404.5 (1457.3) -> 1404.0 (1457.3) MB, 1138.2 / 0 ms (+ 113.5 ms in 263 steps since start of marking, biggest step 6.8 ms) [last resort gc]. 26824 ms: Mark-sweep 1404.0 (1457.3) -> 1403.9 (1457.3) MB, 1079.2 / 0 ms [last resort gc]. <--- JS stacktrace ---> ==== JS stack trace ========================================= – sudan kanakavel Jul 21 '16 at 14:13
You seems to doesn't have enough memory. Can you post the JS stack trace please ? – F. Kauder Jul 21 '16 at 14:17
Security context: 0x2870cbf36ae1 1: constructor(aka EventEmitter) [events.js:~5] [pc=0x257c998548c] (this=0x1da12cab3c49 ) 2: new constructor(aka Request) [/home/sudan/charmboard_dev/charmboardwebsite/node_modules/request/request.js:~95] [pc=0x257c998f74e] (this=0x1da12cab3c49 ,options=0x1da12cab3cb1 ) 4: request(a... FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory Aborted – sudan kanakavel Jul 21 '16 at 14:20
It's effectively an problem of memory. Your process needs too much memory, so your process is crashing. – F. Kauder Jul 21 '16 at 14:31
I have posted the code which result in server running out of memory.I think I made mistake in code,which result in server running out of memory.because just sending one url and checking http response at a time and printing the result wont bring down server right. – sudan kanakavel Jul 22 '16 at 12:13

How to make node js to wait for function to execute (even if it take time) and then execute the next statement

1 Answers1