nodejs looping with async function inside

Question

I'm having a problem where for(var x=1; x < 6; x++) is getting called because too fast axios.get() is async, but I have no idea how to counter that without the solution being too complicated

const axios = require("axios");
const cheerio = require("cheerio");

function imdbGetData(id) {
  var title, show, $;
  var arr = [];
  var airdates = [];
  show = {
    seasons: []
  };

  axios.get(`http://www.imdb.com/title/${id}/`).then((body) => {
    $ = cheerio.load(body.data);
    title = $("div h1").text()
  });
  for(var x=1; x < 6; x++) {
    console.log(x); // Will count too 1,2,3,4,5,6
    url = `http://www.imdb.com/title/${id}/episodes?season=${x}`
    axios.get(url).then((body) => {
      $ = cheerio.load(body.data);
      console.log(x);// 6, 6, 6, 6
      $("div .info .airdate").each(function(index, item) {
        var airdate = String($(this).text());
        airdates.push(airdate.trim());
      });


      $(".info strong a").each(function(i, item){
          var airdate = airdates[i];

          var epsiode_name = $(this).text()
          if (epsiode_name && !epsiode_name.includes("#"))
            arr.push({epsiode_name, airdate});
      });
      show.seasons.push(arr);
      arr = []
      // console.log(show.seasons);
    });
    setTimeout(() => {console.log(show.seasons)}, 10000) // ghetto
  }
}

// season = {
//   seasons: [[ {epsiode_name} ], [{Epsiode name}]]
// }

imdbGetData("tt2193021");

Possible duplicate of [async for loop in node.js](https://stackoverflow.com/questions/21184340/async-for-loop-in-node-js) — Heretic Monkey, Feb 15 '18 at 21:38

score 4 · Answer 1 · answered Feb 15 '18 at 21:41

4

You can construct and push all promises to array, and then use Promise.all(arrayOfPromises). This way you will keep your asynchronous chain and you can easily handle results very similar to regular single asynchronous operation:

var promises = [];
for (var x = 1; x < 6; x++) {
  url = `http://www.imdb.com/title/${id}/episodes?season=${x}`
  promises.push(axios.get(url));
}

Promise.all(promises)
  .then(body => {
    // all results of promises will be in 'body' parameter
  })
  .catch(err => console.error(err));

answered Feb 15 '18 at 21:41

P.S.

15,970
14
62
86

That's good if you have 30 urls, but for the whole EPG it'll flood the webserver. – Michał Karpacki Feb 17 '18 at 08:46
@MichałCzapracki, flood the server? How? This way you make only the number of requests you really need and include to the array of promises. I mean, what's the difference for server, when I send new request in every `then` block and when I send them all at the same time? – P.S. Feb 17 '18 at 10:43
You're openning a lot of connections and try to access lots of data at the same time - this goes to the server that usually has a limited threadpool or even if not, it will use a lot of resources for that. You should control how many requests you have pending so that you do 10-20 simultaneous calls. Promise.all will not solve that, because your requests are already made before Promise.all is called – Michał Karpacki Feb 18 '18 at 20:40

Terry Lennox · Accepted Answer · 2018-02-15T22:10:41.223

You can also use async/await (in newer versions of Node.js), so you can make the code a little easier to read, I've made a few little changes to update progress too.

const axios = require("axios");
const cheerio = require("cheerio");

async function imdbGetData(id) {

    var title, show, $;
    var arr = [];
    var airdates = [];
    show = {
    seasons: []
    };

    console.log('Getting from ' + `http://www.imdb.com/title/${id}/`);
    let body = await axios.get(`http://www.imdb.com/title/${id}/`);

    $ = cheerio.load(body.data);
    title = $("div h1").text()

    for(var x=1; x < 6; x++) {
        console.log('Getting season: ' + x); // Will count too 1,2,3,4,5,6
        url = `http://www.imdb.com/title/${id}/episodes?season=${x}`
        let body = await axios.get(url);
        $ = cheerio.load(body.data);
        $("div .info .airdate").each(function(index, item) {
            var airdate = String($(this).text());
            airdates.push(airdate.trim());
        });

        $(".info strong a").each(function(i, item){
            var airdate = airdates[i];

            var epsiode_name = $(this).text()
            if (epsiode_name && !epsiode_name.includes("#"))
                arr.push({epsiode_name, airdate});
        });
        show.seasons.push(arr);
        arr = []

    }

    console.log("Result: ", show.seasons);
}

imdbGetData("tt2193021");

While this answer is correct for the most recent version of node, it fails to illuminate the cause of the problem encountered by the OP, which is that you cannot instrument asynchronous calls as if they were synchronous because you cannot predict when an async call will complete. I think the next answer, while incomplete, provides the basis for a better solution since it shows how multiple parallel operations can be managed using `Promise.all()`. — Rob Raisch, Feb 16 '18 at 00:49

score 0 · Answer 3 · answered Jun 21 '19 at 13:09

You can simply use ES6 let instead of var , your code will be:

for(let i=0; i<length; i++){
   asyncCall(function(){
    console.log(i);// will print 0,1,2,3,...
    });
}

Please check this article https://codeburst.io/asynchronous-code-inside-an-array-loop-c5d704006c99

nodejs looping with async function inside

3 Answers3