Holding loop till i get resonse from API

Question

I want scrape youtube channel i have list 1500 channel list. I want to scrape it one by one through the node. The problem is in the loop I want to stop loop until getting response and process response of the current channel.

async and await

const cheerio = require('cheerio');
const request = require('request');
var fs = require('fs');

async function getTitle(){
    var data =  await fs.readFileSync('./channels.json');
    var Parsed_Data = await JSON.parse(data);
    console.log(Parsed_Data);
    for (x of Parsed_Data){
        // var url = 'https://www.youtube.com/channel/' + x.id + '/about';
        // console.log(url);
        request( 'https://www.youtube.com/channel/' + x.id + '/about', function(err, res, body) {
        let $ =  cheerio.load(body);
        console.log($);
        let title =  $('title');
        console.log(title.text());
        }

    )};

want the title of each channel

`fs.readFileSync` and `JSON.parse` are both synchronous. The only asynchronous thing here is request. — pguardiario, Jan 16 '19 at 23:39

score 0 · Accepted Answer · answered Jan 16 '19 at 11:29

You can do:

function getBody(url) {
    return new Promise((resolve, reject) => {
        request( 'https://www.youtube.com/channel/' + x.id + '/about', function(err, res, body) {
            if (err) {
                return reject(err);
            }
            resolve(body);
        });
    })
}

async function getTitle(){
    var data =  await fs.readFileSync('./channels.json');
    var Parsed_Data = await JSON.parse(data);
    console.log(Parsed_Data);
    for (x of Parsed_Data){
        const body = await getBody('https://www.youtube.com/channel/' + x.id + '/about');

        let $ =  cheerio.load(body);
        console.log($);
        let title =  $('title');
        console.log(title.text());

)};

Alvin Lau · Answer 2 · 2019-01-16T11:37:11.137

First, install request-promise:

npm i request-promise

Then, change your code into this:

const cheerio = require('cheerio');
const rp = require('request-promise');
var fs = require('fs');

async function getTitle(){
    var data =  await fs.readFileSync('./channels.json');
    var Parsed_Data = JSON.parse(data);
    console.log(Parsed_Data);
    for (x of Parsed_Data){
        // var url = 'https://www.youtube.com/channel/' + x.id + '/about';
        // console.log(url);
        let body=await rp( 'https://www.youtube.com/channel/' + x.id + '/about', 
        let $ =  cheerio.load(body);
        console.log($);
        let title =  $('title');
        console.log(title.text());
    }
}

Actually, you don't need an await next to JSON.parse(data);

actually you may not need this package – brk Jan 16 '19 at 11:36 — brk, Jan 16 '19 at 11:36
Yes, you're right but it can make things clean. – Alvin Lau Jan 16 '19 at 11:38 — Alvin Lau, Jan 16 '19 at 11:38

Holding loop till i get resonse from API

2 Answers2