Using multiple page.open in one script

Question

My goal is open many pages(with a short delay) and save my data to a file.

But my code does not work.

var gamesList = [url1,url2,url3];
//gamesList is getting from a file

var urls = [];
var useragent = [];
useragent.push('Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14');
useragent.push('Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50');

var page = require('webpage').create();
page.settings.userAgent = useragent[Math.floor(Math.random() * useragent.length)];
console.log('Loading a web page');


function handle_page(url){
    page.open(url,function(){
        //...
        var html= page.evaluate(function(){
            // ...do stuff...
            page.injectJs('jquery.min.js');
            return $('body').html();
        });
        //save to file
        var file = fs.open('new_test.txt', "w");
        file.write(html + '\n');
        file.close();    

        console.log(html);

        setTimeout(next_page,1000);
    });
}

function next_page(urls){
    var url=urls.shift();
    if(!urls){
        phantom.exit(0);
    }
    handle_page(url);
}

next_page(urls);
phantom.exit();

Does it matter where I am writing phantom.exit();? If I am writing it in the page.open() callback in the end then the 1st page opens well.

Answer 1

Your idea of opening multiple pages with recursion is correct, but you have some problems.

Exit

As you correctly noted, you have a problem with phantom.exit(). Since page.open() and setTimeout() are asynchronous, you only need to exit when you are done. When you call phantom.exit() at the end of the script, you're exiting before the first page is even loaded.

Simply remove that last phantom.exit(), because you already have another exit at the correct place.

Page context

page.evaluate() provides access to the DOM context (page context). The problem is that it is sandboxed. Inside of that callback you have no access to variables defined outside. You can explicitly pass variables in, but they have to be primitive objects which page is not. You simply have to access to page inside of page.evaluate(). You need to inject jQuery before calling page.evaluate().

Files

You're overwriting the file in every iteration by not changing the file name. Either you need to change the filename or use the appending mode 'a' instead of 'w'.

Then you don't need to open a stream when you simply want to write once. Change:

var file = fs.open('new_test.txt', "w");
file.write(html + '\n');
file.close();

to

fs.write('new_test.txt', html + '\n', 'a');

Recursive step

The recursive step with calling the next_page() function requires that you pass in the urls. Since urls is already a global variable and you change it in each iteration, you don't need to pass in the urls.

You also don't need to add a setTimeout(), because everything before inside of the page.open() callback was synchronous.

Fixed Script

//...
var urls = [/*....*/];

function handle_page(url){
    page.open(url, function(){
        //...
        page.injectJs('jquery.min.js');
        var html = page.evaluate(function(){
            // ...do stuff...
            return $('body').html();
        });
        //save to file
        fs.write('new_test.txt', html + '\n', 'a');

        console.log(html);

        next_page();
    });
}

function next_page(){
    var url = urls.shift();
    if(!url){
        phantom.exit(0);
    }
    handle_page(url);
}

next_page();

Answer 2

Explanation above is very helpful to me. So, thanx a lot. Come to the point, sometimes js function renders even after the page has already loaded, in this scenario setTimeout() method, is very helpful. And,I faced problem like this when scraping many site....I used setTimeout() method in this way,

`

        function handle_page(url){page.open(url, function() {setTimeout(function() {var html_text=page.evaluate(function(){ 
            var is= document.querySelectorAll("li.bookinDetails_c1_180616")[0].textContent;
            var isbn=is.trim();
            //return s1;
            var w,x,y,z,z1,w1,u,a;
            a= document.querySelectorAll("li.selectableBook");
            if(a.length==5){
                w1=document.querySelectorAll("span.bookPrice")[0].textContent;
                w=w1.trim();
                x1=document.querySelectorAll("span.bookPrice")[1].textContent;
                x=x1.trim();
                y1=document.querySelectorAll("span.bookPrice")[2].textContent;
                y=y1.trim();
                z1=document.querySelectorAll("span.bookPrice")[3].textContent;
                z=z1.trim();
                u=isbn+"=>["+"RENT USED:-"+w+","+"RENT NEW:-"+x+","+"BUY USED:-"+y+","+"BUY NEW:-"+z+"]";
                return u;
            }else{
                y1=document.querySelectorAll("span.bookPrice")[0].textContent;
                y=y1.trim();
                z1=document.querySelectorAll("span.bookPrice")[1].textContent;
                z=z1.trim();
                u=isbn+"=>["+"BUY USED:-"+y+","+"BUY NEW:-"+z+"]";
                return u;
            }

        });
        fs.write('html.txt',html_text+'\r\n','a');
        next_page();
    }, 400);

});
}

`

Answer 3

I ran into a similar problem recently. I found an answer here：https://stackoverflow.com/questions/34120421/scraping-multiple-urls-by-looping-in-phantomjs. But it did not work for me. I think recursion should be used instead of loop. But I don't know how to write the code. Fortunately, I found a solution here. These answers above are very helpful to me. I will post my code here, hoping to help others in the future.

var urls = new Array("1.html", "2.html", "3.html");
var page = new WebPage();
var fs = require('fs');
var count = 1;

function handle_page(url) {
    page.open(url, function () {
        setTimeout(function () {
            var ct = page.evaluate(function () {
                return document.getElementsByClassName('content');
            });
            var fn = '00' + count + '.html';
            console.log(fn);
            try {
                fs.write(fn, ct[0].textContent, 'w');
            } catch (e) {
                console.log(e);
            };
            count += 1;
            next_page();
        }, 1000);
    });
}

function next_page() {
    var url = urls.shift();
    if (!url) {
        phantom.exit(0);
    }
    handle_page(url);
}

next_page();