How to insert data in json after crawling through casperjs?

Question

I wrote code that parsing a lot of words (innerHTML) from some webpages.

and I'd like to insert data to json file directly..

Here is my js code...

var words = [];
var casper = require('casper').create();

    function getWords() {
        var words = document.querySelectorAll('td.subject a');
        return Array.prototype.map.call(words, function(e) {
            return e.innerHTML;
        });
    }


        casper.start('http://www.todayhumor.co.kr/board/list.php?table=bestofbest', function() {                
            words = this.evaluate(getWords);
        });

        for (var i=2; i <=5; i++) {
        casper.thenOpen('http://www.todayhumor.co.kr/board/list.php?table=bestofbest&page='+i, function() {              

            words = words.concat(this.evaluate(getWords));
        });
        }

    casper.run(function() {
        // echo results in some pretty fashion
        this.echo(words.length + ' links found:').exit();
        this.echo(words.join('\n')).exit();
});

and

I run this code through terminal like this!

username@wow:~/workspace/app/assets/javascripts $ casperjs application.js

and the result is (for example)

150 words found:
apple
banana
melon
kiwi
citrus
watermelon
passionfruit
mango
orange
...

So I want to insert this data in "word" part of my json file (example code of json below)

and make other columns("type": "fruit" and "spell":) automatically added

{ "my_initial_words": [
    {
    "type": "fruit",
    "word": "apple",
    "spell": "ap"
    },
    {
    "type": "fruit",
    "word": "banana",
    "spell": "ba"
    },
    {
    "type": "fruit",
    "word": "melon",
    "spell": "me"
    }   

]
}
----------------------------------------------------------------------------

thanks for adding more answer!.. but I couldn't catch where should I put these code

Could you tell me once more that... Which code you gave me executes "Saving the results to JSON file?" because I have to read json file(makeyourap.json) in my seeds.rb file like this

require 'json'
file = File.open(Rails.root.join('db','makeyourap.json'))
contents = file.read
json = ActiveSupport::JSON.decode(contents)["my_initial_words"]

Answer 1

So, something like this?

function makeTypeObject(name, type) {
  return {
    name: name,
    type: type,
    spell: name.substr(0,2)
  };
}

var wordDesc = words.map(function (word) { 
   return makeTypeObject(word, "fruit"); 
});

var finalObject = {
  my_initial_words: wordDesc
};

var jsonString = JSON.stringify(finalObject);
// if you want prettyprint, try JSON.stringify(finalObject, null, "\t");

I hope this helps.

Answer 2

Write to file via casper

If you want to have a file from which you read and write, appending content, you can do it like this:

var fs = require('fs');
var FILENAME = 'makeyourap.json';
function add_new_fruits(fruits) {
    var data;
    if ( fs.isFile(FILENAME) ) {
        data = fs.read(FILENAME);
    } else {
        data = JSON.stringify({'my_initial_words' : [] });
    }
    var json = JSON.parse(data);
    fruits.forEach(function(word) {
        json.my_initial_words.push({"type": "fruit",
                                    "name": word,
                                    "spell": word.slice(0,2)});
    });
    data = JSON.stringify(json, null, '\t');
    fs.write(FILENAME, data, "w");
}

Use this instead of the older this.echo. Just call it as

casperjs application.js

This either reads the object from a file, or creates it if it does not exist. Then, it appends each new object from the new fruits (including duplicates), and writes it back to FILENAME.

Previous approach: how to roll your own

create Object

So first, you want to create an object that only has the parameter my_initial_words with values as above.

You can create a function via

function createFinal(wordArray) {
    var out = [];
    wordArray.forEach(function(word) {
        out.push({"type": "fruit", "name": word, "spell": word.slice(0,2)});
    });
    return out;
}

to create the array. Then, create the object via

var my_object = { "my_initial_words": createFinal(words) };

to JSON

Javascript has a built-in JSON-object. With a javascript-object like

var my_object = { "my_initial_words": ...

as above, use

JSON.stringify(my_object) 

to get the JSON representation to write.

Older: write to file via redirection

Before, you had

this.echo(words.join('\n')).exit();

which gave you the basic list. Using this.echo, try replacing this by

var my_object = { "my_initial_words": createFinal(words) };
this.echo(JSON.stringify(my_object)).exit();

This prints to standard output. Just remove the other this.echo line (150 words found) and redirect the output via

casperjs application.js > makeyourap.json

If you want to write to file in casperjs, look at write-results-into-a-file-using-casperjs.