Nodejs - read line by line from file, perform async action for each line and resume

Question

I'm trying to read a file line by line, perform some action that has a callback and when the function finishes to resume line reading. For example:

var fs = require('fs');
var readline = require('readline');
var stream = require('stream');
var instream = fs.createReadStream('./phrases.txt');
var outstream = new stream;
var rl = readline.createInterface(instream, outstream);
rl.on('line', function (line) {
  rl.pause();
  setTimeout(function () {
    console.log('resuming');
    rl.resume();
  }, 2000);
});

I was under the impression the example above should basically read a line, wait for 2 seconds, console.log and then continue to the next line. What really happens is that it waits for the first 2 seconds and then spews out lots of console.log

Answer 1

Line by Line module helps you reading large text files, line by line, without buffering the files into memory.

You can process the lines asynchronously. This is the example provided:

var LineByLineReader = require('line-by-line'),
    lr = new LineByLineReader('big_file.txt');

lr.on('error', function (err) {
    // 'err' contains error object
});

lr.on('line', function (line) {
    // pause emitting of lines...
    lr.pause();

    // ...do your asynchronous line processing..
    setTimeout(function () {

        // ...and continue emitting lines.
        lr.resume();
    }, 100);
});

lr.on('end', function () {
    // All lines are read, file is closed now.
});

Answer 2

Solution without installing any external library. You only need the native node.js "readline" module. Just do the following:

import fs from "fs";
import readline from "readline";

const readInterface = readline.createInterface({
  input: fs.createReadStream(path.join(__dirname, 'myfile.txt'))
});

for await (const line of readInterface){
  await someAsynchronousOperation();
}

Source (Official documentation): https://nodejs.org/api/readline.html#rlsymbolasynciterator

Answer 3

A very nice line-reader module exists, https://github.com/nickewing/line-reader

simple code:

 var lineReader = require('line-reader');
   lineReader.eachLine('file.txt', function(line, last) {
      // do whatever you want with line...
      console.log(line);
      if(last){
         // or check if it's the last one
      }
   });

also "java-style" interface for more control:

lineReader.open('file.txt', function(reader) {
  if (reader.hasNextLine()) {
    reader.nextLine(function(line) {
      console.log(line);
    });
  }
});

Another cool solution:

var fs = require('fs'),
    sleep = require('sleep'),
    readline = require('readline');

var rd = readline.createInterface({
    input: fs.createReadStream('phrases.txt'),
    output: process.stdout,
    terminal: false
});

rd.on('line', function(line) {
    console.log('-------')
    console.log(line);
    sleep.sleep(2)

});

Answer 4

function createLineReader(fileName){
    var EM = require("events").EventEmitter
    var ev = new EM()
    var stream = require("fs").createReadStream(fileName)
    var remainder = null;
    stream.on("data",function(data){
        if(remainder != null){//append newly received data chunk
            var tmp = new Buffer(remainder.length+data.length)
            remainder.copy(tmp)
            data.copy(tmp,remainder.length)
            data = tmp;
        }
        var start = 0;
        for(var i=0; i<data.length; i++){
            if(data[i] == 10){ //\n new line
                var line = data.slice(start,i)
                ev.emit("line", line)
                start = i+1;
            }
        }
        if(start<data.length){
            remainder = data.slice(start);
        }else{
            remainder = null;
        }
    })

    stream.on("end",function(){
        if(null!=remainder) ev.emit("line",remainder)
    })

    return ev
}


//---------main---------------
fileName = process.argv[2]

lineReader = createLineReader(fileName)
lineReader.on("line",function(line){
    console.log(line.toString())
    //console.log("++++++++++++++++++++")
})

Answer 5

Here is a simple solution in typescript using line-reader that can run in nodejs 8:

import lineReader from 'line-reader';

function readLines(filename: string, processLine: (line: string) => Promise<void>): Promise<void> {
  return new Promise((resolve, reject) => {
    lineReader.eachLine(filename, (line, last, callback) => {
      if (!callback) throw new Error('panic');
      processLine(line)
        .then(() => last ? resolve() : callback())
        .catch(reject);
    });
  });
}

async function echo(): Promise<void> {
  await readLines('/dev/stdin', async (line) => {
    console.log(line);
  });
}

echo();

Note that it does not buffer the whole file before executing, therefore it is suitable for processing large text files.

Answer 6

I suggest to use stdio for this kind of things, as input stream is paused and resumed automatically and you don't need to worry about your system resources. You'll be able to read really huge files with just a few MBs of memory:

This example prints a line every 2 seconds:

$ node myprogram.js < file.txt

import { read } from 'stdio';

async function onLine (line) {
  console.log(line);
  await sleep(2000);
}

read(onLine)
  .then(() => console.log('finished'));

Note I'm using an asynchronous sleep to represent any asynchronous task. It is not included in Node.js by default but it would be as follows:

const sleep = (delay) => new Promise((resolve) => setTimeout(resolve, delay));

Answer 7

const readline = require('readline');
const fs = require('fs');

const rl = readline.createInterface({
  input: fs.createReadStream('sample.txt')
});

rl.on('line', (line) => {
  console.log(`Line from file: ${line}`);
});

source: https://nodejs.org/api/readline.html#readline_example_read_file_stream_line_by_line