How to read a file line by line in Dart

Question

This question is a continuation of a previous question. I wrote the following piece of code to determine if File.openRead() created a Stream that could be streamed line-by-line. It turns out that the answer is no. The entire file is read and then passed to the next transform. My question is then: How do you Stream a file line-by-line in Dart?

import 'dart:async';
import 'dart:convert';
import 'dart:io';


void main(List<String> arguments) {

  Stream<List<int>> stream = new File('Data.txt').openRead();

   stream
      .transform(const Utf8InterceptDecoder())
        .transform(const LineSplitterIntercept())
          .listen((line) {
//            stdout.writeln(line);
          }).asFuture().catchError((_) => print(_));
}

int lineSplitCount = 0;

class LineSplitterIntercept extends LineSplitter {

  const LineSplitterIntercept() : super();
  // Never gets called
  List<String> convert(String data) {
    stdout.writeln("LineSplitterIntercept.convert : Data:" + data);
    return super.convert(data);
  }

  StringConversionSink startChunkedConversion(ChunkedConversionSink<String> sink) {
    stdout.writeln("LineSplitterIntercept.startChunkedConversion Count:"+lineSplitCount.toString()+ " Sink: " + sink.toString());
    lineSplitCount++;
    return super.startChunkedConversion(sink);
  }
}

int utfCount = 0;

class Utf8InterceptDecoder extends Utf8Decoder {

  const Utf8InterceptDecoder() : super();

  //never gets called
  String convert(List<int> codeUnits) {
    stdout.writeln("Utf8InterceptDecoder.convert : codeUnits.length:" + codeUnits.length.toString());
    return super.convert(codeUnits);
  }


  ByteConversionSink startChunkedConversion(ChunkedConversionSink<String> sink) {
    stdout.writeln("Utf8InterceptDecoder.startChunkedConversion Count:"+ utfCount.toString() + " Sink: "+ sink.toString());
    utfCount++;
    return super.startChunkedConversion(sink);
  }
}

Answer 1

I think this code is useful:

import 'dart:io';
import 'dart:convert';
import 'dart:async';

main() {
  final file = new File('file.txt');
  Stream<List<int>> inputStream = file.openRead();

  inputStream
    .transform(utf8.decoder)       // Decode bytes to UTF-8.
    .transform(new LineSplitter()) // Convert stream to individual lines.
    .listen((String line) {        // Process results.
        print('$line: ${line.length} bytes');
      },
      onDone: () { print('File is now closed.'); },
      onError: (e) { print(e.toString()); });
}

Answer 2

If a stream is necessary, you can create it from the future that readAsLines() returns:

   Stream<List<String>> stream = 
      new Stream.fromFuture(new File('Data.txt').readAsLines());

However it looks simpler to me to plainly process the lines one by one,

  List<String> lines = new File('Data.txt').readAsLinesSync();
  for (var line in lines) {
    stdout.writeln(line); 
  } 

Answer 3

The converter's startChunkedConversion is only called once, when the transformation is started. However, the returned sink's add method is invoked multiple times with parts of the file.

It's up to the source to decide how big the chunks are, but a 37MB file (as mentioned in your previous question) will definitely be sent in smaller chunks.

If you want to see the chunks you can either intercept startChunkedConversion and return a wrapped sink, or you can put yourself between the openRead and the transformer.

Intercept:

class InterceptSink {
  static int lineSplitCount = 0;

  final _sink;
  InterceptSink(this._sink);
  add(x) {
    print("InterceptSink.add Count: $lineSplitCount");
    lineSplitCount++;
    _sink.add(x);
  }
  close() { _sink.close(); }
}

class LineSplitterIntercept extends Converter {
  convert(x) { throw "unimplemented"; }
  startChunkedConversion(outSink) {
    var lineSink = new LineSplitter().startChunkedConversion(outSink);
    return new InterceptSink(lineSink);
  }
}

After openRead:

file.openRead()
  .transform(UTF8.decoder)
  .map(x) {
    print("chunk size: ${x.length)");
    return x;
  }
  .transform(new LineSplitter())
  ...

Answer 4

Because none of the other answers suited my situation, here is another technique:

import 'dart:io';
import 'dart:convert';


void main()
{
    var file = File('/path/to/some/file.txt');
    var raf = file.openSync(mode: fileMode.read);
 
    String line;
    while ((line = readLine(raf)) != null)
    {
        print(line);
    }
  }

  String readLine(RandomAccessFile raf, {String lineDelimiter = '\n'}) {
 
    var line = '';
    int byte;
    var priorChar = '';

    var foundDelimiter = false;

    while ((byte = raf.readByteSync()) != -1) {
      var char = utf8.decode([byte]);

      if (isLineDelimiter(priorChar, char, lineDelimiter)) {
        foundDelimiter = true;
        break;
      }

      line += char;
      priorChar = char;
    }
    if (line.isEmpty && foundDelimiter == false) {
      line = null;
    }
    return line;
  }


  bool isLineDelimiter(String priorChar, String char, String lineDelimiter) {
    if (lineDelimiter.length == 1) {
      return char == lineDelimiter;
    } else {
      return priorChar + char == lineDelimiter;
    }
  }

Answer 5

Adjusting Brett Sutton's answer for sound null safety and wider availability:

import 'dart:io';
import 'dart:convert';

bool isLineDelimiter(String priorChar, String char, String lineDelimiter) 
{
  if (lineDelimiter.length == 1) {
    return char == lineDelimiter;
  } else {
    return priorChar + char == lineDelimiter;
  }
}

/// Reads one line and returns its contents.
///
/// If end-of-file has been reached and the line is empty null is returned.
String? readLine(RandomAccessFile raf,
    {String lineDelimiter = '\n', void Function()? onEOF}) {
  String line = '';
  int byte;
  String priorChar = '';
  byte = raf.readByteSync();
  while (byte != -1) {
    String char = utf8.decode([byte]);
    if (isLineDelimiter(priorChar, char, lineDelimiter)) return line;
    line += char;
    priorChar = char;
    byte = raf.readByteSync();
  }
  onEOF?.call();
  if (line.isEmpty) return null;
  return line;
}

EDIT 1:

I wanted to add some more line-specific functions I made:

/// Skips one line and returns the last byte read.
///
/// If end-of-file has been reached -1 is returned.
int skipLine(RandomAccessFile raf,
    {String lineDelimiter = '\n', void Function()? onEOF}) {
  int byte;
  String priorChar = '';
  byte = raf.readByteSync();
  while (byte != -1) {
    String char = utf8.decode([byte]);
    if (isLineDelimiter(priorChar, char, lineDelimiter)) return byte;
    priorChar = char;
    byte = raf.readByteSync();
  }
  return byte;
}

/// Reads all lines in the file and executes [onLine] per each.
///
/// If [onLine] returns true the function terminates.
void processLines(
  RandomAccessFile raf, {
  String lineDelimiter = '\n',
  required bool? Function(String line, bool eofReached) onLine,
}) {
  bool _eofReached = false;
  do {
    String? _line;
    _line = readLine(raf,
        lineDelimiter: lineDelimiter, onEOF: () => _eofReached = true);
    if (_line == null) return;
    if (onLine(_line, _eofReached) == true) return;
  } while (!_eofReached);
}