Append JSON to a file using node streams

Question

Based on what I have read here, I'm using fs.createWriteStream to write some JSON to a file. I am processing the data in chunks of about 50. So at the beginning of the script, I open my streamand then use a function to pass it in, along with some JSON, which is working pretty well for writing.

const myStream = fs.createWriteStream(
  path.join(RESULTS_DIR, `my-file.json`),
  {
    flags: 'a'
  }
)


function appendJsonToFile(stream, jsonToAppend) {
  return new Promise((resolve, reject) => {
    try {
      stream.write(JSON.stringify(jsonToAppend, null, 2)
      resolve('STREAM_WRITE_SUCCESS')
    } catch (streamError) {
      reject('STREAM_WRITE_FAILURE', streamError)
    }
  })
}

appendJsonToFile(myStream, someJson)

However, because each piece of data to be written is an array of objects, the structure I eventually get in my file will look like this:

[
    {
        "data": "test data 1",
    },
        {
        "data": "test data 2",
    }
][
    {
        "data": "test data 3",
    },
        {
        "data": "test data 4",
    }
]

How can I append these pieces of data so that the result is properly formatted JSON, rather than just a series of arrays?

score 1 · Answer 1 · answered Feb 06 '19 at 11:53

If the file is always formatted as you say above the three things you'd need to do are:

Find out the current length of the file and substract 2 chars (\n], since there's no newline at end),
Remove the first char of the JSON you're saving,
Save the file using r+ mode and with start
End the stream after each save.

Here's the link to createWriteStream options.

Now the other thing is point 4 makes this rather inefficient and questions the whole idea if streaming should be used here. I think it does make sense, but the question here is if you need the file to be readable between writes - if not then you should use a transform stream in the middle and add flush in between the files and after all work is done (beforeExit) you simply end the stream.

You can do this by definition, but I'm the author of a framework called scramjet which makes these cases much easier:

const myStream = new scramjet.DataStream();

const file = path.join(RESULTS_DIR, `my-file.json`)
const start = fs.statSync(file).size - 2;

myStream
    .flatten()
    .toJSONArray()
    .shift(1)
    .pipe(fs.createWriteStream(
        file,
        {flags: 'r+', start}
    ));

function appendJsonToFile(stream, jsonToAppend) {
    return new Promise((resolve, reject) => {
        try {
            stream.write(jsonToAppend)
            resolve('STREAM_WRITE_SUCCESS')
        } catch (streamError) {
            reject('STREAM_WRITE_FAILURE', streamError)
        }
    })
}

appendJsonToFile(myStream, someJson)

process.on('beforeExit', myStream.end());

You can use this as above, but if you'd prefer to work on this with plain node streams this should nudge you in the right direction.

This is really great! Can you show an example using the transform/flush. — 1252748, Jul 22 '21 at 15:17

score 0 · Answer 2 · answered Jul 21 '21 at 03:40

I'll resolved the code, with Error and FILE NOT FOUND handler. Solution derived from Michał Karpacki.

const path = require('path');
const fs = require('fs');

const getFolderPath = () => __dirname || process.cwd();
const getFilePath = (fileName) => path.join(getFolderPath(), `${fileName}`);

/**
 * @param {string} fileName - Included File Name & its Extension
 * @param {Array<*>} arrayData
 * @return {Promise<*>}
 */
const writeFileAsync = async (fileName, arrayData) => {
    const filePath = getFilePath(fileName);

    return new Promise((resolve, reject) => {
        try {
            const _WritableStream = fs.createWriteStream(filePath, {flags: 'r+', start: fs.statSync(filePath).size - 2});
            _WritableStream.write(JSON.stringify(arrayData, null, 2).replace(/\[/, ','), (streamError) => {
                return reject(['STREAM_WRITE_FAILURE', streamError]);
            });
            return resolve('STREAM_WRITE_SUCCESS');
        } catch (streamError) {
            /** ERROR NOT FOUND SUCH FILE OR DIRECTORY !*/
            if (streamError.code === 'ENOENT') {
                fs.mkdirSync(getFolderPath(), {recursive: true});
                return resolve(fs.writeFileSync(filePath, JSON.stringify(
                    Array.from({...arrayData, length: arrayData.length}), null, 2
                )));
            }
            /** ERROR OUT OF BOUND TO FILE SIZE RANGE - INVALID START POSITION FOR WRITE STREAM !*/
            if (streamError instanceof RangeError) {
                console.error(`> [ERR_OUT_OF_RANGE] =>`, streamError);
                const _WritableStream = fs.createWriteStream(filePath, {flags: 'r+'});
                return resolve(_WritableStream.write(JSON.stringify(arrayData, null, 2), (streamError) => {
                    return reject(['STREAM_WRITE_FAILURE', streamError]);
                }));
            }
            return reject(['STREAM_WRITE_FAILURE', streamError]);
        }
    });
};

(() => writeFileAsync('test1.json',
    [{
        key: "value 1"
    }, {
        key: "value 2"
    }]
))();

/* Output after 1st time run =>
[
  {
    "key": "value 1"
  },
  {
    "key": "value 2"
  }
]
*/
/* Output after 2nd time run => 
[
  {
    "key": "value 1"
  },
  {
    "key": "value 2"
  },
  {
    "key": "value 1"
  },
  {
    "key": "value 2"
  }
]
*/

Please provide an explanation of your answer so that the next user knows why this solution worked for you. — Elydasian, Jul 21 '21 at 08:39

Append JSON to a file using node streams

How can I append these pieces of data so that the result is properly formatted JSON, rather than just a series of arrays?

2 Answers2