Creating a Readable Stream to use with FormData

Question

I'm using an API to upload a CSV file. I create the CSV file in memory from a String and upload it using the request module. However, I'm having trouble creating the Readable Stream from the String. I followed a SO answer on How to create streams from string in Node.Js. Here is my code for that solution:

var importResponse = function(csv, callback){
    stringify(csv, function(err, output){

        const s = new Readable();
        s._read = () => {}; 
        s.push(output);
        s.push(null);

        request.post({
          headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
          url: 'https://ca1.qualtrics.com/API/v3/responseimports',
          formData: {
            surveyId: 'SV_123',
            file: {
                value: s,
                options: {
                    contentType: 'text/csv; charset=utf-8'
                }
            }
          }
        }, function(err, res, body){
            if(err || res.statusCode !== 200){
              console.log(err || "Error status code: " + res.statusCode);
              console.log(body);
              return;
            }
        });
    });

}

The csv variable looks like [["QID1","QID2"],["1","2"]] and the output from stringify looks like "QID1,QID2\n,1,2\n".

This solution gives me the error Unexpected end of input

{"meta":{"httpStatus":"400 - Bad Request","error":{"errorMessage":"Unexpected end of input"}}}

If instead I use memfs, it works fine

const fs = require('memfs');

var importResponse = function(csv, callback){
    stringify(csv, function(err, output){
        // Create file in memory
        fs.writeFileSync('/data.csv', output); 

        request.post({
          headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
          url: 'https://ca1.qualtrics.com/API/v3/responseimports',
          formData: {
            surveyId: 'SV_123',
            file: {
                value: fs.createReadStream('/data.csv'),
                options: {
                    contentType: 'text/csv; charset=utf-8'
                }
            }
          }
        }, function(err, res, body){
            if(err || res.statusCode !== 200){
              console.log(err || "Error status code: " + res.statusCode);
              console.log(body);
              return;
            }
        });
    });

}

How can I convert the output from stringify to a Stream that I can use to upload via the api?

Answer 1

It looks like you're using the request library. You may be coming across this caveat as documented in their readme:

// Pass optional meta-data with an 'options' object with style: {value: DATA, options: OPTIONS}
// Use case: for some types of streams, you'll need to provide "file"-related information manually.
// See the `form-data` README for more information about options: https://github.com/form-data/form-data
custom_file: {
  value:  fs.createReadStream('/dev/urandom'),
  options: {
    filename: 'topsecret.jpg',
    contentType: 'image/jpeg'
  }
}

Since you're using a non-file stream, simply providing a dummy filename should work:

request.post({
  headers: {'X-API-TOKEN':token, 'content-type' : 'multipart/form-data'},
  url: 'https://ca1.qualtrics.com/API/v3/responseimports',
  formData: {
    surveyId: 'SV_123',
    file: {
      value: s,
      options: {
        contentType: 'text/csv; charset=utf-8',
        filename: 'dummy.csv'
      }
    }
  }
}, function(err, res, body){
  if(err || res.statusCode !== 200){
    console.log(err || "Error status code: " + res.statusCode);
    console.log(body);
    return;
  }
});

Answer 2

The sample snippet is incorrect or possibly outdated for current node versions. A really easy way to implement your readable:

const s = new Readable({
  encoding: 'utf8',
  read(size) {
    // Possibly respect the requested size to make for a good consumer experience
    // Otherwise:
    this.push(output, 'utf8');
    this.push(null); // This signals that there's no more data.
  }
});

Here's how you can respect the wishes of the reader:

let data = output;
const s = new Readable({
  encoding: 'utf8',
  read(size) {
    let wantsMore = true;
    while (wantsMore) {
      const chunk = data.slice(0, size);          
      if (!chunk) {
        return void this.push(null);            
      }

      wantsMore = this.push(chunk, 'utf8');
      data = data.slice(size);
    }
  }
});