Streaming file from S3 with Express including information on length and filetype

Question

Using the aws-sdk module and Express 4.13, it's possible to proxy a file from S3 a number of ways.

This callback version will return the file body as a buffer, plus other relevant headers like Content-Length:

function(req,res){

  var s3 = new AWS.S3();

  s3.getObject({Bucket: myBucket, Key: myFile},function(err,data){

    if (err) {
      return res.status(500).send("Error!");
    }

    // Headers
    res.set("Content-Length",data.ContentLength)
       .set("Content-Type",data.ContentType);

    res.send(data.Body); // data.Body is a buffer

  });

}

The problem with this version is that you have to get the entire file before sending it, which is not great, especially if it's something large like a video.

This version will directly stream the file:

function(req,res){

  var s3 = new AWS.S3();

  s3.getObject({Bucket: myBucket, Key: myFile})
    .createReadStream()
    .pipe(res);

}

But unlike the first one, it won't do anything about the headers, which a browser might need to properly handle the file.

Is there a way to get the best of both worlds, passing through the correct headers from S3 but sending the file as a stream? It could be done by first making a HEAD request to S3 to get the metadata, but can it be done with one API call?

Answer 1

One approach is listening the httpHeaders event and creating a stream within it.

s3.getObject(params)
    .on('httpHeaders', function (statusCode, headers) {
        res.set('Content-Length', headers['content-length']);
        res.set('Content-Type', headers['content-type']);
        this.response.httpResponse.createUnbufferedStream()
            .pipe(res);
    })
    .send();

Answer 2

For my project, I simply do a headObject in order to retrieve the object metadata only (it's really fast and avoid to download the object). Then I add in the response all the headers I need to propagate for the piping:

    var s3 = new AWS.S3();

    var params = {
        Bucket: bucket,
        Key: key
    };
    s3.headObject(params, function (err, data) {
        if (err) {
            // an error occurred
            console.error(err);
            return next();
        }
        var stream = s3.getObject(params).createReadStream();

        // forward errors
        stream.on('error', function error(err) {
            //continue to the next middlewares
            return next();
        });

        //Add the content type to the response (it's not propagated from the S3 SDK)
        res.set('Content-Type', mime.lookup(key));
        res.set('Content-Length', data.ContentLength);
        res.set('Last-Modified', data.LastModified);
        res.set('ETag', data.ETag);

        stream.on('end', () => {
            console.log('Served by Amazon S3: ' + key);
        });
        //Pipe the s3 object to the response
        stream.pipe(res);
    });

Answer 3

Building on André Werlang's answer, we have done the following to augment AWS Request objects with a forwardToExpress method:

const _ = require('lodash');
const AWS = require('aws-sdk');

AWS.Request.prototype.forwardToExpress = function forwardToExpress(res, next) {
    this
    .on('httpHeaders', function (code, headers) {
        if (code < 300) {
            res.set(_.pick(headers, 'content-type', 'content-length', 'last-modified'));
        }                            
    })
    .createReadStream()
    .on('error', next)
    .pipe(res);
};    

Then, in our route handlers, we can do something like this:

s3.getObject({Bucket: myBucket, Key: myFile}).forwardToExpress(res, next);

Answer 4

here an 20222 solution using AWS JS SDK and Client. This solution is using s3client to stream the object data to the response stream 'res'. A simple call to GetObjectCommand returns only the raw data of the object but not the content-length neither tags, that is will prior to stream the object data, 1 call is done to get the object data like "ETag", "Content-Type" and 1 optional call is done to get the object tags and forward it to the UI.

It's up to you to customize it for your project.

const { S3, CreateBucketCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteBucketCommand, } = require("@aws-sdk/client-s3");

/**
 * 
 * @param {*} res 
 * @param {string} bucketName Bucket Name
 * @param {string} key Object key
 * @param {number} cacheExpiration Cache expiration in ms
 * @param {boolean} streamTags Forward object tags in http-headers
 */
async function streamGetObject(res, bucketName, key, cacheExpiration, streamTags) {
    try {
        const params = {
            Bucket: bucketName,
            Key: key,
        };
        // Head the object to get classic the bare minimum http-headers information
        const headResponse = await s3Client.send(new HeadObjectCommand(params));
        res.set({
            "Content-Length": headResponse.ContentLength,
            "Content-Type": headResponse.ContentType,
            "ETag": headResponse.ETag,
        });
        // Get the object taggings (optional)
        if (streamTags === true) {
            const taggingResponse = await s3Client.send(new GetObjectTaggingCommand(params));
            taggingResponse.TagSet.forEach((tag) => {
                res.set("X-TAG-" + tag.Key, tag.Value);
            });
        }
        // Prepare cache headers
        if (typeof cacheExpiration === "number") {
            res.setHeader("Cache-Control", "public, max-age=" + cacheExpiration / 1000);
            res.setHeader("Expires", new Date(Date.now() + cacheExpiration).toUTCString());
        } else {
            res.setHeader("Pragma", "no-cache");
            res.setHeader("Cache-Control", "no-cache");
            res.setHeader("Expires", 0);
        }

        // Now get the object data and stream it
        const response = await s3Client.send(new GetObjectCommand(params));
        const stream = response.Body;
        stream.on("data", (chunk) => res.write(chunk));
        stream.once("end", () => {
            res.end();
        });
        stream.once("error", () => {
            res.end();
        });
    } catch (err) {
        console.log("Error", err);
        throw err;
    }
}