Node.js: chunked transfer encoding

Question

Is that code valid HTTP/1.1?

var fs = require('fs')
var http = require('http')

var buf=function(res,fd,i,s,buffer){
 if(i+buffer.length<s){
  fs.read(fd,buffer,0,buffer.length,i,function(e,l,b){
   res.write(b.slice(0,l))
   //console.log(b.toString('utf8',0,l))
   i=i+buffer.length
   buf(res,fd,i,s,buffer)
  })
 }
 else{
  fs.read(fd,buffer,0,buffer.length,i,function(e,l,b){
   res.end(b.slice(0,l))
   fs.close(fd)
  })
 }
}

var app = function(req,res){
 var head={'Content-Type':'text/html; charset=UTF-8'}
 switch(req.url.slice(-3)){
  case '.js':head={'Content-Type':'text/javascript'};break;
  case 'css':head={'Content-Type':'text/css'};break;
  case 'png':head={'Content-Type':'image/png'};break;
  case 'ico':head={'Content-Type':'image/x-icon'};break;
  case 'ogg':head={'Content-Type':'audio/ogg'};break;
  case 'ebm':head={'Content-Type':'video/webm'};break;
 }
 head['Transfer-Encoding']='chunked'
 res.writeHead(200,head)
 fs.open('.'+req.url,'r',function(err,fd){
  fs.fstat(fd,function(err, stats){
   console.log('.'+req.url+' '+stats.size+' '+head['Content-Type']+' '+head['Transfer-Encoding'])
   var buffer = new Buffer(100)
   buf(res,fd,0,stats.size,buffer)
  })
 })
}

http.createServer(app).listen(8000,"127.0.0.1")
console.log('GET http://127.0.0.1:8000/appwsgi/www/index.htm')

I think I am violating HTTP/1.1 here? Text files do seem to work fine, but that could be coincidental. Is my header "200 OK" or need it to be "100"? Is one header sufficient?

score 15 · Answer 1 · answered Jun 26 '12 at 14:53

15

Since Node.js implicitly sets 'Transfer-Encoding: chunked', all I needed to send in headers was the content type with charset like:

'Content-Type': 'text/html; charset=UTF-8'

Initially it was:

'Content-Type': 'text/html'

... which didn't work. Specifying "charset=UTF-8" immediately forced Chrome to render chunked responses.

answered Jun 26 '12 at 14:53

esengineer

9,514
7
45
69

2

with the explicit charset my code worked, but I noticed that it doesn't work with text/plain content-type (tested on chrome)... weird behavior... – Francesco Casula Oct 16 '12 at 12:23
I'm not very sure about why you've got this behavior. I'll test this and reply later. – esengineer Oct 22 '12 at 12:33
in my tests it works on firefox, but not on chrome (only for text/plain content-type)... – Francesco Casula Oct 22 '12 at 17:11
3

@fra_casula I had the same problem. It seems to be a bug in WebKit: https://code.google.com/p/chromium/issues/detail?id=156023 There's a workaround to add "X-Content-Type-Options: nosniff" header. – Dmitri Shuralyov Nov 22 '13 at 09:38
Pretty old, but does one know, if "end" event fires after "error"? – decades Dec 27 '22 at 09:51
Maybe things have changed over the years but adding `charset=UTF-8` is not making Chrome display content as it arrives. – Brian White Feb 13 '23 at 18:51

score 14 · Accepted Answer · edited Jun 10 '14 at 23:11

If you're doing chunked transfer encoding, you actually need to set that header:

Transfer-Encoding: chunked

You can see from the headers returned by google, which does chunked transfers for the homepage and most likely other pages:

HTTP/1.1 200 OK
Date: Sat, 04 Jun 2011 00:04:08 GMT
Expires: -1
Cache-Control: private, max-age=0
Content-Type: text/html; charset=ISO-8859-1
Set-Cookie: PREF=ID=f9c65f4927515ce7:FF=0:TM=1307145848:LM=1307145848:S=fB58RFtpI5YeXdU9; expires=Mon, 03-Jun-2013 00:04:08 GMT; path=/; domain=.google.com
Set-Cookie: NID=47=UiPfl5ew2vCEte9JyBRkrFk4EhRQqy4dRuzG5Y-xeE---Q8AVvPDQq46GYbCy9VnOA8n7vxR8ETEAxKCh-b58r7elfURfiskmrOCgU706msiUx8L9qBpw-3OTPsY-6tl; expires=Sun, 04-Dec-2011 00:04:08 GMT; path=/; domain=.google.com; HttpOnly
Server: gws
X-XSS-Protection: 1; mode=block
Transfer-Encoding: chunked

EDIT Yikes, that read is way too complicated:

var app = function(req,res){
 var head={'Content-Type':'text/html'}
 switch(req.url.slice(-3)){
  case '.js':head={'Content-Type':'text/javascript'};break;
  case 'css':head={'Content-Type':'text/css'};break;
  case 'png':head={'Content-Type':'image/png'};break;
  case 'ico':head={'Content-Type':'image/x-icon'};break;
  case 'ogg':head={'Content-Type':'audio/ogg'};break;
  case 'ebm':head={'Content-Type':'video/webm'};break;
 }
 res.writeHead(200,head)
 var file_stream = fs.createReadStream('.'+req.url);
 file_stream.on("error", function(exception) {
   console.error("Error reading file: ", exception);
 });
 file_stream.on("data", function(data) {
   res.write(data);
 });
 file_stream.on("close", function() {
   res.end();
 });
}

There you go, a nice streamed buffer for you to write with. ~~Here's a blog post I wrote on different ways to read in files. I recommend looking that over so you can see how to best work with files in node's asynchronous environment.~~

This is confusing because does the response need to be a 200 when chunked packages are being send? — Gert Cuykens, Jun 04 '11 at 00:20
@Gert read the headers from google I posted in my response that has a 200 OK in them. Of course, if you're redirecting or something else outside of a "yes I have this file right here right now and am going to send it to you" type situation, you should use the appropriate header instead. I recommend taking a look at all of the headers and what they stand for: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html . This will be beneficial to you if you want to continue to use node.js on a low level like this. — onteria_, Jun 04 '11 at 00:25

score 4 · Answer 3 · answered Jun 03 '11 at 23:31

Why are you doing all the fs operations manually? You'd probably be better off using the fs.createReadStream() function.

On top of that, my guess is that Chrome is expecting you to return a 206 response code. Check req.headers.range, and see if Chrome is expecting a "range" of the media file to be returned. If it is, then you will have to only send back the portion of the file requested by the web browser.

By why reinvent the wheel? There's tons of node modules that do this sort of thing for you. Try Connect/Express' static middleware. Good luck!

After fixing the code, it plays the media only once. I guess you are right about the range response. — Gert Cuykens, Jun 05 '11 at 01:44

Node.js: chunked transfer encoding

3 Answers3

Linked