OK, since one obviously gets downvoted for providing just a working example, let me Elaborate:
First, you don't do 
textWriter.Flush()
and expect the Content of textwriter to have been flushed to memorystream.
Then you don't do
memoryStream.Position = 0
And expect the memorystream to be "written" from Position 0.
Then you do
memoryStream.Write(bytesInStream, 0, bytesInStream.Length);
but what you actually mean is
memoryStream.Read(bytesInStream, 0, CInt(memoryStream.Length))
You also missed that length is Long, while read uses an integer, so you can get an exception there.
So this is your code minimally adapted to "work" (i copied it into a vb Project)
Imports System.Web
Imports System.Web.Services
Public Class TextHandler
    Implements System.Web.IHttpHandler
    Sub ProcessRequest(ByVal context As HttpContext) Implements IHttpHandler.ProcessRequest
        'context.Response.ContentType = "text/plain"
        'context.Response.Write("Hello World!")
        Dim memoryStream As New System.IO.MemoryStream()
        Dim textWriter As System.IO.TextWriter = New System.IO.StreamWriter(memoryStream)
        textWriter.WriteLine("Something")
        textWriter.Flush()
        memoryStream.Position = 0
        Dim bytesInStream As Byte() = New Byte(memoryStream.Length - 1) {}
        'memoryStream.Write(bytesInStream, 0, bytesInStream.Length)
        memoryStream.Read(bytesInStream, 0, CInt(memoryStream.Length))
        memoryStream.Close()
        context.Response.Clear()
        context.Response.ContentType = "application/force-download"
        context.Response.AddHeader("content-disposition", "attachment; filename=name_you_file.txt")
        context.Response.BinaryWrite(bytesInStream)
        context.Response.End()
    End Sub
    ReadOnly Property IsReusable() As Boolean Implements IHttpHandler.IsReusable
        Get
            Return False
        End Get
    End Property
End Class
Then you use
Content-Type: application/force-download 
which means 
"I, the web server, am going to lie to you (the browser) about what
  this file is so that you will not treat it as a PDF/Word
  Document/MP3/whatever and prompt the user to save the mysterious file
  to disk instead". It is a dirty hack that breaks horribly when the
  client doesn't do "save to disk".
...
And finally, you don't encode the filename correctly, so if one uses non-ASCII characters for the filename, it will garble the filename, which is very funny if you happen to be Chinese or Russian and operate entirely outside the ASCII character set.
Original
Here a quick excerpt from one of my ajax handlers.
It's VB.NET, when converting, take care on the length -1 things.
Sub ProcessRequest(ByVal context As HttpContext) Implements IHttpHandler.ProcessRequest
    Dim strFileName As String = "Umzugsmitteilung.doc"
    Dim strUID As String = context.Request.QueryString("ump_uid")
    context.Response.Clear()
    'If String.IsNullOrEmpty(strUID) Or fileData Is Nothing Then
    '    context.Response.Write("<script type=""text/javascript"">alert('File does not exist !')</script>")
    '    context.Response.End()
    'End If
    context.Response.ClearContent()
    'context.Response.AddHeader("Content-Disposition", "attachment; filename=" + strFileName)
    context.Response.AddHeader("Content-Disposition", GetContentDisposition(strFileName))
    'context.Response.ContentType = "application/msword"
    context.Response.ContentType = "application/octet-stream"
    GetUmzugsMitteilung(strUID)
    context.Response.End()
End Sub ' ProcessRequest
Public Shared Sub SaveWordDocumentToOutputStream(strUID As String, doc As Aspose.Words.Document)
    Using ms As System.IO.MemoryStream = New System.IO.MemoryStream()
        CreateWordDocumentFromTemplate(strUID, doc, ms)
        ms.Position = 0
        Dim bytes As Byte() = New Byte(ms.Length - 1) {}
        ms.Read(bytes, 0, CInt(ms.Length))
        System.Web.HttpContext.Current.Response.OutputStream.Write(bytes, 0, ms.Length)
        ms.Close()
    End Using ' ms
End Sub ' SaveWordDocumentToOutputStream
    Public Shared Function StripInvalidPathChars(str As String) As String
        If str Is Nothing Then
            Return Nothing
        End If
        Dim strReturnValue As String = ""
        Dim strInvalidPathChars As New String(System.IO.Path.GetInvalidPathChars())
        Dim bIsValid As Boolean = True
        For Each cThisChar As Char In str
            bIsValid = True
            For Each cInvalid As Char In strInvalidPathChars
                If cThisChar = cInvalid Then
                    bIsValid = False
                    Exit For
                End If
            Next cInvalid
            If bIsValid Then
                strReturnValue += cThisChar
            End If
        Next cThisChar
        Return strReturnValue
    End Function ' StripInvalidPathChars
    Public Shared Function GetContentDisposition(ByVal strFileName As String) As String
        ' http://stackoverflow.com/questions/93551/how-to-encode-the-filename-parameter-of-content-disposition-header-in-http
        Dim contentDisposition As String
        strFileName = StripInvalidPathChars(strFileName)
        If System.Web.HttpContext.Current IsNot Nothing AndAlso System.Web.HttpContext.Current.Request.Browser IsNot Nothing Then
            If (System.Web.HttpContext.Current.Request.Browser.Browser = "IE" And (System.Web.HttpContext.Current.Request.Browser.Version = "7.0" Or System.Web.HttpContext.Current.Request.Browser.Version = "8.0")) Then
                contentDisposition = "attachment; filename=" + Uri.EscapeDataString(strFileName).Replace("'", Uri.HexEscape("'"c))
            ElseIf (System.Web.HttpContext.Current.Request.Browser.Browser = "Safari") Then
                contentDisposition = "attachment; filename=" + strFileName
            Else
                contentDisposition = "attachment; filename*=UTF-8''" + Uri.EscapeDataString(strFileName)
            End If
        Else
            contentDisposition = "attachment; filename*=UTF-8''" + Uri.EscapeDataString(strFileName)
        End If
        Return contentDisposition
    End Function ' GetContentDisposition