I finally found a solution that satisfies all my requirements, and works in IE11, FF and Chrome (and degrades kind of OK in Safari...).
The idea is to create a Blob object containing the data from the response, then force the browser to open it as a file. It is slightly different for IE (proprietary API) and Chrome/FF (using a link element).
Here is the implementation, as a small Angular service:
myApp.factory('Download', [function() {
    return {
        openAsFile : function(response){
            // parse content type header
            var contentTypeStr = response.headers('Content-Type');
            var tokens = contentTypeStr.split('/');
            var subtype = tokens[1].split(';')[0];
            var contentType = {
                type : tokens[0],
                subtype : subtype
            };
            // parse content disposition header, attempt to get file name
            var contentDispStr = response.headers('Content-Disposition');
            var proposedFileName = contentDispStr ? contentDispStr.split('"')[1] : 'data.'+contentType.subtype;
            // build blob containing response data
            var blob = new Blob([response.data], {type : contentTypeStr});
            if (typeof window.navigator.msSaveBlob !== 'undefined'){
                // IE : use proprietary API
                window.navigator.msSaveBlob(blob, proposedFileName);
            }else{
                var downloadUrl = URL.createObjectURL(blob);
                // build and open link - use HTML5 a[download] attribute to specify filename
                var a = document.createElement("a");
                // safari doesn't support this yet
                if (typeof a.download === 'undefined') {
                    window.open(downloadUrl);
                }
                var link = document.createElement('a');
                link.href = downloadUrl;
                link.download = proposedFileName;
                document.body.appendChild(link);
                link.click();
                document.body.removeChild(link);
            }
        }
    }
}]);
The response argument expects a $http response object. Here is an example of use with a POST request:
$http.post(url, {property : 'value'},  {responseType: 'blob'}).then(function(response){
    Download.openAsFile(response);
});
Note the responseType parameter. Without this, my CSV data was being read as text and stored in memory as UTF-8 (or 16), and subsequently the file was saved in the same encoding, causing Excel to not recognize special characters such as éè etc. Since my CSVs are intended to be opened by Excel, the server encodes them Windows 1252, I wanted to keep them that way. Setting the responseType parameter to blob achieves this.
Disclaimer: It should work with any file type. But I tested it only with CSV files ! Binary files might behave somehow differently !