You are applying the function to the whole heading's innerHTML, which includes the anchor's href attribute. This should work for your fiddle example:
$('h1 a, h2 a, h3 a, h4 a').ligature( //...
However, it will only work on links inside the headings, and I'm not sure that's what you're looking for. If you want something that works for any contents inside a certain element (with any level of tag nesting), then you'll need a recursive approach. Here is an idea, which is basically plain JavaScript since jQuery does not provide a way to target DOM text nodes:
$.fn.ligature = function(str, lig) {
    return this.each(function() {
        recursiveLigatures(this, lig);
    });
    function recursiveLigatures(el, lig) {
        if(el.childNodes.length) {
            for(var i=0, len=el.childNodes.length; i<len; i++) {
                if(el.childNodes[i].childNodes.length > 0) {
                    recursiveLigatures(el.childNodes[i], lig);
                } else {
                    el.childNodes[i].nodeValue = htmlDecode(el.childNodes[i].nodeValue.replace(new RegExp(str, 'g'), lig));
                }
            }
        } else {
            el.nodeValue = htmlDecode(el.nodeValue.replace(new RegExp(str, 'g'), lig));
        }
    }
    // http://stackoverflow.com/a/1912522/825789
    function htmlDecode(input){
      var e = document.createElement('div');
      e.innerHTML = input;
      return e.childNodes.length === 0 ? "" : e.childNodes[0].nodeValue;
    }
};
// call this from the document.ready handler
$(function(){
    $('h3').ligature('ffi', 'ffi')
           .ligature('ffl', 'ffl')
           .ligature('ff', 'ff')
           .ligature('fi', 'fi')
           .ligature('fl', 'fl');
});
That should work on contents like this:
<h3>
    mixed ffi content 
    <span>this is another tag ffi <span>(and this is nested ffi</span></span>
    <a href="/news/here-is-a-url-with-ffi-ligature">Here is a ffi ligature</a>
</h3>
http://jsfiddle.net/JjLZR/