I wasn't really satisfied with the solutions so I created one too.
You can play with it at https://bl.ocks.org/stephanbogner/06c3e0d3a1c8fcca61b5345e1af80798
The code is more complex, but it behaves like on iOS (tested on iPad and iPhone):
- You double-tap onto the image and it zooms into that exact location
 
- While pinching the zoom center is between your fingers and you can drag while pinching
 
<!DOCTYPE html>
<html> 
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>HammerJS Pinch Zoom and Drag</title>
    <style type="text/css">
    body{
        margin: 0;
        padding: 0;
    }
    #dragWrapper{
        margin: 40px;
        background: whitesmoke;
        width: calc(100vw - 80px);
        height: calc(100vh - 80px);
        position: relative;
    }
    #drag {
        touch-action: auto;
        height: 300px;
        width: 200px;
        position: absolute;
        left: 0;
        top: 0;
        background-image: url('https://upload.wikimedia.org/wikipedia/commons/thumb/4/4a/Bergelut_dengan_asap_nan_beracun.jpg/318px-Bergelut_dengan_asap_nan_beracun.jpg');
            background-size: cover;
    }
    </style>
</head>
<body>
    <div id="dragWrapper">
        <div id="drag"></div>
    </div>
    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/hammer.js/2.0.8/hammer.min.js"></script>
    <script type="text/javascript">
    var element = document.getElementById('drag')
    var hammertime = new Hammer(element, {});
    hammertime.get('pinch').set({ enable: true });
    hammertime.get('pan').set({ threshold: 0 });
    var fixHammerjsDeltaIssue = undefined;
    var pinchStart = { x: undefined, y: undefined }
    var lastEvent = undefined;
    var originalSize = {
        width: 200,
        height: 300
    }
    var current = {
        x: 0,
        y: 0,
        z: 1,
        zooming: false,
        width: originalSize.width * 1,
        height: originalSize.height * 1,
    }
    var last = {
        x: current.x,
        y: current.y,
        z: current.z
    }
    function getRelativePosition(element, point, originalSize, scale) {
        var domCoords = getCoords(element);
        var elementX = point.x - domCoords.x;
        var elementY = point.y - domCoords.y;
        var relativeX = elementX / (originalSize.width * scale / 2) - 1;
        var relativeY = elementY / (originalSize.height * scale / 2) - 1;
        return { x: relativeX, y: relativeY }
    }
    function getCoords(elem) { // crossbrowser version
        var box = elem.getBoundingClientRect();
        var body = document.body;
        var docEl = document.documentElement;
        var scrollTop = window.pageYOffset || docEl.scrollTop || body.scrollTop;
        var scrollLeft = window.pageXOffset || docEl.scrollLeft || body.scrollLeft;
        var clientTop = docEl.clientTop || body.clientTop || 0;
        var clientLeft = docEl.clientLeft || body.clientLeft || 0;
        var top  = box.top +  scrollTop - clientTop;
        var left = box.left + scrollLeft - clientLeft;
        return { x: Math.round(left), y: Math.round(top) };
    }
    function scaleFrom(zoomOrigin, currentScale, newScale) {
        var currentShift = getCoordinateShiftDueToScale(originalSize, currentScale);
        var newShift = getCoordinateShiftDueToScale(originalSize, newScale)
        var zoomDistance = newScale - currentScale
        var shift = {
            x: currentShift.x - newShift.x,
            y: currentShift.y - newShift.y,
        }
        var output = {
            x: zoomOrigin.x * shift.x,
            y: zoomOrigin.y * shift.y,
            z: zoomDistance
        }
        return output
    }
    function getCoordinateShiftDueToScale(size, scale){
        var newWidth = scale * size.width;
        var newHeight = scale * size.height;
        var dx = (newWidth - size.width) / 2
        var dy = (newHeight - size.height) / 2
        return {
            x: dx,
            y: dy
        }
    }
    hammertime.on('doubletap', function(e) {
        var scaleFactor = 1;
        if (current.zooming === false) {
            current.zooming = true;
        } else {
            current.zooming = false;
            scaleFactor = -scaleFactor;
        }
        element.style.transition = "0.3s";
        setTimeout(function() {
            element.style.transition = "none";
        }, 300)
        var zoomOrigin = getRelativePosition(element, { x: e.center.x, y: e.center.y }, originalSize, current.z);
        var d = scaleFrom(zoomOrigin, current.z, current.z + scaleFactor)
        current.x += d.x;
        current.y += d.y;
        current.z += d.z;
        last.x = current.x;
        last.y = current.y;
        last.z = current.z;
        update();
    })
    hammertime.on('pan', function(e) {
        if (lastEvent !== 'pan') {
            fixHammerjsDeltaIssue = {
                x: e.deltaX,
                y: e.deltaY
            }
        }
        current.x = last.x + e.deltaX - fixHammerjsDeltaIssue.x;
        current.y = last.y + e.deltaY - fixHammerjsDeltaIssue.y;
        lastEvent = 'pan';
        update();
    })    
    hammertime.on('pinch', function(e) {
        var d = scaleFrom(pinchZoomOrigin, last.z, last.z * e.scale)
        current.x = d.x + last.x + e.deltaX;
        current.y = d.y + last.y + e.deltaY;
        current.z = d.z + last.z;
        lastEvent = 'pinch';
        update();
    })
    var pinchZoomOrigin = undefined;
    hammertime.on('pinchstart', function(e) {
        pinchStart.x = e.center.x;
        pinchStart.y = e.center.y;
        pinchZoomOrigin = getRelativePosition(element, { x: pinchStart.x, y: pinchStart.y }, originalSize, current.z);
        lastEvent = 'pinchstart';
    })
    hammertime.on('panend', function(e) {
        last.x = current.x;
        last.y = current.y;
        lastEvent = 'panend';
    })
    hammertime.on('pinchend', function(e) {
        last.x = current.x;
        last.y = current.y;
        last.z = current.z;
        lastEvent = 'pinchend';
    })
    function update() {
        current.height = originalSize.height * current.z;
        current.width = originalSize.width * current.z;
        element.style.transform = "translate3d(" + current.x + "px, " + current.y + "px, 0) scale(" + current.z + ")";
    }
    </script>
</body>
</html>