One possible solution is to override the mousePressEvent and mouseMoveEvent methods to apply the scroll, and correct using the axes ranges if necessary:
#include <QtWidgets>
#include <QtCharts>
#include <algorithm>
QT_CHARTS_USE_NAMESPACE
class ChartView: public QChartView{
public:
    using QChartView::QChartView;
    void setRange(qreal xmin, qreal xmax, qreal ymin, qreal ymax){
        if(!chart()) return;
        if(QValueAxis *xaxis = qobject_cast<QValueAxis *>(chart()->axes(Qt::Horizontal).first())){
            xaxis->setRange(xmin, xmax);
        }
        if(QValueAxis *yaxis = qobject_cast<QValueAxis *>(chart()->axes(Qt::Vertical).first())){
            yaxis->setRange(ymin, ymax);
        }
    }
    void setLimits(qreal min, qreal max, Qt::Orientation orientation){
        m_limit_min = min;
        m_limit_max = max;
        m_orientation = orientation;
    }
protected:
    void mousePressEvent(QMouseEvent *event)
    {
        if (event->button() == Qt::LeftButton && chart())
            m_lastMousePos = mapToScene(event->pos());
        QGraphicsView::mousePressEvent(event);
    }
    void mouseMoveEvent(QMouseEvent *event)
    {
        if(event->buttons() & Qt::LeftButton && chart()){
            QPointF newValue = mapToScene(event->pos());
            QPointF delta = newValue - m_lastMousePos;
            if(m_orientation == Qt::Horizontal)
                chart()->scroll(-delta.x(), 0);
            else
                chart()->scroll(0, -delta.y());
            if(QValueAxis * axis = qobject_cast<QValueAxis *>(chart()->axes(m_orientation).first()) ){
                qreal deltaX = axis->max() - axis->min();
                if(axis->min() < m_limit_min){
                    axis->setRange(m_limit_min, m_limit_min + deltaX);
                }
                else if(axis->max() > m_limit_max){
                    axis->setRange(m_limit_max - deltaX, m_limit_max);
                }
            }
            m_lastMousePos = newValue;
        }
        QGraphicsView::mouseMoveEvent(event);
    }
private:
    QPointF m_lastMousePos;
    qreal m_limit_min;
    qreal m_limit_max;
    Qt::Orientation m_orientation;
};
int main(int argc, char *argv[])
{
    QApplication a(argc, argv);
    ChartView chartView;
    chartView.setRenderHint(QPainter::Antialiasing);
    chartView.resize(640, 480);
    QLineSeries *series = new QLineSeries();
    series->append(0, 6);
    series->append(2, 4);
    series->append(3, 8);
    series->append(7, 4);
    series->append(10, 5);
    *series << QPointF(11, 1) << QPointF(13, 3) << QPointF(17, 6) << QPointF(18, 3) << QPointF(20, 2);
    QChart *chart = chartView.chart();
    chart->legend()->hide();
    chart->addSeries(series);
    chart->createDefaultAxes();
    chartView.show();
    chartView.setRange(5, 15, 0, 8);
    chartView.setLimits(0, 20, Qt::Horizontal);
    return a.exec();
}