The solution proposed by Lior Iluz with overriding onInterceptTouchEvent() has a serious issue.
If the content scrollable container is not fully scrolled-up, then it may be not possible to activate swipe-to-refresh in the same scroll-up gesture.
Indeed, when you start scrolling the inner container and move finger horizontally more then mTouchSlop unintentionally (which is 8dp by default),
the proposed CustomSwipeToRefresh declines this gesture. So a user has to try once more to start refreshing. This may look odd for the user.
I extracted the source code of the original SwipeRefreshLayout from the support library to my project and re-wrote the onInterceptTouchEvent(). The new class name is TouchSafeSwipeRefreshLayout
private boolean mPendingActionDown;
private float mInitialDownY;
private float mInitialDownX;
private boolean mGestureDeclined;
@Override
public boolean onInterceptTouchEvent(MotionEvent ev) {
    ensureTarget();
    final int action = ev.getActionMasked();
    int pointerIndex;
    if (mReturningToStart && action == MotionEvent.ACTION_DOWN) {
        mReturningToStart = false;
    }
    if (!isEnabled() || mReturningToStart || mRefreshing ) {
        // Fail fast if we're not in a state where a swipe is possible
        if (D) Log.e(LOG_TAG, "Fail because of not enabled OR refreshing OR returning to start. "+motionEventToShortText(ev));
        return false;
    }
    switch (action) {
        case MotionEvent.ACTION_DOWN:
            setTargetOffsetTopAndBottom(mOriginalOffsetTop - mCircleView.getTop());
            mActivePointerId = ev.getPointerId(0);
            if ((pointerIndex = ev.findPointerIndex(mActivePointerId)) >= 0) {
                if (mNestedScrollInProgress || canChildScrollUp()) {
                    if (D) Log.e(LOG_TAG, "Fail because of nested content is Scrolling. Set pending DOWN=true. "+motionEventToShortText(ev));
                    mPendingActionDown = true;
                } else {
                    mInitialDownX = ev.getX(pointerIndex);
                    mInitialDownY = ev.getY(pointerIndex);
                }
            }
            return false;
        case MotionEvent.ACTION_MOVE:
            if (mActivePointerId == INVALID_POINTER) {
                if (D) Log.e(LOG_TAG, "Got ACTION_MOVE event but don't have an active pointer id.");
                return false;
            } else if (mGestureDeclined) {
                if (D) Log.e(LOG_TAG, "Gesture was declined previously because of horizontal swipe");
                return false;
            } else if ((pointerIndex = ev.findPointerIndex(mActivePointerId)) < 0) {
                return false;
            } else if (mNestedScrollInProgress || canChildScrollUp()) {
                if (D) Log.e(LOG_TAG, "Fail because of nested content is Scrolling. "+motionEventToShortText(ev));
                return false;
            } else if (mPendingActionDown) {
                // This is the 1-st Move after content stops scrolling.
                // Consider this Move as Down (a start of new gesture)
                if (D) Log.e(LOG_TAG, "Consider this move as down - setup initial X/Y."+motionEventToShortText(ev));
                mPendingActionDown = false;
                mInitialDownX = ev.getX(pointerIndex);
                mInitialDownY = ev.getY(pointerIndex);
                return false;
            } else if (Math.abs(ev.getX(pointerIndex) - mInitialDownX) > mTouchSlop) {
                mGestureDeclined = true;
                if (D) Log.e(LOG_TAG, "Decline gesture because of horizontal swipe");
                return false;
            }
            final float y = ev.getY(pointerIndex);
            startDragging(y);
            if (!mIsBeingDragged) {
                if (D) Log.d(LOG_TAG, "Waiting for dY to start dragging. "+motionEventToShortText(ev));
            } else {
                if (D) Log.d(LOG_TAG, "Dragging started! "+motionEventToShortText(ev));
            }
            break;
        case MotionEvent.ACTION_POINTER_UP:
            onSecondaryPointerUp(ev);
            break;
        case MotionEvent.ACTION_UP:
        case MotionEvent.ACTION_CANCEL:
            mIsBeingDragged = false;
            mGestureDeclined = false;
            mPendingActionDown = false;
            mActivePointerId = INVALID_POINTER;
            break;
    }
    return mIsBeingDragged;
}
See my example project on Github.