Use a variable to track the status of the scrolling, and a timer to determine if the scrolling has been stopped for N seconds.
var scroll_active = false;
var scroll_timer = new Date();
Start your scroll timer:
check_scroll_time();
Then, on scroll, re-set the time-stamp in the scroll_timer variable:
$(window).scroll(function(){
    scroll_timer = new Date();
    //Your existing code goes here; No-scroll events are handled in check_scroll_time()
    var scroll = jQuery(window).scrollTop();
    if(scroll > position) { // scrolling down
       // do something when scrolling down
    } else if(scroll < position) { // scrolling up
       // do something when scrolling up
    }
});
Your scroll-monitoring function just needs to subtract the scroll_timer time from the current time and determine how long since the last scroll event:
function check_scroll_time(){
  now = new Date();
  if ((now.getTime() - scroll_timer.getTime())/1000 > 2){
    //No scrolling - do something
  }else{
    //Scrolling is active - do nothing
  }
  setTimeout(function(){ check_scroll_time() },300); //<==== call ad-infinitum
}
Example:
var scroll_active = false;
var scroll_timer = new Date();
check_scroll_time();
$(window).scroll(function(){
  scroll_timer = new Date();
});
function check_scroll_time(){
  now = new Date();
  if ((now.getTime() - scroll_timer.getTime())/1000 > 2){
    $('#msg').html('Paused');
  }else{
    $('#msg').html('Scrolling');
  }
  setTimeout(function(){ check_scroll_time() },300);
}
body{height:700vh;}
#blurb{position:fixed;top:50px;width:100vw;text-align:center;}
#msg{position:fixed;top:0;right:0;text-align:center;padding:5px;background:wheat;}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script>
<div id="msg"></div>
<div id="blurb">Scroll away - watching for a 2-sec pause</div>
 
 
References:
Javascript Date Reference