I was able to achieve that by first downloading the videos from url and then playing it with custom player. Here is how i did in case if anyone else needed that:
1) Get all url's need to be played
2) Start downloading videos (in queue) from urls in local storage and keep a flag in preferences (that a video is already downloaded or not)
3) Assign urls to Adapter in which initialize object of video player controller that handles video playbacks
4) Set addOnScrollListener to check which position/video is currently visible and check if video is already downloaded or not if yes then play it.
Following is complete code:
MainActivity
public class MainActivity extends ActionBarActivity implements IVideoDownloadListener {
private static String TAG = "MainActivity";
private Context context;
private RecyclerView mRecyclerView;
private ProgressBar progressBar;
private VideosAdapter mAdapter;
private RecyclerView.LayoutManager mLayoutManager;
private ArrayList<Video> urls;
VideosDownloader videosDownloader;
@Override
protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(R.layout.activity_main);
    context = MainActivity.this;
    mRecyclerView = (RecyclerView) findViewById(R.id.my_recycler_view);
    progressBar = (ProgressBar) findViewById(R.id.progressBar);
    urls = new ArrayList<Video>();
    mRecyclerView.setHasFixedSize(true);
    mLayoutManager = new LinearLayoutManager(this);
    mRecyclerView.setLayoutManager(mLayoutManager);
    mAdapter = new VideosAdapter(MainActivity.this, urls);
    mRecyclerView.setAdapter(mAdapter);
    videosDownloader = new VideosDownloader(context);
    videosDownloader.setOnVideoDownloadListener(this);
    if(Utils.hasConnection(context))
    {
        getVideoUrls();
        mRecyclerView.addOnScrollListener(new RecyclerView.OnScrollListener() {
            @Override
            public void onScrolled(RecyclerView recyclerView, int dx, int dy) {
                super.onScrolled(recyclerView, dx, dy);
            }
            @Override
            public void onScrollStateChanged(RecyclerView recyclerView, int newState) {
                super.onScrollStateChanged(recyclerView, newState);
                if (newState == RecyclerView.SCROLL_STATE_IDLE) {
                    LinearLayoutManager layoutManager = ((LinearLayoutManager) recyclerView.getLayoutManager());
                    int firstVisiblePosition = layoutManager.findFirstVisibleItemPosition();
                    int findFirstCompletelyVisibleItemPosition = layoutManager.findFirstCompletelyVisibleItemPosition();
                    Video video;
                    if (urls != null && urls.size() > 0)
                    {
                        if (findFirstCompletelyVisibleItemPosition >= 0) {
                            video = urls.get(findFirstCompletelyVisibleItemPosition);
                            mAdapter.videoPlayerController.setcurrentPositionOfItemToPlay(findFirstCompletelyVisibleItemPosition);
                            mAdapter.videoPlayerController.handlePlayBack(video);
                        }
                        else
                        {
                            video = urls.get(firstVisiblePosition);
                            mAdapter.videoPlayerController.setcurrentPositionOfItemToPlay(firstVisiblePosition);
                            mAdapter.videoPlayerController.handlePlayBack(video);
                        }
                    }
                }
            }
        });
    }
    else
        Toast.makeText(context, "No internet available", Toast.LENGTH_LONG).show();
}
@Override
public void onVideoDownloaded(Video video) {
    mAdapter.videoPlayerController.handlePlayBack(video);
}
private void getVideoUrls()
{
    Video video1 = new Video("0", "1", "http://techslides.com/demos/sample-videos/small.mp4");
    urls.add(video1);
    Video video2 = new Video("1", "2", "http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4");
    urls.add(video2);
    Video video3 = new Video("2", "3", "http://sample-videos.com/video/mp4/720/big_buck_bunny_720p_1mb.mp4");
    urls.add(video3);
    Video video4 = new Video("3", "4", "http://dev.exiv2.org/attachments/341/video-2012-07-05-02-29-27.mp4");
    urls.add(video4);
    Video video5 = new Video("4", "5", "http://techslides.com/demos/sample-videos/small.mp4");
    urls.add(video5);
    Video video6 = new Video("5", "6", "http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4");
    urls.add(video6);
    Video video7 = new Video("6", "7", "http://sample-videos.com/video/mp4/720/big_buck_bunny_720p_1mb.mp4");
    urls.add(video7);
    mAdapter.notifyDataSetChanged();
    progressBar.setVisibility(View.GONE);
    videosDownloader.startVideosDownloading(urls);
 }
}
VideosAdapter
public class VideosAdapter extends RecyclerView.Adapter<VideosAdapter.ViewHolder> {
private static String TAG = "VideosAdapter";
Context context;
private ArrayList<Video> urls;
public VideoPlayerController videoPlayerController;
public static class ViewHolder extends RecyclerView.ViewHolder {
    public TextView textView;
    public ProgressBar progressBar;
    public RelativeLayout layout;
    public ViewHolder(View v) {
        super(v);
        layout = (RelativeLayout) v.findViewById(R.id.layout);
        textView = (TextView) v.findViewById(R.id.textView);
        progressBar = (ProgressBar) v.findViewById(R.id.progressBar);
    }
}
public VideosAdapter(Context context, final ArrayList<Video> urls) {
    this.context = context;
    this.urls = urls;
    videoPlayerController = new VideoPlayerController(context);
}
// Create new views (invoked by the layout manager)
@Override
public ViewHolder onCreateViewHolder(ViewGroup parent, int viewType) {
    // create a new view
    View v = LayoutInflater.from(parent.getContext()).inflate(R.layout.view_main, parent, false);
    Configuration configuration = context.getResources().getConfiguration();
    int screenWidthDp = configuration.screenWidthDp; //The current width of the available screen space, in dp units, corresponding to screen width resource qualifier.
    int smallestScreenWidthDp = configuration.smallestScreenWidthDp; //The smallest screen size an application will see in normal operation, corresponding to smallest screen width resource qualifier.
    ViewHolder viewHolder = new ViewHolder(v);
    int screenWidthPixels = Utils.convertDpToPixel(screenWidthDp, context);
    RelativeLayout.LayoutParams rel_btn = new RelativeLayout.LayoutParams(
            ViewGroup.LayoutParams.MATCH_PARENT, screenWidthPixels);
    viewHolder.layout.setLayoutParams(rel_btn);
    return viewHolder;
}
// Replace the contents of a view (invoked by the layout manager)
@Override
public void onBindViewHolder(final ViewHolder holder, int position) {
    Video video = urls.get(position);
    holder.textView.setText("Video " + video.getId());
    final VideoPlayer videoPlayer = new VideoPlayer(context);
    RelativeLayout.LayoutParams params = new RelativeLayout.LayoutParams(RelativeLayout.LayoutParams.MATCH_PARENT, RelativeLayout.LayoutParams.MATCH_PARENT);
    videoPlayer.setLayoutParams(params);
    holder.layout.addView(videoPlayer);
    videoPlayerController.loadVideo(video, videoPlayer, holder.progressBar);
    videoPlayer.setOnClickListener(new View.OnClickListener() {
        @Override
        public void onClick(View v) {
            videoPlayer.changePlayState();
        }
    });
}
@Override
public void onViewRecycled(ViewHolder holder) {
    super.onViewRecycled(holder);
    Log.d(TAG, "onViewRecycledCalled");
    holder.layout.removeAllViews();
}
@Override
public int getItemCount() {
    return urls.size();
}
}
VideosDownloader
public class VideosDownloader {
private static String TAG = "VideosDownloader";
Context context;
FileCache fileCache;
IVideoDownloadListener iVideoDownloadListener;
public VideosDownloader(Context context) {
    this.context = context;
    fileCache = new FileCache(context);
}
/////////////////////////////////////////////////////////////////
// Start downloading all videos from given urls
public void startVideosDownloading(final ArrayList<Video> videosList)
{
    Thread thread = new Thread(new Runnable() {
        @Override
        public void run()
        {
            for(int i=0; i<videosList.size(); i++)
            {
                final Video video = videosList.get(i);
                String id = video.getId();
                String url = video.getUrl();
                String isVideoDownloaded = Utils.readPreferences(context, video.getUrl(), "false");
                boolean isVideoAvailable = Boolean.valueOf(isVideoDownloaded);
                if(!isVideoAvailable)
                {
                    //Download video from url
                    String downloadedPath = downloadVideo(url);
                    //Log.i(TAG, "Vides downloaded at: " + downloadedPath);
                    Activity activity = (Activity) context;
                    activity.runOnUiThread(new Runnable() {
                        @Override
                        public void run() {
                            Utils.savePreferences(context, video.getUrl(), "true");
                             iVideoDownloadListener.onVideoDownloaded(video);
                        }
                    });
                }
            }
        }
    });
    thread.start();
}
/////////////////////////////////////////////////////////////////
private String downloadVideo(String urlStr)
{
    URL url = null;
    File file = null;
    try
    {
        file = fileCache.getFile(urlStr);
        url = new URL(urlStr);
        long startTime = System.currentTimeMillis();
        URLConnection ucon = null;
        ucon = url.openConnection();
        InputStream is = ucon.getInputStream();
        BufferedInputStream inStream = new BufferedInputStream(is, 1024 * 5);
        FileOutputStream outStream = new FileOutputStream(file);
        byte[] buff = new byte[5 * 1024];
        //Read bytes (and store them) until there is nothing more to read(-1)
        int len;
        while ((len = inStream.read(buff)) != -1) {
            outStream.write(buff, 0, len);
        }
        //clean up
        outStream.flush();
        outStream.close();
        inStream.close();
    }
    catch (MalformedURLException e) {
        e.printStackTrace();
    }
    catch (IOException e) {
        e.printStackTrace();
    }
    return file.getAbsolutePath();
}
public void setOnVideoDownloadListener(IVideoDownloadListener iVideoDownloadListener) {
    this.iVideoDownloadListener = iVideoDownloadListener;
}
}
VideoPlayerController
public class VideoPlayerController {
private static String TAG = "VideoPlayerController";
Context context;
FileCache fileCache;
int currentPositionOfItemToPlay = 0;
Video currentPlayingVideo;
private Map<String, VideoPlayer> videos = Collections.synchronizedMap(new WeakHashMap<String, VideoPlayer>());
private Map<String, ProgressBar> videosSpinner = Collections.synchronizedMap(new WeakHashMap<String, ProgressBar>());
public VideoPlayerController(Context context) {
    this.context = context;
    fileCache = new FileCache(context);
}
public void loadVideo(Video video, VideoPlayer videoPlayer, ProgressBar progressBar) {
    //Add video to map
    videos.put(video.getIndexPosition(), videoPlayer);
    videosSpinner.put(video.getIndexPosition(), progressBar);
    handlePlayBack(video);
}
//This method would check two things
//First if video is downloaded or its local path exist
//Second if the videoplayer of this video is currently showing in the list or visible
public void handlePlayBack(Video video)
{
    //Check if video is available
    if(isVideoDownloaded(video))
    {
        // then check if it is currently at a visible or playable position in the listview
        if(isVideoVisible(video))
        {
            //IF yes then playvideo
            playVideo(video);
        }
    }
}
private void playVideo(final Video video)
{
    //Before playing it check if this video is already playing
    if(currentPlayingVideo != video)
    {
        //Start playing new url
        if(videos.containsKey(video.getIndexPosition()))
        {
            final VideoPlayer videoPlayer2 = videos.get(video.getIndexPosition());
            String localPath = fileCache.getFile(video.getUrl()).getAbsolutePath();
            if(!videoPlayer2.isLoaded)
            {
                videoPlayer2.loadVideo(localPath, video);
                videoPlayer2.setOnVideoPreparedListener(new IVideoPreparedListener() {
                    @Override
                    public void onVideoPrepared(Video mVideo) {
                        //Pause current playing video if any
                        if(video.getIndexPosition() == mVideo.getIndexPosition())
                        {
                            if(currentPlayingVideo!=null)
                            {
                                VideoPlayer videoPlayer1 = videos.get(currentPlayingVideo.getIndexPosition());
                                videoPlayer1.pausePlay();
                            }
                            videoPlayer2.mp.start();
                            currentPlayingVideo = mVideo;
                        }
                    }
                });
            }
            else
            {
                //Pause current playing video if any
                if(currentPlayingVideo!=null)
                {
                    VideoPlayer videoPlayer1 = videos.get(currentPlayingVideo.getIndexPosition());
                    videoPlayer1.pausePlay();
                }
                boolean isStarted = videoPlayer2.startPlay();
                {
                    //Log.i(TAG, "Started playing Video Index: " + video.getIndexPosition());
                    //Log.i(TAG, "Started playing Video: " + video.getUrl());
                }
                currentPlayingVideo = video;
            }
        }
    }
    else
    {
        //Log.i(TAG, "Already playing Video: " + video.getUrl());
    }
}
private boolean isVideoVisible(Video video) {
    //To check if the video is visible in the listview or it is currently at a playable position
    //we need the position of this video in listview and current scroll position of the listview
    int positionOfVideo = Integer.valueOf(video.getIndexPosition());
    if(currentPositionOfItemToPlay == positionOfVideo)
        return true;
    return false;
}
private boolean isVideoDownloaded(Video video) {
    String isVideoDownloaded = Utils.readPreferences(context, video.getUrl(), "false");
    boolean isVideoAvailable = Boolean.valueOf(isVideoDownloaded);
    if(isVideoAvailable)
    {
        //If video is downloaded then hide its progress
        hideProgressSpinner(video);
        return true;
    }
    showProgressSpinner(video);
    return false;
}
private void showProgressSpinner(Video video) {
    ProgressBar progressBar = videosSpinner.get(video.getIndexPosition());
    if(progressBar!=null)
        progressBar.setVisibility(View.VISIBLE);
}
private void hideProgressSpinner(Video video) {
    ProgressBar progressBar = videosSpinner.get(video.getIndexPosition());
    if(progressBar!=null && progressBar.isShown())
    {
        progressBar.setVisibility(View.GONE);
        Log.i(TAG, "ProgressSpinner Hided Index: " + video.getIndexPosition());
    }
}
public void setcurrentPositionOfItemToPlay(int mCurrentPositionOfItemToPlay) {
    currentPositionOfItemToPlay = mCurrentPositionOfItemToPlay;
}
}
VideoPlayer
public class VideoPlayer extends TextureView implements TextureView.SurfaceTextureListener {
private static String TAG = "VideoPlayer";
/**This flag determines that if current VideoPlayer object is first item of the list if it is first item of list*/
boolean isFirstListItem;
boolean isLoaded;
boolean isMpPrepared;
IVideoPreparedListener iVideoPreparedListener;
Video video;
String url;
MediaPlayer mp;
Surface surface;
SurfaceTexture s;
public VideoPlayer(Context context) {
    super(context);
}
public VideoPlayer(Context context, AttributeSet attrs)
{
    super(context, attrs);
}
public void loadVideo(String localPath, Video video) {
    this.url = localPath;
    this.video = video;
    isLoaded = true;
    if (this.isAvailable()) {
        prepareVideo(getSurfaceTexture());
    }
    setSurfaceTextureListener(this);
}
@Override
public void onSurfaceTextureAvailable(final SurfaceTexture surface, int width, int height) {
    isMpPrepared = false;
    prepareVideo(surface);
}
@Override
public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) {
}
@Override
public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) {
    if(mp!=null)
    {
        mp.stop();
        mp.reset();
        mp.release();
        mp = null;
    }
    return false;
}
@Override
public void onSurfaceTextureUpdated(SurfaceTexture surface) {
}
public void prepareVideo(SurfaceTexture t)
{
    this.surface = new Surface(t);
    mp = new MediaPlayer();
    mp.setSurface(this.surface);
    try {
        mp.setDataSource(url);
        mp.prepareAsync();
        mp.setOnPreparedListener(new MediaPlayer.OnPreparedListener() {
            public void onPrepared(MediaPlayer mp) {
                isMpPrepared = true;
                mp.setLooping(true);
                iVideoPreparedListener.onVideoPrepared(video);
            }
        });
    } catch (IllegalArgumentException e1) {
        e1.printStackTrace();
    } catch (SecurityException e1) {
        e1.printStackTrace();
    } catch (IllegalStateException e1) {
        e1.printStackTrace();
    } catch (IOException e1) {
        e1.printStackTrace();
    }
    try {
    } catch (IllegalArgumentException e) {
        e.printStackTrace();
    } catch (SecurityException e) {
        e.printStackTrace();
    } catch (IllegalStateException e) {
        e.printStackTrace();
    }
    try {
    } catch (IllegalStateException e) {
        e.printStackTrace();
    }
}
@Override
protected void onAttachedToWindow() {
    super.onAttachedToWindow();
}
@Override
protected void onVisibilityChanged(View changedView, int visibility) {
    super.onVisibilityChanged(changedView, visibility);
}
public boolean startPlay()
{
    if(mp!=null)
        if(!mp.isPlaying())
        {
            mp.start();
            return true;
        }
    return false;
}
public void pausePlay()
{
    if(mp!=null)
        mp.pause();
}
public void stopPlay()
{
    if(mp!=null)
        mp.stop();
}
public void changePlayState()
{
    if(mp!=null)
    {
        if(mp.isPlaying())
            mp.pause();
        else
            mp.start();
    }
}
public void setOnVideoPreparedListener(IVideoPreparedListener iVideoPreparedListener) {
    this.iVideoPreparedListener = iVideoPreparedListener;
}
}
IVideoDownloadListener
public interface IVideoDownloadListener {
public void onVideoDownloaded(Video video);
}
IVideoPreparedListener
public interface IVideoPreparedListener {
public void onVideoPrepared(Video video);
}