Element not interactable (clickable) while scraping a website

Question

Can anyone please tell me what's wrong with the code?

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import csv
import requests
from bs4 import BeautifulSoup

rows, cols = (1000,11)
rows = [[0 for i in range(cols)] for j in range(rows)]
driver_path = 'C:/Users/adith/Downloads/chromedriver_win32/chromedriver.exe'
brave_path = 'C:/Program Files/BraveSoftware/Brave-Browser/Application/brave.exe'
option = webdriver.ChromeOptions()
option.binary_location = brave_path
browser = webdriver.Chrome(executable_path=driver_path, options=option)

browser.get('https://www.dell.com/community/Laptops/ct-p/Laptops')

cookies = {
    'lithiumLogin:vjauj58549': '~2TtiW3OEsenvCn5Ir~fBcCal7YbmhAmxNWLe4LgaSRCss_g69Gqm2CAs-fDA_FtccFLDK3AoWuzXHz72fb'
}


def load():
    count = 0
    while True:
        page = requests.get(browser.current_url)
        soup = BeautifulSoup(page.content)
        button = [z.text for z in soup.find_all(class_='lia-link-navigation')]
        print(button[-1])
        if(button[-1]=='Load more'):
            count = count + 1
            browser.find_element(By.XPATH, '//*[@id="btn-load-more"]').click()
            if(count>12):
                break
        else:
            break
            
try:
    load()
except:
    load()

It is able to click the load button 3 times but then I get an error as shown below.

Load more
Load more
Load more
Load more
Load more
Load more
---------------------------------------------------------------------------
ElementClickInterceptedException          Traceback (most recent call last)
<ipython-input-6-ef6cae973b0d> in <module>
     18 try:
---> 19     load()
     20 except:

<ipython-input-6-ef6cae973b0d> in load()
     11             count = count + 1
---> 12             browser.find_element(By.XPATH, '//*[@id="btn-load-more"]').click()
     13             if(count>15):

~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in click(self)
     87         """Clicks the element."""
---> 88         self._execute(Command.CLICK_ELEMENT)
     89 

~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in _execute(self, command, params)
    395         params['id'] = self._id
--> 396         return self._parent.execute(command, params)
    397 

~\anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
    434         if response:
--> 435             self.error_handler.check_response(response)
    436             response['value'] = self._unwrap_value(

~\anaconda3\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
    246             raise exception_class(message, screen, stacktrace, alert_text)  # type: ignore[call-arg]  # mypy is not smart enough here
--> 247         raise exception_class(message, screen, stacktrace)
    248 

ElementClickInterceptedException: Message: element click intercepted: Element is not clickable at point (470, 1629)
  (Session info: chrome=103.0.5060.114)

Can someone please explain what's wrong with the code? Also, can anyone help me out to eliminate the use of BeautifulSoup library completely to get the required solution?

There is no BeautifulSoup in your code. Tell us what are ytou trying to achieve after you load all the data in your page. — Barry the Platipus, Jul 21 '22 at 10:38

score 1 · Answer 1 · answered Jul 21 '22 at 13:11

As you are able to click the load button 3 times, that implies the locator strategy

(By.XPATH, "//*[@id='btn-load-more']")

is perfecto.

Solution

Ideally to click on the clickable element you need to induce WebDriverWait for the element_to_be_clickable() and you can use the following solution:

WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[@id='btn-load-more']"))).click()

Note: You have to add the following imports :

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

Initially, I got the same error when I ran the code but running the same code again gave me no errors at all. Weird but working perfectly now. Thanks a lot!! — Adithya Jere, Jul 23 '22 at 07:45

score 0 · Answer 2 · answered Jul 21 '22 at 10:35

Try scrolling to the end of the page (where the button is located) before clicking on it:

def load():
    count = 0
    while True:
        page = requests.get(browser.current_url)
        soup = BeautifulSoup(page.content)
        button = [z.text for z in soup.find_all(class_='lia-link-navigation')]
        print(button[-1])
        if(button[-1]=='Load more'):
            count = count + 1
            browser.execute_script( "window.scrollTo(0,document.body.scrollHeight);")
            time.sleep(1)
            browser.find_element(By.XPATH, '//*[@id="btn-load-more"]').click()
            if(count>12):
                break
        else:
            break

score 0 · Accepted Answer · answered Jul 21 '22 at 10:57

You're using requests and bs4 when you already have the browser via selenium, you should just use selenium without requests to do the same job:

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import csv

import requests
from bs4 import BeautifulSoup

rows, cols = (1000,11)
rows = [[0 for i in range(cols)] for j in range(rows)]


driver_path = 'C:/Users/adith/Downloads/chromedriver_win32/chromedriver.exe'
brave_path = 'C:/Program Files/BraveSoftware/Brave-Browser/Application/brave.exe'
option = webdriver.ChromeOptions()
option.binary_location = brave_path
browser = webdriver.Chrome(executable_path=driver_path, options=option)

browser.get('https://www.dell.com/community/Laptops/ct-p/Laptops')

def load():
    count = 0
    while True:
        button = [z.text for z in driver.find_elements(By.CLASS_NAME, 'lia-link-navigation')]
        print(button[-1])

        if(button[-1]=='Load more'):
            count = count + 1
            browser.execute_script( "window.scrollTo(0,document.body.scrollHeight);")
            browser.find_element(By.XPATH, '//*[@id="btn-load-more"]').click()
            if(count>12):
                break
        else:
            break
            
try:
    load()
except:
    load()

Element not interactable (clickable) while scraping a website

3 Answers3

Solution