How to find the product name and price using the python scraper?

Question

I am writing a selenium based webscraper in python and it keeps throwing:

no such element: unable to locate element

Even though I can see the element in the selenium browser that is launched. here is the link it keeps failing on: https://www.neimanmarcus.com/p/givenchy-g-chain-ring-prod250190244?childItemId=NMY5X1R_&navpath=cat000000_cat4870731_cat50910737_cat2650734&page=0&position=11

Here is my code for the driver:

def getDriver():
    try:
        options = webdriver.ChromeOptions()
        options.add_argument("start-maximized")
        options.add_experimental_option("excludeSwitches", ["enable-automation"])
        options.add_experimental_option('useAutomationExtension', False)
        options.add_experimental_option('excludeSwitches', ['enable-logging'])
        driver = webdriver.Chrome(ChromeDriverManager().install(), chrome_options=options)
        return driver
    except Exception:
        traceback.print_exc()
        print(Exception)

Here is the scraper:

def getProduct(domain, url):
    # find the yaml file for the domain
    driver = getDriver()

    noYaml = False
    try:
        markupPath = 'markups/%s.yml' % domain
    except Exception as error:
        noYaml = True
    if(noYaml == False):
        with open(markupPath, 'r') as file:
            yamlElements = yaml.safe_load(file)

        titleXpath = '//span[contains(concat(" ",normalize-space(@class)," ")," Titlestyles__ProductName-fRyAwr ")]'
        priceXpath = '//span[contains(concat(" ",normalize-space(@class)," ")," Pricingstyles__RetailPrice-eYMMwV ")]'
        print("XXXXXXPATHHHHHHHHHHHHHS: ",titleXpath, priceXpath)
        driver.get(url)
        driver.implicitly_wait(10)
        
        try:
            # WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,titleXpath)))
            
            title = driver.find_element(By.XPATH, titleXpath)
        except Exception as error:
            print("ERROR: ", error)
            title = None

        try: 
            # WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,priceXpath)))
            price = driver.find_element_by_xpath(priceXpath)
            # price = driver.find_element(By.XPATH,priceXpath)
            print(price)
        except Exception as error:
            print("ERRROR: ", error)
            price = None
        driver.execute_script("window.stop();")
        data={}
        if(title != None):
            print(title.get_attribute('innerHTML'))
            data['title'] = title.get_attribute('innerHTML')

        if(price != None):
            print(price.get_attribute('innerHTML'))
            data['price'] = price.get_attribute('innerHTML')

        if 'price' not in data or data['price'] == "" or data['price'] == None:
            driver.quit()
            return False
            getProduct(domain, url)
        driver.quit()
        return data```

I have been stuck here for a while and can't figure out why it is throwing an error 
  Message: no such element: Unable to locate element: {"method":"xpath","selector":"//span[contains(concat(" ",normalize-space(@class)," ")," Pricingstyles__RetailPrice-eYMMwV ")]"}

undetected Selenium · Answer 1 · 2022-07-25T09:42:23.997

The second part of the xpaths i.e. fRyAwr, eYMMwV are dynamically generated and is bound to change sooner/later. They may change next time you access the application afresh or even while next application startup. So can't be used in locators.

Solution

To print the product name and price instead of presence_of_element_located() you need to induce WebDriverWait for the visibility_of_element_located() and you can use either of the following locator strategies:

Using CSS_SELECTOR:

driver.execute("get", {'url': 'https://www.neimanmarcus.com/en-in/p/givenchy-g-chain-ring-prod250190244?childItemId=NMY5X1R_&navpath=cat000000_cat4870731_cat50910737_cat2650734&page=0&position=11'})
# Close Now shipping to India popup
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a#closeButton"))).click()
# Close cookie consent
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.close_button_panel>i.close_icon"))).click()
print(WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "span[data-test='pdp-title']"))).text)
print(WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div[data-test='pdp-pricing']>span"))).text)

Using XPATH:

driver.execute("get", {'url': 'https://www.neimanmarcus.com/en-in/p/givenchy-g-chain-ring-prod250190244?childItemId=NMY5X1R_&navpath=cat000000_cat4870731_cat50910737_cat2650734&page=0&position=11'})
# Close Now shipping to India popup
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[@id='closeButton']"))).click()
# Close cookie consent
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[@class='close_button_panel']/i[@class='close_icon']"))).click()
print(WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//span[@data-test='pdp-title']"))).text)
print(WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//div[@data-test='pdp-pricing']/span"))).text)

Console output:
```
G-Chain Ring
INR 27573
```

Note : You have to add the following imports :

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

you were correct, those selectors are better. Thank you. – Tadewos Bellete Jul 25 '22 at 14:28 — Tadewos Bellete, Jul 25 '22 at 14:28
Glad to be able to help you. – undetected Selenium Jul 25 '22 at 19:30 — undetected Selenium, Jul 25 '22 at 19:30

score 0 · Answer 2 · answered Jul 25 '22 at 04:00

0

It seems like your xpaths are missing the last bit of the string. You may want to consider simplifying them as well by doing this:

titleXpath = '//span[@class="Titlestyles__ProductName-fRyAwr elVgyx"]'
priceXpath = '//span[@class="Pricingstyles__RetailPrice-eYMMwV jjJaCp"]'

And also, if you are still recieving the error you may want to look into adding a user agent to your webdriver options. Hope this helps

answered Jul 25 '22 at 04:00

taylon

27
3

those xpaths and user agent both not working. The data appars in my selectlib extension with the xpaths I use. I don't know why i doesn't work here. – Tadewos Bellete Jul 25 '22 at 04:53

score 0 · Answer 3 · answered Jul 25 '22 at 04:31

0

You are having trouble debugging your xpaths, and I am too--they are hard to read. These CSS Selectors work for me:

CssSelectorTitle = 'span.Titlestyles__ProductName-fRyAwr'
CssSelectorPrice = 'span.Pricingstyles__RetailPrice-eZcFGu'

answered Jul 25 '22 at 04:31

C. Peck

3,641
3
19
36

when using selectlib extension for chrome those xpaths work. It only fails in the code. I fixed my wrong quotes in my original posts for the xpaths. – Tadewos Bellete Jul 25 '22 at 04:38
how can these xpath's work for selectlib and fail on selenium. I see the chrome window open and the elements are present – Tadewos Bellete Jul 25 '22 at 04:39
This is aa CSS selector, not xpath, may I assume you're trying that? – C. Peck Jul 25 '22 at 05:02
(Your xpath doesn't work for me from devtools) – C. Peck Jul 25 '22 at 05:04
css selectors are giving me the same error – Tadewos Bellete Jul 25 '22 at 05:14

score 0 · Accepted Answer · answered Jul 25 '22 at 07:55

It appears that website will automatically redirect visitor according to the country determined from visitor's IP, and it will also change the classes for item name and price accordingly. Here is a more robust solution, which will account for such changes, and will also wait for the element to load in page:

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


chrome_options = Options()
chrome_options.add_argument("--no-sandbox")

webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
browser = webdriver.Chrome(service=webdriver_service, options=chrome_options)

url='https://www.neimanmarcus.com/en-jp/p/givenchy-g-chain-ring-prod250190244?childItemId=NMY5X1R_&navpath=cat000000_cat4870731_cat50910737_cat2650734&page=0&position=11'

browser.get(url) 

try:
    WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.ID, "closeButton"))).click()
except Exception as e:
    print('no pop-up, moving on')

title = WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.XPATH, "//span[@data-test='pdp-title']")))
price = WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[@data-test='pdp-pricing']")))
print(title.text.strip(), price.text.strip())

Result:

G-Chain Ring JPY 46709

How to find the product name and price using the python scraper?

4 Answers4

Solution