I am trying to scrape the first 200 rows from a webpage. I am starting with just being able to print out the scraped-out data before loading it into a data frame. Still, my code keeps generating errors or sometimes returns an empty list, and also when the code in the container variable is split into two with its attribute name, only the first 77 rows are scraped.
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
import time
url = 'https://coinmarketcap.com/all/views/all/'
path= xxxxxxxxx
service=Service(executable_path=path)
driver=webdriver.Chrome(service=service)
driver.get(url)
time.sleep(60)
containers = driver.find_elements(by='xpath',value='//tr[@class="cmc-table-row" and (@style="display: table-row" or @style="display: table-row;")]')
Ranks = []
Names = []
Images = []
Symbols = []
MarketCaps = []
Prices = []
CirculatingSupplys = []
Volume24hs = []
Hr_percnt_1s = []
Hr_percnt_24s = []
day_percent_7s = []
for i in containers:
Rank = i.find_element(by='xpath',value='./td/div').text
Name = i.find_element(by='xpath',value='./td/div/a[2]').text
Symbol = i.find_element(by='xpath',value='./td[3]/div').text
MarketCap = i.find_element(by='xpath',value='./td/p/span[2]').text
Price = i.find_element(by='xpath',value='./td[5]/div/a/span').text
CirculatingSupply = i.find_element(by='xpath',value='./td[6]/div').text
Volume24h = i.find_element(by='xpath',value='./td[7]/a').text
Hr_percnt_1 = i.find_element(by='xpath',value='./td[8]/div').text
Hr_percnt_24 = i.find_element(by='xpath',value='./td[9]/div').text
day_percent_7 = i.find_element(by='xpath',value='./td[10]/div').text
Ranks.append(Rank)
Names.append(Name)
Symbols.append(Symbol)
MarketCaps.append(MarketCap)
Prices.append(Price)
CirculatingSupplys.append(CirculatingSupply)
Volume24hs.append(Volume24h)
Hr_percnt_1s.append(Hr_percnt_1)
Hr_percnt_24s.append(Hr_percnt_24)
day_percent_7s.append(day_percent_7)
print(Ranks)
print(Names)
print(Symbols)
print(MarketCaps)
print(Prices)
print(CirculatingSupplys)
print(Volume24hs)
print(Hr_percnt_1s)
print(Hr_percnt_24s)
print(day_percent_7s)
driver.quit()
