3
import requests
from bs4 import BeautifulSoup
import pandas as pd
import matplotlib.pyplot as plt

plt.style.use('ggplot')


url = "https://www.google.com/finance/historical?cid=207437&startdate=Jan%201%2C%201971&enddate=Jul%201%2C%202017&start={0}&num=30"
how_many_pages=3
start=0

for i in range(how_many_pages):
    new_url = url.format(start)
    page = requests.get(new_url)
    soup = BeautifulSoup(page.content, "lxml")
    table = soup.find_all('table', class_='gf-table historical_price')[0]

    columns_header = [th.getText() for th in table.findAll('tr')[0].findAll('th')]
    data_rows=table.findAll('tr')[1:]
    data=[[td.getText() for td in data_rows[i].findAll(['td'])] for i in range(len(data_rows))]

    if start == 0:
        final_df = pd.DataFrame(data, columns=columns_header)
    else:
        df = pd.DataFrame(data, columns=columns_header)
        final_df = pd.concat([final_df, df],axis=0)
    start += 30
    final_df.to_csv('nse_data.csv', sep='\t', encoding='utf-8')


final_df.columns = ['Date']
final_df['Date'] = pd.to_datetime(df['Date'], format='%Y-%m-%d', utc=True)


df.plot(x='Date', y='Close')


plt.savefig('foo.png')

The data downloaded is in the following format

    "Date
"   "Open
"   "High
"   "Low
"   "Close
"   "Volume
"
0   "Jun 30, 2017
"   "9,478.50
"   "9,535.80
"   "9,448.75
"   "9,520.90
"   "-
"
1   "Jun 29, 2017
"   "9,522.95
"   "9,575.80
"   "9,493.80
"   "9,504.10
"   "-

For the time being I only want to plot Date (on X-axis) against Close (on Y-axis)

However I am getting the error

ValueError: Length mismatch: Expected axis has 6 elements, new values have 1 elements

liv2hak
  • 14,472
  • 53
  • 157
  • 270

1 Answers1

2

  • Your headers and data contain newline characters. print(final_df.columns) returns:

    Index(['Date\n', 'Open\n', 'High\n', 'Low\n', 'Close\n', 'Volume\n'], dtype='object')

    Use rstrip to get rid of them:

    columns_header = [th.getText().rstrip() for th in table.findAll('tr')[0].findAll('th')]

    and

    data = [[td.getText().rstrip() for td in data_rows[i].findAll(['td'])] for i in range(len(data_rows))]

  • final_df.columns = ['Date'] produces your error. A dataframe requires as many headers as its number of columns. Therefore, in your case a list of 6 elements is expected. I'm not sure what you want to do here, I think you can simply remove this line.

  • The format you specify for date parsing does not match your data ['Apr 4, 2017', 'Apr 5, 2017', 'Apr 6, 2017',...]. Documentation on format codes here. Use instead:

    final_df['Date'] = pd.to_datetime(df['Date'], format='%b %d, %Y')

  • Convert your data to numeric values so you can plot them:

    final_df['Close'] = [float(val.replace(',', '')) for val in final_df['Close']]

  • Finally you can call:

    final_df.plot(x='Date', y='Close')
sim0ne
  • 292
  • 1
  • 7