error message I am getting is: exceptions.TypeError: 'NoneType' object has no attribute 'getitem'. Here is my code:
# -*- coding: utf-8 -*-
import scrapy,MySQLdb,codecs
from scrapy import Request
class MyItems(scrapy.Item):
topLinks = scrapy.Field() #links at top of home page to grab artist page
artists = scrapy.Field() # list of all the artists
artists_urls = scrapy.Field() #url of the page for the artist + songs
song_name = scrapy.Field() #name of each song
song_duration = scrapy.Field() #duration of the song, duh
song_dl = scrapy.Field() #dl link for the proxy site for songs
rd = scrapy.Field()
class UpdaterSpider(scrapy.Spider):
name = "updater"
allowed_domains = [
'myfreemp3.cc', 'unref.eu', 'safeurl.eu'
]
start_urls = (
'http://www.example.com/',
)
def __init__(self, *a, **kw):
super(UpdaterSpider, self).__init__(*a, **kw)
self.item = MyItems()
self.db = MySQLdb.connect( #setup my SQL database info
"127.0.0.1", #host
"root", #user
"uconn3", #password
"Music" #database
)
self.cursor = self.db.cursor() #create a cursor to execute SQL statements
self.item = MyItems()
self.y = 0
def parse(self, response):
sql = "SELECT * FROM Songs"
self.cursor.execute(sql)
result = self.cursor.fetchone()
while result is not None:
yield Request(url=result[3], callback= lambda r: self.parse_dl(r, result[1], result[2]))
result = self.cursor.fetchone()
def parse_dl(self, response, songname, duration):
self.item['rd'] = response.xpath("//head/script/text()").extract()
for x in range(len(self.item['rd'])):
self.item['rd'][x] = self.item['rd'][x].split('"')[1]
sql = "INSERT INTO Songs (" \
"Artist, songName, Duration, URL, Genre) " \
"VALUES ('Placeholder', '%s', '%s', '%s', 'Placeholder')" % \
(songname.decode('utf-8'), duration.decode('utf-8'), self.item['rd'][x])
self.cursor.execute(sql)
self.db.commit()
The error is fixed when I remove the sql statement lines, but I cannot figure out why that is messing things up.
edit1: traceback
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/twisted/internet/base.py", line 1201, in mainLoop
self.runUntilCurrent()
File "/usr/lib/python2.7/dist-packages/twisted/internet/base.py", line 824, in runUntilCurrent
call.func(*call.args, **call.kw)
File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line 382, in callback
self._startRunCallbacks(result)
File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line 490, in _startRunCallbacks
self._runCallbacks()
--- <exception caught here> ---
File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line 577, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/user/PycharmProjects/untitled/mp3_scraper/mp3_scraper/spiders/updater.py", line 48, in <lambda>
yield Request(url=result[3], callback= lambda r: self.parse_dl(r, result[1], result[2]))
exceptions.TypeError: 'NoneType' object has no attribute '__getitem__'
