Modelling data using dataframes

Question

I am trying to train a dataset to predict whether an inputted text is from a science fiction novel or not. I am relatively new to python, so I don't know exactly what I am doing wrong.

Code:

#class17.py
"""
Created on Fri Nov 17 14:07:36 2017

@author: twaters

Read three science fiction novels
Predict a sentence or paragraph
see whether sentence/phrase/book is from a science fiction novel or not
"""

import nltk
import pandas as pd
import csv
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression

from sklearn import model_selection
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from nltk.corpus import stopwords

#nltk.download()


irobot = "C:/Users/twaters/Desktop/Assignments/SQL/Python/DA Project/irobot.txt"
enders_game = "C:/Users/twaters/Desktop/Assignments/SQL/Python/DA Project/endersgame.txt"
space_odyssey ="C:/Users/twaters/Desktop/Assignments/SQL/Python/DA Project/spaceodyssey.txt"
to_kill_a_mockingbird = "C:/Users/twaters/Desktop/Assignments/SQL/Python/DA Project/tokillamockingbird.txt"

sr = set(stopwords.words('english'))
freq = {}

def main():
    #read_novels()
    model_novels()


def read_novel(b, is_scifi):

    read_file = open(b)

    text = read_file.read()
    words = text.split()
    clean_tokens = words[:]
    filtered_list = []

    for word in clean_tokens:
        word = word.lower()
        if word not in sr:
            filtered_list.append(word)

    freq = nltk.FreqDist(clean_tokens)
    #print(filtered_list)
    for word in clean_tokens:
       count = freq.get(word,0)
       freq[word] = count + 1



    frequency_list = freq.keys()

    with open('C:/Users/twaters/Desktop/Assignments/SQL/Python/DA Project/novels_data.txt', 'w', encoding='utf-8') as csvfile:
        fieldnames = ['word','frequency','is_scifi']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames, lineterminator = '\n')
        writer.writeheader()

        for words in frequency_list:
            writer.writerow({'word': words,'frequency': freq[words],'is_scifi':is_scifi})

    print("List compiled.")

def read_novels(): 

    read_novel(enders_game, 0)
    read_novel(space_odyssey, 0)
    read_novel(irobot, 0)
    read_novel(to_kill_a_mockingbird, 1)

def model_novels():

    df = pd.read_csv('C:/Users/twaters/Desktop/Assignments/SQL/Python/DA Project/novels_data.txt', 'rb', delimiter='\t', encoding='utf-8')
    print(df)

    #for index in range(2, df.shape[0], 100):
    df_subset = df.loc[1:]
    #print(df_subset)
    X = df_subset.loc[:, 'frequency':'is_scifi']
    Y = df_subset.loc[:, 'frequency':'is_scifi']
    testing_size = 0.2
    seed = 7
    X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size=testing_size, random_state=seed)

    selectedModel = LogisticRegression()
    selectedModel.fit(X_train, Y_train)  
    predictions = selectedModel.predict(X_validation)

#%%
#print("Accuracy Score:\n", accuracy_score(Y_validation, predictions))
#print("Confusion Matrix:\n",confusion_matrix(predictions, Y_validation))
#print("Class report:\n", classification_report(Y_validation, predictions))
#df_test = pd.read_csv('C:/Users/twaters/Desktop/Assignments/SQL/Python/DA Project/novels_data.txt', delimiter='\t')
#predictions_test = selectedModel.predict(df_test)
#test_frame = pd.DataFrame(predictions_test)
#test_frame.to_csv('C:/Users/twaters/Desktop/Assignments/SQL/Python/DA Project/novels_data_result.txt', sep='\t')

Error: Traceback (most recent call last):

File "", line 1, in main()

File "C:/Users/user/Desktop/Assignments/SQL/Python/DA Project/class17.py", line 36, in main model_novels()

File "C:/Users/user/Desktop/Assignments/SQL/Python/DA Project/class17.py", line 95, in model_novels selectedModel.fit(X_train, Y_train)

File "D:\Program Files (x86)\Anaconda\lib\site-packages\sklearn\linear_model\logistic.py", line 1216, in fit order="C")

File "D:\Program Files (x86)\Anaconda\lib\site-packages\sklearn\utils\validation.py", line 573, in check_X_y ensure_min_features, warn_on_dtype, estimator)

File "D:\Program Files (x86)\Anaconda\lib\site-packages\sklearn\utils\validation.py", line 453, in check_array _assert_all_finite(array)

File "D:\Program Files (x86)\Anaconda\lib\site-packages\sklearn\utils\validation.py", line 44, in _assert_all_finite " or a value too large for %r." % X.dtype)

ValueError: Input contains NaN, infinity or a value too large for dtype('float64').

If you need access to the files I am reading from, I can link them.

Thank you for your help!

Based on `Input contains NaN, infinity or a value too large for dtype('float64')`, I'd start by printing the contents of `X_train` and `Y_train` and checking for NaN. Maybe `df_subset` contains some NaN rows that make it through `train_test_split`. The fix *may* be to call `df_subset.dropna(inplace=True)`. — Peter Leimbigler, Dec 06 '17 at 00:41
Thanks, running df_subset.dropna(inplace=True) fixed my issue. Turns out there were 2 records with NaN data. — Trevor Waters, Dec 06 '17 at 20:18

score 0 · Answer 1 · answered Dec 06 '17 at 01:03

Here are the points in the stacktrace which you should pay attention to:

File "C:/Users/user/Desktop/Assignments/SQL/Python/DA Project/class17.py", line 95, in model_novels selectedModel.fit(X_train, Y_train)

File "D:\Program Files (x86)\Anaconda\lib\site-packages\sklearn\utils\validation.py", line 44, in _assert_all_finite " or a value too large for %r." % X.dtype)

That tells us there is a problem with formatting X so the logistic regression will accept it.

You should check X_train and X to see if they contain errant values.

This answer will give you some pointers on how to do that.

Python pandas: check if any value is NaN in DataFrame

Modelling data using dataframes

1 Answers1