bookgenreprediction.py

# -*- coding: utf-8 -*-
"""BookGenreOneLastTime.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1DCA911kn_k-zsGK2dMeoa0wZgrBPFNKw

**Importing neccasry Packages**
"""

import pandas as pd
import numpy as np
import nltk
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split

"""**Uploading the dataset on the google colab**"""

from google.colab import files
uploaded = files.upload()

"""**Creating the books dataframe**"""

books=pd.read_csv('BooksDataSet.csv')

books=pd.DataFrame(books,columns=['book_id','book_name','genre','summary'])

"""**Snapshot of the dataframe**"""

books

"""**Grouping all the rows by genre**"""

books.groupby('genre').count()

"""**Preprocessing the data**

**1) Filtering out any character which is not an alphabet and then converting each character into lowercase from column 'summary'**
"""

books['summary']

# function for text cleaning 
def clean_text(text):
    # remove backslash-apostrophe 
    text = re.sub("\'", "", text) 
    # remove everything except alphabets 
    text = re.sub("[^a-zA-Z]"," ",text) 
    # remove whitespaces 
    text = ' '.join(text.split()) 
    # convert text to lowercase 
    text = text.lower() 
    
    return text

books.loc[:,'summary']=books.loc[:,'summary'].apply(lambda x: clean_text(x))

books['summary']

"""**2) Removing stopwords from the column summary.**"""

nltk.download('stopwords')

from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))

# function to remove stopwords
def remove_stopwords(text):
    no_stopword_text = [w for w in text.split() if not w in stop_words]
    return ' '.join(no_stopword_text)

books['summary'] = books['summary'].apply(lambda x: remove_stopwords(x))

books['summary']

"""**3) Performing lemmatisation on 'summary'**"""

nltk.download('wordnet')
from nltk.stem import WordNetLemmatizer

lemma=WordNetLemmatizer()

def lematizing(sentence):
    stemSentence = ""
    for word in sentence.split():
        stem = lemma.lemmatize(word)
        stemSentence += stem
        stemSentence += " "
    stemSentence = stemSentence.strip()
    return stemSentence


books['summary'] = books['summary'].apply(lambda x: lematizing(x))

books['summary']

"""**4) Performing stemming on 'summary'**"""

from nltk.stem import PorterStemmer
stemmer = PorterStemmer()
def stemming(sentence):
    stemSentence = ""
    for word in sentence.split():
        stem = stemmer.stem(word)
        stemSentence += stem
        stemSentence += " "
    stemSentence = stemSentence.strip()
    return stemSentence


books['summary'] = books['summary'].apply(lambda x: stemming(x))

books['summary']

"""**Labeling each 'genre' with an unique number**"""

from sklearn.preprocessing import LabelEncoder

LE = LabelEncoder()

y=LE.fit_transform(books['genre'])

LE.inverse_transform([0,1,2,3,4,5])

"""**At first we try with a 80-20% split on the dataset**"""

xtrain, xval, ytrain, yval = train_test_split(books['summary'], y, test_size=0.2, random_state=557)

"""**Performing tf-idf on 'summary' for both train(i.e xtrain) and test(i.e xval)**"""

tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=10000)

xtrain_tfidf = tfidf_vectorizer.fit_transform(xtrain.values.astype('U'))

xval_tfidf = tfidf_vectorizer.transform(xval.values.astype('U'))

"""**--Logistic Regression--**"""

from sklearn.linear_model import LogisticRegression

# Binary Relevance
from sklearn.multiclass import OneVsRestClassifier

# Performance metric
from sklearn.metrics import accuracy_score

lr = LogisticRegression()
clf = OneVsRestClassifier(lr)

# fit model on train data
clf.fit(xtrain_tfidf, ytrain)

# make predictions for validation set
y_pred_lr = clf.predict(xval_tfidf)


from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 
  


print( 'Accuracy Score :',accuracy_score(yval,y_pred_lr) )
print ('Report : ')
print(classification_report(yval,y_pred_lr))

"""**--KNN--**"""

from sklearn.neighbors import KNeighborsClassifier
knn=KNeighborsClassifier(n_neighbors=65) #2000

knn.fit(xtrain_tfidf,ytrain)

knnop=knn.predict(xval_tfidf)
# Performance metric

from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 
  


print( 'Accuracy Score :',accuracy_score(yval,knnop) )
print ('Report : ')
print(classification_report(yval,knnop))

"""**--SVM('linear')--**"""

from sklearn import svm

svc = svm.SVC(kernel='linear').fit(xtrain_tfidf,ytrain)

svpred=svc.predict(xval_tfidf)

from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 
  


print( 'Accuracy Score :',accuracy_score(yval,svpred) )
print ('Report : ')
print(classification_report(yval,svpred))

"""**--SVM(kernel=rbf)--**"""

from sklearn import svm

svc = svm.SVC(kernel='rbf',gamma=1).fit(xtrain_tfidf,ytrain)

svpred=svc.predict(xval_tfidf)

from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 
  


print( 'Accuracy Score :',accuracy_score(yval,svpred) )
print ('Report : ')
print(classification_report(yval,svpred))

"""**Now we try with a 85-15% split on the dataset**"""

xtrain, xval, ytrain, yval = train_test_split(books['summary'], y, test_size=0.15, random_state=246)

"""**Performing tf-idf on 'summary' for both train(i.e xtrain) and test(i.e xval)**"""

tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=10000)

xtrain_tfidf = tfidf_vectorizer.fit_transform(xtrain.values.astype('U'))

xval_tfidf = tfidf_vectorizer.transform(xval.values.astype('U'))

"""**--Logistic Regression--**"""

from sklearn.linear_model import LogisticRegression

# Binary Relevance
from sklearn.multiclass import OneVsRestClassifier

# Performance metric
from sklearn.metrics import accuracy_score

lr = LogisticRegression()
clf = OneVsRestClassifier(lr)

# fit model on train data
clf.fit(xtrain_tfidf, ytrain)

# make predictions for validation set
y_pred_lr = clf.predict(xval_tfidf)


from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 
  


print( 'Accuracy Score :',accuracy_score(yval,y_pred_lr) )
print ('Report : ')
print(classification_report(yval,y_pred_lr))

"""**--KNN--**"""

from sklearn.neighbors import KNeighborsClassifier
knn=KNeighborsClassifier(n_neighbors=65) #2000

knn.fit(xtrain_tfidf,ytrain)

knnop=knn.predict(xval_tfidf)
# Performance metric

from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 
  


print( 'Accuracy Score :',accuracy_score(yval,knnop) )
print ('Report : ')
print(classification_report(yval,knnop))

"""**--SVM('linear')--**"""

from sklearn import svm

svc = svm.SVC(kernel='linear').fit(xtrain_tfidf,ytrain)

svpred=svc.predict(xval_tfidf)

from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 
  


print( 'Accuracy Score :',accuracy_score(yval,svpred) )
print ('Report : ')
print(classification_report(yval,svpred))

"""**--SVM(kernel=rbf)--**"""

from sklearn import svm

svc = svm.SVC(kernel='rbf',gamma=1).fit(xtrain_tfidf,ytrain)

svpred=svc.predict(xval_tfidf)

from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 
  


print( 'Accuracy Score :',accuracy_score(yval,svpred) )
print ('Report : ')
print(classification_report(yval,svpred))

"""**--Executing the inference function on SVM(kernel=rbf) to predict future unknown genre--**"""

def infer_tags(q):
    q = clean_text(q)
    q = remove_stopwords(q)
    q = lematizing(q)
    q = stemming(q)
    q_vec = tfidf_vectorizer.transform([q])
    q_pred = svc.predict(q_vec)
    return LE.inverse_transform(q_pred)[0]
    #return q_pred[0]


for i in range(10): 
  k = xval.sample(1).index[0] 
  print("Book: ", books['book_name'][k], "\nPredicted genre: ", infer_tags(xval[k]),"\nActual genre: ",books['genre'][k], "\n")