.Rhistory

library(dplyr)
library(readr)
dataebay <- read_csv("eBayiPadTrain.csv")
temp <- read_csv("eBayiPadTest.csv")
rm(temp)
str(dataebay)
summary(dataebay)
fix(dataebay)
table(dataebay$biddable)
table(dataebay$condition)
dataebay$condition <- factor(dataebay$condition)
table(dataebay$condition)
str(dataebay)
levels(dataebay$condition)
dataebay$carrier <- factor(dataebay$carrier)
table(dataebay$carrier)
table(dataebay$biddable)
dataebay$biddable <- factor(dataebay$biddable)
table(dataebay$cellular, dataebay$carrier)
dataebay$cellular <- factor(dataebay$cellular)
dataebay$carrier <- factor(dataebay$carrier)
str(dataebay)
table(dataebay$color)
dataebay$color <- factor(dataebay$color)
table(dataebay$storage)
dataebay$storage <- factor(dataebay$storage)
table(dataebay$productline)
dataebay$productline <- factor(dataebay$productline)
str(dataebay)
library(tm)
summary(dataebay)
eBayTrain = read.csv("eBayiPadTrain.csv", stringsAsFactors=FALSE)
eBayTest = read.csv("eBayiPadTest.csv", stringsAsFactors=FALSE)
CorpusDescription = Corpus(VectorSource(c(eBayTrain$description, eBayTest$description)))
CorpusDescription = tm_map(CorpusDescription, content_transformer(tolower), lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, PlainTextDocument, lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, removePunctuation, lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, removeWords, stopwords("english"), lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, stemDocument, lazy=TRUE)
dtm = DocumentTermMatrix(CorpusDescription)
sparse = removeSparseTerms(dtm, 0.995)
DescriptionWords = as.data.frame(as.matrix(sparse))
2659-1861
colnames(DescriptionWords) = make.names(colnames(DescriptionWords))
DescriptionWordsTrain = head(DescriptionWords, nrow(eBayTrain))
DescriptionWordsTest = tail(DescriptionWords, nrow(eBayTest))
str(eBayTest)
dataebay <- select(dataebay, -description, -UniqueID)
str(dataebay)
library(randomForest)
dataebay <- cbind(dataebay, DescriptionWordsTrain)
set.seed(123)
ebayRF <- randomForest(sold ~ ., data = dataebay)
names(dataebay)
colnames(dataebay) = make.names(colnames(dataebay))
names(dataebay)
colnames(dataebay[,33])
names(dataebay[,33])
names(dataebay)
dataebay[,33]
names(dataebay)[33]
names(dataebay)[33] <- "condition1"
ebayRF <- randomForest(sold ~ ., data = dataebay)
eBayTest$condition <- factor(eBayTest$condition)
eBayTest$carrier <- factor(eBayTest$carrier)
eBayTest$biddable <- factor(eBayTest$biddable)
eBayTest$cellular <- factor(eBayTest$cellular)
eBayTest$carrier <- factor(eBayTest$carrier)
eBayTest$color <- factor(eBayTest$color)
eBayTest$storage <- factor(eBayTest$storage)
eBayTest$productline <- factor(eBayTest$productline)
str(eBayTest)
eBayTest <- select(eBayTest, -description, -UniqueID)
str(eBayTest)
eBayTest <- cbind(eBayTest, DescriptionWordsTest)
PredTest = predict(ebayRF, newdata=eBayTest, type="response")
names(eBayTest)
names(eBayTest)[32] <- "condition1"
PredTest = predict(ebayRF, newdata=eBayTest, type="response")
str(eBayTest)
str(dataebay)
PredTest = predict(ebayRF, newdata=eBayTest, type="prob")
dataebay$sold <- factor(dataebay$sold)
eBayTest$sold <- factor(eBayTest$sold)
ebayRF <- randomForest(sold ~ ., data = dataebay)
PredTest = predict(ebayRF, newdata=eBayTest, type="prob")
levels(dataebay$productline)
levels(eBayTest$productline)
levels(eBayTest$productline) <- c(levels(eBayTest$productline), "iPad 5", "iPad mini Retina")
PredTest = predict(ebayRF, newdata=eBayTest, type="prob")
eBayTest2 = read.csv("eBayiPadTest.csv", stringsAsFactors=FALSE)
subm = read.csv("eSampleSubmission.csv", stringsAsFactors=FALSE)
subm = read.csv("SampleSubmission.csv", stringsAsFactors=FALSE)
str(subm)
head(PredTest)
MySubmission = data.frame(UniqueID = eBayTest2$UniqueID, Probability1 = PredTest[,2])
write.csv(MySubmission, "SubmissionDescriptionLog.csv", row.names=FALSE)
library(dplyr)
library(readr)
dataebay <- read_csv("eBayiPadTrain.csv")
library(tm)
str(dataebay)
table(dataebay$description == "")
table(dataebay$storage)
table(dataebay$biddable)
table(dataebay$biddable, dataebay$description == "")
library(ggplot2)
plotmatrix(dataebay[, 2:10])
install.packages("GGally")
library(GGally)
ggpairs(data = dataebay)
ggpairs(data = dataebay, columns = 2:ncol(dataebay))
str(dataebay)
ggpairs(data = dataebay, columns = c(2,3,5,8,10))
ggpairs(data = dataebay, columns = c(2,3,8))
table(dataebay$biddable)
str(dataebay)
table(dataebay$storage)
summary(dataebay)
table(dataebay$startprice)
str(dataebay)
dataebay$is_descpr <- as.factor(dataebay$description == "")
table(dataebay$is_descpr)
table(dataebay$is_descpr, dataebay$description == "")
table(dataebay$startprice, dataebay$biddable == 1)
str(dataebay)
ggplot(data = dataebay, aes(startprice)) + geom_histogram()
ggplot(data = dataebay, aes(startprice)) + geom_histogram() + facet_grid(~factor(biddable))
ggplot(data = dataebay, aes(startprice)) + geom_histogram() + facet_grid(.~factor(biddable))
ggplot(data = dataebay, aes(startprice)) + geom_histogram() + facet_grid(.~ biddable)
table(dataebay$biddable, dataebay$sold)
summary(dataebay)
ggplot(dataebay, aes(x = startprice, y = sold)) + geom_point()
ggplot(dataebay, aes(x = startprice, y = sold)) + geom_point() + facet_grid(biddable ~ .)
ggplot(data = dataebay, aes(startprice)) + geom_histogram() + facet_grid(.~ biddable)
ggplot(dataebay, aes(x = startprice, y = sold)) + geom_point() + facet_grid(biddable ~ .) + geom_smooth(method = lm)
str(dataebay)
dataebay1 <- select(dataebay, factor(condition))
dataebay1 <- select(dataebay, as,factor(condition))
dataebay1 <- select(dataebay, as.factor(condition))
dataebay1 <- mutate(dataebay, as.factor(condition))
dataebay1
str(dataebay1)
table(dataebay$condition)
dataebay1 <- mutate(dataebay, condition = as.factor(condition))
str(dataebay1)
table(dataebay1$condition)
dataebay %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular), carrier = as.factor(carrier), color = as.factor(color), storage = as.factor(storage), productline = as.factor(productline))
library(magrittr)
dataebay %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular), carrier = as.factor(carrier), color = as.factor(color), storage = as.factor(storage), productline = as.factor(productline))
str(dataebay)
dataebay %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular), carrier = as.factor(carrier), color = as.factor(color), storage = as.factor(storage), productline = as.factor(productline), isd = (description == ""))
str(dataebay)
dataebay %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular), carrier = as.factor(carrier), color = as.factor(color), storage = as.factor(storage), productline = as.factor(productline), isd = as.factor(description == ""))
str(dataebay)
dataebay %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular), carrier = as.factor(carrier), color = as.factor(color), storage = as.factor(storage), productline = as.factor(productline), isd = as.factor(description == "")) %>% select(-isd, -UniqueID)
str(dataebay)
table(dataebay$sold)
library(caTools)
set.seed(1000)
split = sample.split(dataebay$sold, SplitRatio = 0.7)
train  <- filter(dataebay, split == T)
test <- filter(dataebay, split == F)
model_glm1 <- glm(sold ~ .-description, data = train, family = binomial)
summary(model_glm1)
predict_test_glm <- predict(model_glm1, newdata = test, type = "response")
library(tm)
rm(list = ls())
dataeBayTrain = read_csv("eBayiPadTrain.csv")
str(dataeBayTrain)
rm(list = ls())
eBayTrain = read_csv("eBayiPadTrain.csv")
eBayTest = read_csv("eBayiPadTest.csv")
CorpusDescription = Corpus(VectorSource(c(eBayTrain$description, eBayTest$description)))
CorpusDescription = tm_map(CorpusDescription, content_transformer(tolower), lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, PlainTextDocument, lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, removePunctuation, lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, removeWords, stopwords("english"), lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, stemDocument, lazy=TRUE)
dtm = DocumentTermMatrix(CorpusDescription)
dtm
inspect(dtm)
removeSparseTerms(dtm, 0.995)
dtm
removeSparseTerms(dtm, 0.999)
removeSparseTerms(dtm, 0.1)
removeSparseTerms(dtm, 0.5)
removeSparseTerms(dtm, 0.9)
removeSparseTerms(dtm, 0.95)
inspect(removeSparseTerms(dtm, 0.95))
head(inspect(removeSparseTerms(dtm, 0.95)))
head(inspect(removeSparseTerms(dtm, 0.955)))
removeSparseTerms(dtm, 0.97)
head(inspect(removeSparseTerms(dtm, 0.97)))
head(inspect(removeSparseTerms(dtm, 0.95)))
inspect(removeSparseTerms(dtm, 0.95))
removeSparseTerms(dtm, 0.97)
removeSparseTerms(dtm, 0.95)
removeSparseTerms(dtm, 0.99)
findFreqTerms(dtm, lowfreq = 1000)
findFreqTerms(dtm, lowfreq = 100)
removeSparseTerms(dtm, 0.99)
removeSparseTerms(dtm, 0.98)
removeSparseTerms(dtm, 0.97)
inspect(removeSparseTerms(dtm, 0.97))
head(inspect(removeSparseTerms(dtm, 0.97)))
sparse = removeSparseTerms(dtm, 0.97)
DescriptionWords = as.data.frame(as.matrix(sparse))
colnames(DescriptionWords) = make.names(colnames(DescriptionWords))
DescriptionWordsTrain = head(DescriptionWords, nrow(eBayTrain))
DescriptionWordsTest = tail(DescriptionWords, nrow(eBayTest))
dataebay <- select(dataebay, -description, -UniqueID)
eBayTrain %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular),
carrier = as.factor(carrier), color = as.factor(color),
storage = as.factor(storage), productline = as.factor(productline),
isd = as.factor(description == ""))
eBayTrain = read_csv("eBayiPadTrain.csv")
eBayTrain %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular),
carrier = as.factor(carrier), color = as.factor(color),
storage = as.factor(storage), productline = as.factor(productline),
is_descr = as.factor(description == ""))
eBayTrain %<>%select(-description, -UniqueID)
eBayTrain <- cbind(eBayTrain, DescriptionWordsTrain)
str(eBayTrain)
set.seed(1000)
split <- sample.split(dataebay$sold, SplitRatio = 0.7)
split <- sample.split(eBayTrain$sold, SplitRatio = 0.7)
train  <- filter(eBayTrain, split == T)
names(eBayTrain)
names(eBayTrain)[3] <- "condition_is"
names(eBayTrain)
train  <- filter(eBayTrain, split == T)
test <- filter(eBayTrain, split == F)
model_glm1 <- glm(sold ~ ., data = train, family = binomial)
summary(model_glm1)
library(ROCR)
predict_glm <- predict(model_glm1, newdata = test, type = response )
predict_glm <- predict(model_glm1, newdata = test, type = "response" )
ROCRpred = prediction(predict_glm, test$sold)
as.numeric(performance(ROCRpred, "auc")@y.values)
library(rpart)
install.packages("rpart")
library(rpart)
library(rpart.plot)
model_cart1 <- rpart(sold ~ ., data = train)
plot(model_cart1)
prp(model_cart1)
model_cart1 <- rpart(sold ~ ., data = train, method = "class")
prp(model_cart1)
predict_cart <- predict(model_cart1, newdata = test, type = "class")
ROCRpred = prediction(predict_cart, test$sold)
predict_cart <- predict(model_cart1, newdata = test, type = "response")
predict_cart <- predict(model_cart1, newdata = test, type = "prob")
ROCRpred = prediction(predict_cart, test$sold)
predict_cart <- predict(model_cart1, newdata = test, type = "prob")[,2]
ROCRpred = prediction(predict_cart, test$sold)
as.numeric(performance(ROCRpred, "auc")@y.values)
summary(model_cart1)
library(caret)
library(caret)
library(e1071)
tr.control = trainControl(method = "cv", number = 10)
(0:10)*0.001
(0:10)*0.001
seq(0.01,0.5,0.01)
cpGrid = expand.grid( .cp = seq(0.01,0.5,0.01))
train(sold ~ ., data = Train, method = "rpart", trControl = tr.control, tuneGrid = cpGrid )
train(sold ~ ., data = train, method = "rpart", trControl = tr.control, tuneGrid = cpGrid )
model_cart2 <- rpart(sold ~ ., data = train, method = "class", cp = 0.06)
prp(model_cart2)
predict_cart <- predict(model_cart2, newdata = test, type = "prob")[,2]
ROCRpred = prediction(predict_cart, test$sold)
as.numeric(performance(ROCRpred, "auc")@y.values)
cpGrid = expand.grid( .cp = seq(0.001,0.1,0.01))
train(sold ~ ., data = train, method = "rpart", trControl = tr.control, tuneGrid = cpGrid )
cpGrid = expand.grid( .cp = seq(0.001,0.1,0.001))
train(sold ~ ., data = train, method = "rpart", trControl = tr.control, tuneGrid = cpGrid )
model_cart2 <- rpart(sold ~ ., data = train, method = "class", cp = 0.064)
predict_cart <- predict(model_cart2, newdata = test, type = "prob")[,2]
ROCRpred = prediction(predict_cart, test$sold)
as.numeric(performance(ROCRpred, "auc")@y.values)
library(randomForest)
set.seed(1000)
model_rf <- randomForest(sold ~ ., data = train)
train$sold <- as.factor(train$sold)
model_rf <- randomForest(sold ~ ., data = train)
predict_rf  <- predict(model_rf, newdata = test, type = "response")
ROCRpred = prediction(predict_rf, test$sold)
test$sold <- as.factor(test$sold)
predict_rf  <- predict(model_rf, newdata = test, type = "response")
ROCRpred = prediction(predict_rf, test$sold)
predict_rf  <- predict(model_rf, newdata = test, type = "prob")[,2]
ROCRpred = prediction(predict_rf, test$sold)
as.numeric(performance(ROCRpred, "auc")@y.values)
PredTest = predict(model_rf, newdata=eBayTest, type="response")
eBayTest$is_descr  <- as.factor(eBayTest$description == "")
PredTest = predict(model_rf, newdata=eBayTest, type="response")
names(eBayTest)
eBayTest <- cbind(eBayTest, DescriptionWordsTest)
names(eBayTest)
eBayTest = read_csv("eBayiPadTest.csv")
eBayTest %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular),
carrier = as.factor(carrier), color = as.factor(color),
storage = as.factor(storage), productline = as.factor(productline),
is_descr = as.factor(description == ""))
names(eBayTest)
eBayTest %<>%select(-description)
eBayTest <- cbind(eBayTest, DescriptionWordsTest)
names(eBayTest)
names(eBayTest)[3] <- "condition_is"
eBayTrain$sold <- as.factor(eBayTrain$sold)
model_rf <- randomForest(sold ~ ., data = eBayTrain)
PredTest = predict(model_rf, newdata=eBayTest, type="response")
str(eBayTest)
str(eBayTrain)
a <- c(1,2,3)
b <- c(3,4,5)
rbind(a,b)
eBayTrain = read_csv("eBayiPadTrain.csv")
eBayTest = read_csv("eBayiPadTest.csv")
ebay <- rbind(eBayTrain, eBayTest)
eBayTrain %<>% select(-sold)
ebay <- rbind(eBayTrain, eBayTest)
eBayTest %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular),
carrier = as.factor(carrier), color = as.factor(color),
storage = as.factor(storage), productline = as.factor(productline),
is_descr = as.factor(description == ""))
eBay %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular),
carrier = as.factor(carrier), color = as.factor(color),
storage = as.factor(storage), productline = as.factor(productline),
is_descr = as.factor(description == ""))
ebay %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular),
carrier = as.factor(carrier), color = as.factor(color),
storage = as.factor(storage), productline = as.factor(productline),
is_descr = as.factor(description == ""))
eBayTest %<>% slice(1862:nrow(ebay))
eBayTest  <-  slice(ebay, 1862:nrow(ebay))
eBayTest %<>%select(-description)
eBayTest <- cbind(eBayTest, DescriptionWordsTest)
names(eBayTest)
names(eBayTest)[3] <- "condition_is"
eBayTrain = read_csv("eBayiPadTrain.csv")
eBayTrain %<>% mutate(condition = as.factor(condition), cellular = as.factor(cellular),
carrier = as.factor(carrier), color = as.factor(color),
storage = as.factor(storage), productline = as.factor(productline),
is_descr = as.factor(description == ""))
eBayTrain %<>%select(-description, -UniqueID)
eBayTrain <- cbind(eBayTrain, DescriptionWordsTrain)
eBayTrain$sold <- as.factor(eBayTrain$sold)
model_rf <- randomForest(sold ~ ., data = eBayTrain)
names(eBayTrain)[3] <- "condition_is"
model_rf <- randomForest(sold ~ ., data = eBayTrain)
PredTest = predict(model_rf, newdata=eBayTest, type="response")
PredTest = predict(model_rf, newdata=eBayTest, type="prob")
names(eBayTest)
MySubmission = data.frame(UniqueID = eBayTest$UniqueID, Probability1 = PredTest[,2])
write.csv(MySubmission, "SubmissionRF2.csv", row.names=FALSE)
train  <- filter(eBayTrain, split == T)
test <- filter(eBayTrain, split == F)
model_rf <- randomForest(sold ~ ., data = train, importance = T)
importance(model_rf)
varImpPlot(model_rf)
set.seed(1000)
model_rf <- randomForest(sold ~ ., data = train)
test$sold <- as.factor(test$sold)
predict_rf  <- predict(model_rf, newdata = test, type = "prob")[,2]
ROCRpred = prediction(predict_rf, test$sold)
as.numeric(performance(ROCRpred, "auc")@y.values)
set.seed(1000)
model_rf <- randomForest(sold ~ ., data = train, importance = T)
varImpPlot(model_rf)
predict_rf  <- predict(model_rf, newdata = test, type = "prob")[,2]
ROCRpred = prediction(predict_rf, test$sold)
as.numeric(performance(ROCRpred, "auc")@y.values)
set.seed(1000)
importance(model_rf)
MDSplot(model_rf)
plot(model_rf)
set.seed(1000)
varImpPlot(model_rf)
set.seed(1000)
model_rf <- randomForest(sold ~ .-veri - new, data = train, importance = T)
predict_rf  <- predict(model_rf, newdata = test, type = "prob")[,2]
ROCRpred = prediction(predict_rf, test$sold)
as.numeric(performance(ROCRpred, "auc")@y.values)
model_rf <- randomForest(sold ~ ..-veri - new, data = eBayTrain, importance = T)
model_rf <- randomForest(sold ~ .-veri - new, data = eBayTrain, importance = T)
varImpPlot(model_rf)
PredTest = predict(model_rf, newdata=eBayTest, type="prob")
MySubmission = data.frame(UniqueID = eBayTest$UniqueID, Probability1 = PredTest[,2])
write.csv(MySubmission, "SubmissionRF3.csv", row.names=FALSE)
eBayTrain = read_csv("eBayiPadTrain.csv")
eBayTest = read_csv("eBayiPadTest.csv")
library(tm)
CorpusDescription = Corpus(VectorSource(c(eBayTrain$description, eBayTest$description)))
CorpusDescription = tm_map(CorpusDescription, content_transformer(tolower), lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, PlainTextDocument, lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, removePunctuation, lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, removeWords, stopwords("english"), lazy=TRUE)
CorpusDescription = tm_map(CorpusDescription, stemDocument, lazy=TRUE)
dtm = DocumentTermMatrix(CorpusDescription)
findFreqTerms(dtm, lowfreq = 100)
sparse = removeSparseTerms(dtm, 0.97)
DescriptionWords = as.data.frame(as.matrix(sparse))
rm(list = ls())
eBayTrain = read_csv("eBayiPadTrain.csv")
eBayTest = read_csv("eBayiPadTest.csv")
library(tm)
CorpusDescription = Corpus(VectorSource(c(eBayTrain$description, eBayTest$description)))
?Sys.timezone
Sys.setenv(TZ='GMT')
library(tm) ## Загружаем библиотеку
## Создаём корпус из текста, необходимый для работы библиотеки
CorpusDescription <-  Corpus(VectorSource(c(eBayTrain$description, eBayTest$description)))
## Приводим текст к строчным буквам
CorpusDescription <-  tm_map(CorpusDescription, content_transformer(tolower))
CorpusDescription <-  tm_map(CorpusDescription, PlainTextDocument, lazy=TRUE)
## Удаляем знаки препинания
CorpusDescription <-  tm_map(CorpusDescription, removePunctuation, lazy=TRUE)
## Удаляем так называемые стоп-слова, т.е. слова, не несущие смысловой нагрузки
CorpusDescription <-  tm_map(CorpusDescription, removeWords, stopwords("english"), lazy=TRUE)
## Производим стемминг, т.е. приводим слова к смысловым основам
CorpusDescription <-  tm_map(CorpusDescription, stemDocument, lazy=TRUE)
## Создаём частотную матрицу
dtm <-  DocumentTermMatrix(CorpusDescription)
## Удаляем редкочастотные слова
sparse <-  removeSparseTerms(dtm, 0.97)