AdTracking_Fraud_Detection/modelTesting.R at master · WPI-DS502-FinalProject/AdTracking_Fraud_Detection · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
library(leaps)
library(MASS)
library(class)

#Constants
NUM_FILES = 5    #Number of files to use
MODEL_TOTAL = 5  #Number of models per branch
BRANCH_TOTAL = 7 #Number of branches

#Results Table: Initialize
final_results_table=data.frame(branch=numeric(), model=character(), predictors=character(), accuracy=numeric(), TP=numeric(), FN=numeric(), FP=numeric(), TN=numeric(), stringsAsFactors=FALSE)

#Balance Table: Initialize
final_balance_table=data.frame(file=numeric(), branch=numeric(), human=numeric(), bot=numeric(), total=numeric(), humanPer=numeric(), botPer=numeric(), totalPer=numeric(), stringsAsFactors=FALSE)

extractFeature <- function(origData){
  origData$attributed_time <- NULL
  origData$hour <- as.numeric(format(as.POSIXct(origData$click_time) ,format = "%H"))
  origData$ip_app <- (origData$ip + 1) * origData$app
  origData$channel_app <- (origData$channel + 1) * origData$app
  origData$channel_ip <- (origData$channel + 1) * origData$ip
  origData$channel_ip_app <- (origData$channel_ip + 1) * origData$app
  origData$click_time <- NULL
  return (origData)
}

#Load Data
origData=read.csv(file="./data/t3p20/ones.csv")
names(origData) = c('ip', 'app', 'device', 'os', 'channel','click_time', 'attributed_time', 'is_attributed')
for(file_num in c(0:NUM_FILES)){
  newData=read.csv(file=paste("./data/t3p20/zeros_", file_num,".csv", sep=""))
  names(newData) = c('ip', 'app', 'device', 'os', 'channel','click_time', 'attributed_time', 'is_attributed')
  origData=rbind(origData,newData)
}
origData <- extractFeature(origData)

for(branch_num in c(1:BRANCH_TOTAL)){
  #Create Branches:
  branch_data = switch(
    branch_num,
    origData[((origData$app< 18.5) & (origData$channel< 114.5) & (origData$channel< 112)),],
    origData[((origData$app< 18.5) & (origData$channel< 114.5) & (origData$channel>=112)),],
    origData[((origData$app< 18.5) & (origData$channel>=114.5)),],
    origData[((origData$app>=18.5) & (origData$app< 19.5)),],
    origData[((origData$app>=18.5) & (origData$app>=19.5) & (origData$app< 28.5)),],
    origData[((origData$app>=18.5) & (origData$app>=19.5) & (origData$app>=28.5) & (origData$channel< 345)),],
    origData[((origData$app>=18.5) & (origData$app>=19.5) & (origData$app>=28.5) & (origData$channel>=345)),])

  #Blance data
  human=nrow(branch_data[branch_data$is_attributed==1,])
  bot=nrow(branch_data[branch_data$is_attributed==0,])
  total=human+bot
  humanPer=human/total
  botPer=bot/total

  final_balance_table=rbind(final_balance_table, data.frame(file=file_num, branch=branch_num, human=human, bot=bot, total=total, humanPer=humanPer, botPer=botPer, totalPer=0))

  #Logistic Regression:
  if(model_per_branch_table[branch_num,]$model=="Logistic Regression"){
    #Logistic Regression: Assign Predictor
    pred=pred_per_model_table[(pred_per_model_table$model=="Logistic Regression")&(pred_per_model_table$file==BEST_FILE)&(pred_per_model_table$branch==branch_num),]$predictors

    msg=sprintf("Testing -> Branch: %d -> Predictor: %s -> Model: %s",branch_num, pred, "Logistic Regression")
    message(msg)

    #Logistic Regression: Test Model
    branch_probs=predict(testing_models[[branch_num]], newdata=branch_data, type="response")
    branch_pred =rep(0, length(branch_probs))#Error
    branch_pred[branch_probs > 0.5] = 1
    branch_mean=mean(branch_pred == branch_data$is_attributed)
    branch_cMatrix=confusionMatrix(branch_pred, branch_data$is_attributed)

    #Logistic Regression: Save Result
    final_results_table=rbind(final_results_table, data.frame(branch=branch_num, model="Logistic Regression", predictors=pred, accuracy=branch_mean, TP=branch_cMatrix$table[1], FN=branch_cMatrix$table[2], FP=branch_cMatrix$table[3], TN=branch_cMatrix$table[4], accuracy_cMatrix=as.numeric(branch_cMatrix$overall["Accuracy"])))

  }else if(model_per_branch_table[branch_num,]$model=="LDA"){
    #LDA:
    #LDA: Assign Predictor
    pred=pred_per_model_table[(pred_per_model_table$model=="LDA")&(pred_per_model_table$file==BEST_FILE)&(pred_per_model_table$branch==branch_num),]$predictors

    msg=sprintf("Testing -> Branch: %d -> Predictor: %s -> Model: %s",branch_num, pred, "LDA")
    message(msg)

    #LDA: Test Model
    branch_probs=predict(testing_models[[branch_num]], newdata=branch_data)
    branch_pred =rep(0, nrow(branch_probs$posterior))#Error
    branch_pred[branch_probs$posterior[,2] > 0.5] = 1
    branch_mean=mean(branch_pred == branch_data$is_attributed)
    branch_cMatrix=confusionMatrix(branch_pred, branch_data$is_attributed)

    #LDA: Save Result
    final_results_table=rbind(final_results_table, data.frame(branch=branch_num, model="LDA",predictors=pred, accuracy=branch_mean, TP=branch_cMatrix$table[1], FN=branch_cMatrix$table[2], FP=branch_cMatrix$table[3], TN=branch_cMatrix$table[4], accuracy_cMatrix=as.numeric(branch_cMatrix$overall["Accuracy"])))

  }else if(model_per_branch_table[branch_num,]$model=="QDA"){
    #QDA:
    #QDA: Assign Predictor
    pred=pred_per_model_table[(pred_per_model_table$model=="QDA")&(pred_per_model_table$file==BEST_FILE)&(pred_per_model_table$branch==branch_num),]$predictors

    msg=sprintf("Testing -> Branch: %d -> Predictor: %s -> Model: %s",branch_num, pred, "QDA")
    message(msg)

    #QDA: Test Model
    branch_probs=predict(testing_models[[branch_num]], newdata=branch_data)
    branch_pred =rep(0, nrow(branch_probs$posterior))#Error
    branch_pred[branch_probs$posterior[,2] > 0.5] = 1
    branch_mean=mean(branch_pred == branch_data$is_attributed)
    branch_cMatrix=confusionMatrix(branch_pred, branch_data$is_attributed)

    #QDA: Save Result
    final_results_table=rbind(final_results_table, data.frame(branch=branch_num, model="QDA",predictors=pred, accuracy=branch_mean, TP=branch_cMatrix$table[1], FN=branch_cMatrix$table[2], FP=branch_cMatrix$table[3], TN=branch_cMatrix$table[4], accuracy_cMatrix=as.numeric(branch_cMatrix$overall["Accuracy"])))

  }else if(model_per_branch_table[branch_num,]$model=="SVM"){
    #SVM:
    #SVM: Assign Predictor
    pred=pred_per_model_table[(pred_per_model_table$model=="SVM")&(pred_per_model_table$file==BEST_FILE)&(pred_per_model_table$branch==branch_num),]$predictors

    msg=sprintf("Testing -> Branch: %d -> Predictor: %s -> Model: %s",branch_num, pred, "SVM")
    message(msg)

    #SVM: Test Model
    branch_probs=predict(testing_models[[branch_num]], newdata=branch_data)
    branch_pred =rep(0, length(branch_probs))#Error
    branch_pred[branch_probs > 0.5] = 1
    branch_mean=mean(branch_pred == branch_test$is_attributed)
    branch_cMatrix=confusionMatrix(branch_pred, branch_data$is_attributed)

    #SVM: Save Result
    final_results_table=rbind(final_results_table, data.frame(branch=branch_num, model="SVM",predictors=pred, accuracy=branch_mean, TP=branch_cMatrix$table[1], FN=branch_cMatrix$table[2], FP=branch_cMatrix$table[3], TN=branch_cMatrix$table[4], accuracy_cMatrix=as.numeric(branch_cMatrix$overall["Accuracy"])))

  }else if(model_per_branch_table[branch_num,]$model=="NaiveBayes"){
    #NaiveBayes:
    #NaiveBayes: Assign Predictor
    pred=pred_per_model_table[(pred_per_model_table$model=="NaiveBayes")&(pred_per_model_table$file==BEST_FILE)&(pred_per_model_table$branch==branch_num),]$predictors

    msg=sprintf("Training and testing -> Branch: %d -> Predictor: %s -> Model: %s",branch_num, pred, "NaiveBayes")
    message(msg)

    #NaiveBayes: Test Model
    branch_probs=predict(testing_models[[branch_num]], newdata=branch_data)
    branch_pred =branch_probs
    branch_mean=mean(branch_pred == branch_test$is_attributed)
    branch_cMatrix=confusionMatrix(branch_pred, branch_data$is_attributed)

    #NaiveBayes: Save Result
    final_results_table=rbind(final_results_table, data.frame(branch=branch_num, model="NaiveBayes",predictors=pred, accuracy=branch_mean, TP=branch_cMatrix$table[1], FN=branch_cMatrix$table[2], FP=branch_cMatrix$table[3], TN=branch_cMatrix$table[4], accuracy_cMatrix=as.numeric(branch_cMatrix$overall["Accuracy"])))
  }else{
    final_results_table=rbind(final_results_table, data.frame(branch=branch_num, model="ERROR",predictors=pred, accuracy=0, TP=0, FN=0, FP=0, TN=0, accuracy_cMatrix=0))
  }
}
final_balance_table$totalPer=(final_balance_table$human + final_balance_table$bot)/sum(final_balance_table$human + final_balance_table$bot)

#Total Accuracy
#total_accuracy=sum(final_balance_table$totalPer * final_results_table$accuracy_cMatrix)
total_accuracy=sum(final_balance_table$totalPer * final_results_table$accuracy)
total_accuracy

write.csv(final_balance_table, file="./results/final_balance_table.csv")
write.csv(final_results_table, file="./results/final_results_table.csv")

final_values=data.frame(BEST_FILE=BEST_FILE, total_accuracy=total_accuracy)
write.csv(final_values, file="./results/final_values.csv")