ROC and Confusion Matrix for Classifier in Python

We use the data from sklearn library(need to download face datasets separately), and the IDE is sublime text3. Most of the code comes from the book: https://www.goodreads.com/book/show/32439431-introduction-to-machine-learning-with-python?from_search=true

###################always keep the below code#####################
import os
import sys
sys.path.append('//anaconda/lib/python3.6/site-packages')
###################always keep the above code#####################

import numpy as np
import matplotlib.pyplot as plt 
from sklearn.model_selection import train_test_split

from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression
iris=load_iris()

X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, random_state=0)

logreg = LogisticRegression(C=0.1).fit(X_train, y_train)
pred_logreg = logreg.predict(X_test)
print("logreg score: %f" % logreg.score(X_test, y_test))

###1. Confusion Matrix
from sklearn.metrics import confusion_matrix
confusion = confusion_matrix(y_test, pred_logreg)
print(confusion)
#[[13  0  0]
# [ 0  2 14]
# [ 0  0  9]]

###2.  ROC curves
from sklearn.metrics import precision_recall_curve
from sklearn.datasets import make_blobs
from sklearn.svm import SVC
from sklearn.datasets import load_digits
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
digits = load_digits()
y = digits.target == 9
X_train, X_test, y_train, y_test = train_test_split(
    digits.data, y, random_state=0)
plt.figure()
for gamma in [1, 0.05, 0.01]:
    svc = SVC(gamma=gamma).fit(X_train, y_train)
    accuracy = svc.score(X_test, y_test)
    auc = roc_auc_score(y_test, svc.decision_function(X_test))
    fpr, tpr, _ = roc_curve(y_test , svc.decision_function(X_test))
    print("gamma = %.02f  accuracy = %.02f  AUC = %.02f" % (gamma, accuracy, auc))
    plt.plot(fpr, tpr, label="gamma=%.03f" % gamma, linewidth=4)
plt.xlabel("FPR")
plt.ylabel("TPR")
plt.xlim(-0.01, 1)
plt.ylim(0, 1.02)
plt.legend(loc="best")
plt.show()
#gamma = 1.00  accuracy = 0.90  AUC = 0.50
#gamma = 0.05  accuracy = 0.90  AUC = 1.00
#gamma = 0.01  accuracy = 0.90  AUC = 1.00

###Two detailed evaluation methods on the task of classifying the 10 different hand-written digits
from sklearn.metrics import accuracy_score
X_train, X_test, y_train, y_test = train_test_split(
    digits.data, digits.target, random_state=0)
lr = LogisticRegression().fit(X_train, y_train)
pred = lr.predict(X_test)
print("accuracy: %0.3f" % accuracy_score(y_test, pred))
print("confusion matrix:")
print(confusion_matrix(y_test, pred))
#accuracy: 0.953
#confusion matrix:
#[[37  0  0  0  0  0  0  0  0  0]
# [ 0 39  0  0  0  0  2  0  2  0]
# [ 0  0 41  3  0  0  0  0  0  0]
# [ 0  0  1 43  0  0  0  0  0  1]
# [ 0  0  0  0 38  0  0  0  0  0]
# [ 0  1  0  0  0 47  0  0  0  0]
# [ 0  0  0  0  0  0 52  0  0  0]
# [ 0  1  0  1  1  0  0 45  0  0]
# [ 0  3  1  0  0  0  0  0 43  1]
# [ 0  0  0  1  0  1  0  0  1 44]]
from sklearn.metrics import classification_report
print(classification_report(y_test, pred))
#             precision    recall  f1-score   support#

#          0       1.00      1.00      1.00        37
#          1       0.89      0.91      0.90        43
#          2       0.95      0.93      0.94        44
#          3       0.90      0.96      0.92        45
#          4       0.97      1.00      0.99        38
#          5       0.98      0.98      0.98        48
#          6       0.96      1.00      0.98        52
#          7       1.00      0.94      0.97        48
#          8       0.93      0.90      0.91        48
#          9       0.96      0.94      0.95        47#

#avg / total       0.95      0.95      0.95       450

figure_1.png

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s