Learn by Coding Examples in Applied Machine Learning

How to Evaluate Your Machine Learning Algorithms in Python using Scikit-Learn?

In [4]:
# ignore warnings
import warnings
warnings.filterwarnings("ignore")

Evaluate using a train and a test set

In [5]:
import pandas
from pandas import read_csv
from sklearn import model_selection
from sklearn.linear_model import LogisticRegression

# load data
filename = 'pima.indians.diabetes.data.csv'

names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']

dataframe = read_csv(filename, names=names)
print(); print(dataframe.head())

array = dataframe.values
X = array[:,0:8]
Y = array[:,8]

test_size = 0.33
seed = 7

X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, test_size=test_size, random_state=seed)

model = LogisticRegression()

model.fit(X_train, Y_train)

result = model.score(X_test, Y_test)

print(); print("Accuracy: %.3f%%" % (result*100.0))

   preg  plas  pres  skin  test  mass   pedi  age  class
0     6   148    72    35     0  33.6  0.627   50      1
1     1    85    66    29     0  26.6  0.351   31      0
2     8   183    64     0     0  23.3  0.672   32      1
3     1    89    66    23    94  28.1  0.167   21      0
4     0   137    40    35   168  43.1  2.288   33      1

Accuracy: 75.591%

Evaluate using Cross Validation

In [6]:
import pandas
from pandas import read_csv
from sklearn import model_selection
from sklearn.linear_model import LogisticRegression

# load data
filename = 'pima.indians.diabetes.data.csv'

names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']

dataframe = read_csv(filename, names=names)
print(); print(dataframe.head())

array = dataframe.values
X = array[:,0:8]
Y = array[:,8]

num_instances = len(X)

seed = 7

kfold = model_selection.KFold(n_splits=10, random_state=seed)

model = LogisticRegression()

results = model_selection.cross_val_score(model, X, Y, cv=kfold)

print(); print("Accuracy: %.3f%% (%.3f%%)" % (results.mean()*100.0, results.std()*100.0))

   preg  plas  pres  skin  test  mass   pedi  age  class
0     6   148    72    35     0  33.6  0.627   50      1
1     1    85    66    29     0  26.6  0.351   31      0
2     8   183    64     0     0  23.3  0.672   32      1
3     1    89    66    23    94  28.1  0.167   21      0
4     0   137    40    35   168  43.1  2.288   33      1

Accuracy: 76.951% (4.841%)

Evaluate using Leave One Out Cross Validation

In [7]:
import pandas
from pandas import read_csv
from sklearn import model_selection
from sklearn.linear_model import LogisticRegression

# load data
filename = 'pima.indians.diabetes.data.csv'

names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']

dataframe = read_csv(filename, names=names)
print(); print(dataframe.head())

array = dataframe.values
X = array[:,0:8]
Y = array[:,8]

num_folds = 10

num_instances = len(X)

loocv = model_selection.LeaveOneOut()

model = LogisticRegression()

results = model_selection.cross_val_score(model, X, Y, cv=loocv)

print(); print("Accuracy: %.3f%% (%.3f%%)" % (results.mean()*100.0, results.std()*100.0))

   preg  plas  pres  skin  test  mass   pedi  age  class
0     6   148    72    35     0  33.6  0.627   50      1
1     1    85    66    29     0  26.6  0.351   31      0
2     8   183    64     0     0  23.3  0.672   32      1
3     1    89    66    23    94  28.1  0.167   21      0
4     0   137    40    35   168  43.1  2.288   33      1

Accuracy: 76.823% (42.196%)

Evaluate using Shuffle Split Cross Validation

In [8]:
import pandas
from pandas import read_csv
from sklearn import model_selection
from sklearn.linear_model import LogisticRegression

# load data
filename = 'pima.indians.diabetes.data.csv'

names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']

dataframe = read_csv(filename, names=names)
print(); print(dataframe.head())

array = dataframe.values
X = array[:,0:8]
Y = array[:,8]

num_samples = 10

test_size = 0.33

num_instances = len(X)

seed = 7

kfold = model_selection.ShuffleSplit(n_splits=10, test_size=test_size, random_state=seed)

model = LogisticRegression()

results = model_selection.cross_val_score(model, X, Y, cv=kfold)

print(); print("Accuracy: %.3f%% (%.3f%%)" % (results.mean()*100.0, results.std()*100.0))

   preg  plas  pres  skin  test  mass   pedi  age  class
0     6   148    72    35     0  33.6  0.627   50      1
1     1    85    66    29     0  26.6  0.351   31      0
2     8   183    64     0     0  23.3  0.672   32      1
3     1    89    66    23    94  28.1  0.167   21      0
4     0   137    40    35   168  43.1  2.288   33      1

Accuracy: 76.535% (1.672%)
In [ ]: