Learn by Coding Examples in Applied Machine Learning

Keras Deep Learning with Grid Search using sklearn

In [3]:
# ignore warnings
import warnings
warnings.filterwarnings("ignore")
In [4]:
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import GridSearchCV
import numpy

# Function to create model, required for KerasClassifier
def create_model(optimizer='rmsprop', init='glorot_uniform'):
    # create model
    model = Sequential()
    model.add(Dense(12, input_dim=8, kernel_initializer=init, activation='relu'))
    model.add(Dense(8, kernel_initializer=init, activation='relu'))
    model.add(Dense(1, kernel_initializer=init, activation='sigmoid'))
    
    # Compile model
    model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
    return model

# fix random seed for reproducibility
seed = 7
numpy.random.seed(seed)

# load pima indians dataset
dataset = numpy.loadtxt("pima.indians.diabetes.data.csv", delimiter=",")

# split into input (X) and output (Y) variables
X = dataset[:,0:8]
Y = dataset[:,8]

# create model
model = KerasClassifier(build_fn=create_model, verbose=0)

# grid search epochs, batch size and optimizer
optimizers = ['rmsprop', 'adam']
init = ['glorot_uniform', 'normal', 'uniform']
epochs = [50, 100, 150]
batches = [5, 10, 20]
param_grid = dict(optimizer=optimizers, epochs=epochs, batch_size=batches, init=init)
grid = GridSearchCV(estimator=model, param_grid=param_grid)
grid_result = grid.fit(X, Y)

# summarize results
print(); print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))

means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']

print()
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))
Best: 0.753906 using {'batch_size': 5, 'epochs': 150, 'init': 'uniform', 'optimizer': 'adam'}

0.699219 (0.017758) with: {'batch_size': 5, 'epochs': 50, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}
0.665365 (0.034987) with: {'batch_size': 5, 'epochs': 50, 'init': 'glorot_uniform', 'optimizer': 'adam'}
0.707031 (0.024080) with: {'batch_size': 5, 'epochs': 50, 'init': 'normal', 'optimizer': 'rmsprop'}
0.716146 (0.015733) with: {'batch_size': 5, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam'}
0.709635 (0.023939) with: {'batch_size': 5, 'epochs': 50, 'init': 'uniform', 'optimizer': 'rmsprop'}
0.680990 (0.032734) with: {'batch_size': 5, 'epochs': 50, 'init': 'uniform', 'optimizer': 'adam'}
0.695313 (0.017758) with: {'batch_size': 5, 'epochs': 100, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}
0.677083 (0.045143) with: {'batch_size': 5, 'epochs': 100, 'init': 'glorot_uniform', 'optimizer': 'adam'}
0.723958 (0.018414) with: {'batch_size': 5, 'epochs': 100, 'init': 'normal', 'optimizer': 'rmsprop'}
0.714844 (0.027805) with: {'batch_size': 5, 'epochs': 100, 'init': 'normal', 'optimizer': 'adam'}
0.731771 (0.018136) with: {'batch_size': 5, 'epochs': 100, 'init': 'uniform', 'optimizer': 'rmsprop'}
0.744792 (0.032106) with: {'batch_size': 5, 'epochs': 100, 'init': 'uniform', 'optimizer': 'adam'}
0.699219 (0.030758) with: {'batch_size': 5, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}
0.690104 (0.040386) with: {'batch_size': 5, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'adam'}
0.734375 (0.029232) with: {'batch_size': 5, 'epochs': 150, 'init': 'normal', 'optimizer': 'rmsprop'}
0.736979 (0.013279) with: {'batch_size': 5, 'epochs': 150, 'init': 'normal', 'optimizer': 'adam'}
0.748698 (0.013279) with: {'batch_size': 5, 'epochs': 150, 'init': 'uniform', 'optimizer': 'rmsprop'}
0.753906 (0.016877) with: {'batch_size': 5, 'epochs': 150, 'init': 'uniform', 'optimizer': 'adam'}
0.661458 (0.028587) with: {'batch_size': 10, 'epochs': 50, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}
0.653646 (0.015733) with: {'batch_size': 10, 'epochs': 50, 'init': 'glorot_uniform', 'optimizer': 'adam'}
0.686198 (0.015733) with: {'batch_size': 10, 'epochs': 50, 'init': 'normal', 'optimizer': 'rmsprop'}
0.703125 (0.031412) with: {'batch_size': 10, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam'}
0.705729 (0.026557) with: {'batch_size': 10, 'epochs': 50, 'init': 'uniform', 'optimizer': 'rmsprop'}
0.709635 (0.036828) with: {'batch_size': 10, 'epochs': 50, 'init': 'uniform', 'optimizer': 'adam'}
0.669271 (0.027126) with: {'batch_size': 10, 'epochs': 100, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}
0.559896 (0.168840) with: {'batch_size': 10, 'epochs': 100, 'init': 'glorot_uniform', 'optimizer': 'adam'}
0.718750 (0.019918) with: {'batch_size': 10, 'epochs': 100, 'init': 'normal', 'optimizer': 'rmsprop'}
0.730469 (0.027251) with: {'batch_size': 10, 'epochs': 100, 'init': 'normal', 'optimizer': 'adam'}
0.717448 (0.020505) with: {'batch_size': 10, 'epochs': 100, 'init': 'uniform', 'optimizer': 'rmsprop'}
0.694010 (0.023939) with: {'batch_size': 10, 'epochs': 100, 'init': 'uniform', 'optimizer': 'adam'}
0.694010 (0.015073) with: {'batch_size': 10, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}
0.703125 (0.028348) with: {'batch_size': 10, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'adam'}
0.733073 (0.020505) with: {'batch_size': 10, 'epochs': 150, 'init': 'normal', 'optimizer': 'rmsprop'}
0.701823 (0.014731) with: {'batch_size': 10, 'epochs': 150, 'init': 'normal', 'optimizer': 'adam'}
0.748698 (0.017566) with: {'batch_size': 10, 'epochs': 150, 'init': 'uniform', 'optimizer': 'rmsprop'}
0.733073 (0.013279) with: {'batch_size': 10, 'epochs': 150, 'init': 'uniform', 'optimizer': 'adam'}
0.601562 (0.154680) with: {'batch_size': 20, 'epochs': 50, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}
0.678385 (0.014382) with: {'batch_size': 20, 'epochs': 50, 'init': 'glorot_uniform', 'optimizer': 'adam'}
0.671875 (0.016877) with: {'batch_size': 20, 'epochs': 50, 'init': 'normal', 'optimizer': 'rmsprop'}
0.692708 (0.022628) with: {'batch_size': 20, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam'}
0.692708 (0.017566) with: {'batch_size': 20, 'epochs': 50, 'init': 'uniform', 'optimizer': 'rmsprop'}
0.696615 (0.004872) with: {'batch_size': 20, 'epochs': 50, 'init': 'uniform', 'optimizer': 'adam'}
0.664062 (0.036782) with: {'batch_size': 20, 'epochs': 100, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}
0.680990 (0.016367) with: {'batch_size': 20, 'epochs': 100, 'init': 'glorot_uniform', 'optimizer': 'adam'}
0.701823 (0.012890) with: {'batch_size': 20, 'epochs': 100, 'init': 'normal', 'optimizer': 'rmsprop'}
0.707031 (0.013902) with: {'batch_size': 20, 'epochs': 100, 'init': 'normal', 'optimizer': 'adam'}
0.720052 (0.015073) with: {'batch_size': 20, 'epochs': 100, 'init': 'uniform', 'optimizer': 'rmsprop'}
0.713542 (0.019488) with: {'batch_size': 20, 'epochs': 100, 'init': 'uniform', 'optimizer': 'adam'}
0.677083 (0.016367) with: {'batch_size': 20, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}
0.701823 (0.019488) with: {'batch_size': 20, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'adam'}
0.726562 (0.026107) with: {'batch_size': 20, 'epochs': 150, 'init': 'normal', 'optimizer': 'rmsprop'}
0.725260 (0.024150) with: {'batch_size': 20, 'epochs': 150, 'init': 'normal', 'optimizer': 'adam'}
0.720052 (0.015073) with: {'batch_size': 20, 'epochs': 150, 'init': 'uniform', 'optimizer': 'rmsprop'}
0.720052 (0.012075) with: {'batch_size': 20, 'epochs': 150, 'init': 'uniform', 'optimizer': 'adam'}
In [ ]: