optunaでCNNのパラメータチューニングしてみる。MNISTコンペ精度向上添え。

#データインポート
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

train = pd.read_csv('../input/digit-recognizer/train.csv') 
test = pd.read_csv('../input/digit-recognizer/test.csv')

#　教師データ、テストデータに分類して、float型に変換する。
X_train = (train.iloc[:,1:].values).astype('float32') 
y_train = train.iloc[:,0].values
X_test = test.values.astype('float32')

#精度確認のため、教師データをさらに教師データ、テストデータに分割
X_train2, X_test2, y_train2, y_test2 = train_test_split(X_train,
                                                        y_train, 
                                                        test_size = 0.2,
                                                        train_size = 0.8,
                                                        stratify = y_train)

#データreshape（28×28の行列に変換）
img_rows, img_cols = 28, 28
num_classes = 10

X_train2 = X_train2.reshape(X_train2.shape[0], img_rows, img_cols, 1)
X_test2 = X_test2.reshape(X_test2.shape[0], img_rows, img_cols, 1)

#y_trainのデータをto_categoricalで2値クラスの行列へ変換
y_train2 = keras.utils.to_categorical(y_train2, num_classes)

import optuna
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D

img_rows, img_cols = 28, 28
num_classes = 10

#CNNモデルの定義
def create_model(num_layer, mid_units, num_filters,dropout_rate):
    
    model = Sequential()
    model.add(Conv2D(filters=num_filters[0], kernel_size=(3, 3),
                 activation="relu",
                 input_shape=(img_rows, img_cols, 1)))
    for i in range(1,num_layer):
        model.add(Conv2D(filters=num_filters[i], kernel_size=(3,3), padding="same", activation="relu"))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(dropout_rate))
    model.add(Flatten())
    model.add(Dense(mid_units))
    model.add(Dropout(dropout_rate))
    model.add(Dense(num_classes, activation='softmax'))
    
    return model

#パラメータ最適化部
def objective(trial):
    print("Optimize Start")
    
    #セッションのクリア
    keras.backend.clear_session()
    
    #畳込み層の数のパラメータ
    num_layer = trial.suggest_int("num_layer", 2, 5)
    
    #全結合層のノード数
    mid_units = int(trial.suggest_discrete_uniform("mid_units", 100, 300, 100))
    
    #各畳込み層のフィルタ数
    num_filters = [int(trial.suggest_discrete_uniform("num_filter_"+str(i), 16, 128, 16)) for i in range(num_layer)]
    
    #Dropout率
    dropout_rate = trial.suggest_uniform('dropout_rate', 0.0, 0.5)
    
    #optimizer
    optimizer = trial.suggest_categorical("optimizer", ["sgd", "adam"])
    
    model = create_model(num_layer, mid_units, num_filters,dropout_rate)
    model.compile(optimizer=optimizer,
          loss="categorical_crossentropy",
          metrics=["accuracy"])
    
    history = model.fit(X_train2, y_train2, verbose=0, epochs=20, batch_size=128, validation_split=0.1)
    
    scores = model.evaluate(X_train2, y_train2)
    print('accuracy={}'.format(*scores))
    
    #検証用データに対する正答率が最大となるハイパーパラメータを求める
    return 1 - history.history["val_accuracy"][-1]

# studyオブジェクト生成
study = optuna.create_study()
# 最適化実行
study.optimize(objective, n_trials=30)#時短のため30回

study.best_params

#出力結果
{'num_layer ': 5,
 'num_filter_0': 48.0,
 'num_filter_1': 64.0,
 'num_filter_2': 96.0,
 'num_filter_3': 96.0,
 'num_filter_4': 128.0,
 'mid_units': 100.0,
 'optimizer': 'sgd',
 'dropout_rate'  0.4916904519518987 }

#最適paramterのセット
num_filters = [48,64,96,96,128]
mid_units= 100
dropout_rate = 0.4916904519518987
optimizer = 'sgd'

#最適化後のモデル(畳み込み層5層のCNN)
model_best = Sequential()
model_best.add(Conv2D(filters=num_filters[0], kernel_size=(3, 3),activation="relu",input_shape=(img_rows, img_cols, 1)))
model_best.add(Conv2D(filters=num_filters[1], kernel_size=(3,3), padding="same", activation="relu"))
model_best.add(Conv2D(filters=num_filters[2], kernel_size=(3,3), padding="same", activation="relu"))
model_best.add(Conv2D(filters=num_filters[3], kernel_size=(3,3), padding="same", activation="relu"))
model_best.add(Conv2D(filters=num_filters[4], kernel_size=(3,3), padding="same", activation="relu"))
model_best.add(MaxPooling2D(pool_size=(2, 2)))
model_best.add(Dropout(dropout_rate))
model_best.add(Flatten())
model_best.add(Dense(mid_units))
model_best.add(Dropout(dropout_rate))
model_best.add(Dense(num_classes, activation='softmax'))

#最適化手法などを決定
model_best.compile(loss=keras.losses.categorical_crossentropy,
              #optimizer=keras.optimizers.Adadelta(),
              optimizer=optimizer,
              metrics=['accuracy'])

#本番データをreshape
X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
y_train = keras.utils.to_categorical(y_train, num_classes)

#学習を開始
hist2 = model_best.fit(X_train, y_train,
                 batch_size=128,
                 epochs=20,
                 validation_split=0.1,
                 verbose=1)

scores_best = model_best.evaluate(X_train, y_train)

#出力結果
・
・
・
ETA: 0s - loss: 0.0118 - accuracy: 0.99 - 
ETA: 0s - loss: 0.0119 - accuracy: 0.99 - 
6s 4ms/step - loss: 0.0119 - accuracy: 0.9966

#loss
plt.plot(hist2.history['loss'])
plt.plot(hist2.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

#Accuracy
plt.figure()
plt.plot(hist2.history['accuracy'])
plt.plot(hist2.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

#テストデータの予測結果を出力（predeict_classes)
y_pred = model_best.predict_classes(X_test_best)
 
my_cnn = pd.DataFrame()
imageid = []
 
for i in range(len(X_test_best)):
    imageid.append(i+1)
    
my_cnn['ImageId'] = imageid
my_cnn["label"] = y_pred
my_cnn.to_csv("./cnn_optuna.csv", index=False)

print('csv書き出し終了')