machine learning - Unable to save model using call backs in keras

Question

I was working on an image recognition problem. I am training my model for 200 epochs. And I want to save the model after every epoch if it has the best validation accuracy so far. This is my code,

from keras.models import Sequential
from keras.models import Model
from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau, TensorBoard
from keras import optimizers, losses, activations, models
from keras.layers import Convolution2D, Dense, Input, Flatten, Dropout, MaxPooling2D, BatchNormalization, GlobalAveragePooling2D, Concatenate
from keras import applications
from keras import backend as K
from keras import callbacks
from keras.preprocessing.image import ImageDataGenerator

ROWS,COLS = 669,1026


input_shape = (ROWS, COLS, 3)

base_model = applications.VGG19(weights='imagenet', 
                                include_top=False, 
                                input_shape=(ROWS, COLS,3))

l = 0
for layer in base_model.layers:
    layer.trainable = False
    l += 1

c = 0
for layer in base_model.layers:
    c += 1
    if c > l-5:
        layer.trainable = True 

for layer in base_model.layers:
    print(layer,layer.trainable)

base_model.summary()

add_model = Sequential()
add_model.add(base_model)
add_model.add(GlobalAveragePooling2D())
add_model.add(Dense(514, activation='relu'))
add_model.add(Dense(128, activation='relu'))
add_model.add(Dense(64, activation='relu'))
add_model.add(Dropout(0.5))
add_model.add(Dense(8, activation='relu'))
add_model.add(Dropout(0.5))
add_model.add(Dense(1, activation='sigmoid'))

model = add_model
# model.compile(loss='binary_crossentropy', 
#               optimizer=optimizers.SGD(lr=1e-, 
#                                        momentum=0.9),
#               metrics=['accuracy'])
model.compile(loss='binary_crossentropy', 
              optimizer='adam',
              metrics=['accuracy'])
model.summary()

train_data_dir = '/home/spectrograms/train'
validation_data_dir = '/home/spectrograms/test'
nb_train_samples = 791
nb_validation_samples = 198
epochs = 200
batch_size = 3

if K.image_data_format() == 'channels_first':
    input_shape = (3, ROWS, COLS)
else:
    input_shape = (ROWS, COLS,3)


# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
    rescale=1. / 255,
    shear_range=0,
    zoom_range=0,
    horizontal_flip=False)

test_datagen = ImageDataGenerator(rescale=1. / 255)

train_generator = train_datagen.flow_from_directory(
    train_data_dir,
    target_size=(ROWS, COLS),
    batch_size=batch_size,
    class_mode='binary')

validation_generator = test_datagen.flow_from_directory(
    validation_data_dir,
    target_size=(ROWS, COLS),
    batch_size=batch_size,
    class_mode='binary')

checkpoint_filepath = '/home/CNN/saved_model/checkpoints/checkpoint-{epoch:02d}-{val_loss:.2f}.h5'

model_checkpoint_callback = callbacks.ModelCheckpoint(
    filepath=checkpoint_filepath,
    save_weights_only=False,
    monitor='val_accuracy',
    mode='max',
    save_best_only=True)

model.fit_generator(
    train_generator,
    steps_per_epoch=nb_train_samples // batch_size,
    epochs=epochs,
    callbacks = [model_checkpoint_callback],
    validation_data=validation_generator,
    validation_steps=nb_validation_samples // batch_size)

But I am getting the error OSError: Unable to create file (unable to open file: name = '/home/CNN/saved_model/checkpoints/checkpoint-01-0.69.h5', errno = 2, error message = 'No such file or directory', flags = 13, o_flags = 242)

But the file directory actually exit. I don't understand what the issue is here.

question from:https://stackoverflow.com/questions/65839760/unable-to-save-model-using-call-backs-in-keras

Answer 1

Add the following in your code

save_dir = '/home/CNN/saved_model/checkpoints/'
if not os.path.exists(save_dir):
   os.makedirs(save_dir)

checkpoint_filepath = os.path.join(save_dir, "checkpoint-{epoch:02d}-{val_loss:.2f}.h5")

model_checkpoint_callback = callbacks.ModelCheckpoint(
filepath=checkpoint_filepath,
save_weights_only=False,
monitor='val_accuracy',
mode='max',
save_best_only=True)