ModelCheckpoint-with-multi_gpu_model

ModelCheckpoint() doesn't work well with multi_gpu_model. In keras/callbacks.py, adding the class MultiGPUCheckpointCallback() to save the base model. Instead of using ModelCheckpoint(), call MultiGPUCheckpointCallback() and pass in the base model.

classMultiGPUCheckpointCallback(Callback):
def__init__(self, filepath, base_model, monitor='val_loss', verbose=0,
save_best_only=False, save_weights_only=False,
mode='auto', period=1):
super(MultiGPUCheckpointCallback, self).__init__()
self.base_model=base_modelself.monitor=monitorself.verbose=verboseself.filepath=filepathself.save_best_only=save_best_onlyself.save_weights_only=save_weights_onlyself.period=periodself.epochs_since_last_save=0ifmodenotin ['auto', 'min', 'max']:
warnings.warn('ModelCheckpoint mode %s is unknown, ''fallback to auto mode.'% (mode),
RuntimeWarning)
mode='auto'ifmode=='min':
self.monitor_op=np.lessself.best=np.Infelifmode=='max':
self.monitor_op=np.greaterself.best=-np.Infelse:
if'acc'inself.monitororself.monitor.startswith('fmeasure'):
self.monitor_op=np.greaterself.best=-np.Infelse:
self.monitor_op=np.lessself.best=np.Infdefon_epoch_end(self, epoch, logs=None):
logs=logsor {}
self.epochs_since_last_save+=1ifself.epochs_since_last_save>=self.period:
self.epochs_since_last_save=0filepath=self.filepath.format(epoch=epoch+1, **logs)
ifself.save_best_only:
current=logs.get(self.monitor)
ifcurrentisNone:
warnings.warn('Can save best model only with %s available, ''skipping.'% (self.monitor), RuntimeWarning)
else:
ifself.monitor_op(current, self.best):
ifself.verbose>0:
print('Epoch %05d: %s improved from %0.5f to %0.5f,'' saving model to %s'% (epoch+1, self.monitor, self.best,
current, filepath))
self.best=currentifself.save_weights_only:
self.base_model.save_weights(filepath, overwrite=True)
else:
self.base_model.save(filepath, overwrite=True)
else:
ifself.verbose>0:
print('Epoch %05d: %s did not improve'%
(epoch+1, self.monitor))
else:
ifself.verbose>0:
print('Epoch %05d: saving model to %s'% (epoch+1, filepath))
ifself.save_weights_only:
self.base_model.save_weights(filepath, overwrite=True)
else:
self.base_model.save(filepath, overwrite=True)