Commit 40b5ea75 authored by Marius Kriegerowski's avatar Marius Kriegerowski

handle OOM exception in optimizer

parent 9a64146e
......@@ -95,7 +95,11 @@ class Optimizer(Object):
args = dict(zip(self.optimizer_keys, args))
self.model.outdir = self.log_dir_name(args)
self.announce_test(args)
return self.model.train(args)['loss']
try:
return self.model.train(args)['loss']
except tf.errors.ResourceExhaustedError as e:
logger.warn(e)
logger.warn('Skipping this test')
def optimize(self, model):
'''Calling this method to optimize a :py:class:`pinky.model.Model`
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment