handle OOM exception in optimizer

pinky/src/optimize.py

@@ -95,7 +95,11 @@ class Optimizer(Object):
         args = dict(zip(self.optimizer_keys, args))
         self.model.outdir = self.log_dir_name(args)
         self.announce_test(args)
-        return self.model.train(args)['loss']
+        try:
+            return self.model.train(args)['loss']
+        except tf.errors.ResourceExhaustedError as e:
+            logger.warn(e)
+            logger.warn('Skipping this test')
 
     def optimize(self, model):
         '''Calling this method to optimize a :py:class:`pinky.model.Model`