Use bigger epsilon for float16 so that it does not become 0.

abergeron · abergeron · commit cfba0cd11bab · 2015-05-21T14:55:44.000-04:00
diff --git a/code/lstm.py b/code/lstm.py
@@ -332,7 +332,11 @@ def build_model(tparams, options):
     f_pred_prob = theano.function([x, mask], pred, name='f_pred_prob')
     f_pred = theano.function([x, mask], pred.argmax(axis=1), name='f_pred')
 
-    cost = -tensor.log(pred[tensor.arange(n_samples), y] + 1e-8).mean()
+    off = 1e-8
+    if pred.dtype == 'float16':
+        off = 1e-6
+
+    cost = -tensor.log(pred[tensor.arange(n_samples), y] + off).mean()
 
     return use_noise, x, mask, y, f_pred_prob, f_pred, cost
 
