Offset categorical probabilities

Offset the categorical probabilities by a small positive constant for
numerical stability.

agent/nonlinear/continuous/policy/CategoricalMLP.go

@@ -17,6 +17,8 @@ import (
 	"gorgonia.org/tensor"
 )
 
+const minProb float64 = 1e-5
+
 // CategoricalMLP implements a categorical policy using an MLP to
 // predict action logits in each state. Given an environment with N
 // actions in each state, the probabilities of selecting any action
@@ -117,6 +119,10 @@ func NewCategoricalMLP(env environment.Environment, batchForLogProb int,
 	logits = G.Must(G.BroadcastSub(logits, max, nil, []byte{1}))
 	probs := G.Must(G.Exp(logits))
 
+	// Offset the probabilities
+	offset := G.NewConstant(minProb, G.WithShape())
+	probs = G.Must(G.Add(probs, offset))
+
 	// Compute the log probability of actions that are input by an
 	// external source using the LogProbOf() method.
 	actionIndices := G.NewMatrix(

expconfig/VPG_Categorical_Cartpole.json

@@ -76,7 +76,7 @@
 						"Beta1": 0.9,
 						"Beta2": 0.999,
 						"Batch": 1,
-						"Clip": -1.0
+						"Clip": 0.5
 					}
 				}
 			],
@@ -89,7 +89,7 @@
 						"Beta1": 0.9,
 						"Beta2": 0.999,
 						"Batch": 1,
-						"Clip": -1.0
+						"Clip": 0.5
 					}
 				}
 			],