reapplying changes

Author: tabergma

CHANGELOG.rst

@@ -36,10 +36,6 @@ Bugfixes
   Therefore, all 1.9 models `except for 1.9.6` will be compatible; a model trained on 1.9.6 will need
   to be retrained on 1.9.7.
 
-Miscellaneous internal changes
-------------------------------
-- #5631
-
 
 [1.9.6] - 2020-04-15
 ^^^^^^^^^^^^^^^^^^^^

changelog/5626.msic.rst

@@ -0,0 +1 @@
+Set regularization constant in ``EmbeddingIntentClassifier`` to 0.001.

changelog/5631.misc.rst

@@ -1255,7 +1255,7 @@ def _create_sequence(
             seq_ids = None
 
         inputs = self._combine_sparse_dense_features(
-            features, mask, name, sparse_dropout, dense_dropout,
+            features, mask, name, sparse_dropout, dense_dropout
         )
 
         inputs = self._tf_layers[f"ffnn.{name}"](inputs, self._training)
@@ -1272,6 +1272,10 @@ def _create_sequence(
             transformer_inputs, 1 - mask, self._training
         )
 
+        if self.config[NUM_TRANSFORMER_LAYERS] > 0:
+            # apply activation
+            outputs = tfa.activations.gelu(outputs)
+
         return outputs, inputs, seq_ids, lm_mask_bool
 
     def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]:
@@ -1283,7 +1287,7 @@ def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]:
         mask_label = self._compute_mask(label_lengths)
 
         x = self._create_bow(
-            self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name,
+            self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name
         )
         all_labels_embed = self._tf_layers[f"embed.{LABEL}"](x)
 
@@ -1432,7 +1436,7 @@ def batch_loss(
 
             label_ids = tf_batch_data[LABEL_IDS][0]
             label = self._create_bow(
-                tf_batch_data[LABEL_FEATURES], mask_label, self.label_name,
+                tf_batch_data[LABEL_FEATURES], mask_label, self.label_name
             )
             loss, acc = self._calculate_label_loss(cls, label, label_ids)
             self.intent_loss.update_state(loss)

rasa/nlu/classifiers/diet_classifier.py

@@ -626,7 +626,4 @@ def call(
         # if normalization is done in encoding layers, then it should also be done
         # on the output, since the output can grow very large, being the sum of
         # a whole stack of unnormalized layer outputs.
-        normalized_x = self._layer_norm(x)  # (batch_size, length, units)
-
-        # apply final activation
-        return tfa.activations.gelu(normalized_x)
+        return self._layer_norm(x)  # (batch_size, length, units)