Merge branch 'master' into issue4896

chkoss · web-flow · commit ca8c0130dd06 · 2020-01-28T18:44:28.000+01:00
diff --git a/changelog/4774.bugfix.rst b/changelog/4774.bugfix.rst
@@ -0,0 +1 @@
+Requests to ``/model/train`` do not longer block other requests to the Rasa server.
diff --git a/changelog/4902.feature.rst b/changelog/4902.feature.rst
@@ -0,0 +1,2 @@
+Added a new configuration parameter, ``ranking_length`` to the ``EmbeddingPolicy``, ``EmbeddingIntentClassifier``,
+and ``ResponseSelector`` classes.
diff --git a/changelog/4902.improvement.rst b/changelog/4902.improvement.rst
@@ -0,0 +1,2 @@
+The ``EmbeddingPolicy``, ``EmbeddingIntentClassifier``, and ``ResponseSelector`` now by default normalize confidence
+levels over the top 10 results. See :ref:`migration-to-rasa-1.7` for more details.
diff --git a/changelog/5029.misc.rst b/changelog/5029.misc.rst
diff --git a/data/test/many_intents.md b/data/test/many_intents.md
@@ -0,0 +1,59 @@
+## intent:handleinsult
+- you are an idiot
+- You lack understanding.
+
+## intent:thank
+- Thanks
+- Thank you
+
+## intent:telljoke
+- Tell me something that you think will make me laugh.
+- Entertain me with a joke.
+
+## intent:signup_newsletter
+- I wanna sign up for the newsletter.
+- I want to sign up for the newsletter.
+
+## intent:react_positive
+- you are funny
+- thats funny
+
+## intent:react_negative
+- i am sad
+- bad
+
+## intent:how_to_get_started
+- how do I get started with rasa
+- how do I use rasa
+
+## intent:technical_question
+- what is duckling
+- where to train intents in rasa?
+
+## intent:source_code
+- how it works?
+- where can i find this code
+
+## intent:pipeline_recommendation
+- what pipeline should I start with?
+- what is the right pipeline to choose?
+
+## intent:rasa_cost
+- is rasa free
+- are you really free
+
+## intent:nicetomeeyou
+- It’s great connecting with you.
+- Hi, nice to meet you!
+
+## intent:nlu_generation_tool_recommendation
+- which tools can I use to create nlu data
+- how can I get nlu data
+
+## intent:install_rasa
+- I want to install Rasa Stack
+- How to install Rasa?
+
+## intent:ask_which_events
+- Which community events do you have
+- Where can I meet Rasas
diff --git a/docs/core/policies.rst b/docs/core/policies.rst
@@ -271,6 +271,9 @@ It is recommended to use
               ``inner`` for ``softmax``, ``cosine`` for ``margin``;
             - ``loss_type`` sets the type of the loss function,
               it should be either ``softmax`` or ``margin``;
+            - ``ranking_length`` defines the number of top confidences over
+              which to normalize ranking results if ``loss_type: "softmax"``;
+              to turn off normalization set it to 0
             - ``mu_pos`` controls how similar the algorithm should try
               to make embedding vectors for correct intent labels,
               used only if ``loss_type`` is set to ``margin``;
diff --git a/docs/migration-guide.rst b/docs/migration-guide.rst
@@ -11,6 +11,18 @@ Migration Guide
 This page contains information about changes between major versions and
 how you can migrate from one version to another.
 
+.. _migration-to-rasa-1.7:
+
+Rasa 1.6 to Rasa 1.7
+--------------------
+
+General
+~~~~~~~
+- By default, the ``EmbeddingIntentClassifier``, ``EmbeddingPolicy``, and ``ResponseSelector`` will
+  now normalize the top 10 confidence results if the ``loss_type`` is ``"softmax"`` (which has been
+  default since 1.3, see :ref:`migration-to-rasa-1.3`). This is configurable via the ``ranking_length``
+  configuration parameter; to turn off normalization to match the previous behavior, set ``ranking_length: 0``.
+
 .. _migration-to-rasa-1.3:
 
 Rasa 1.2 to Rasa 1.3
@@ -82,7 +94,7 @@ General
 - If you were previously importing the ``Button`` or ``Element`` classes from
   ``rasa_core.dispatcher``, these are now to be imported from ``rasa_sdk.utils``.
 
-- Rasa NLU and Core previously used `separate configuration files 
+- Rasa NLU and Core previously used `separate configuration files
   <https://legacy-docs.rasa.com/docs/nlu/0.15.1/migrations/?&_ga=2.218966814.608734414.1560704810-314462423.1543594887#id1>`_.
   These two files should be merged into a single file either named ``config.yml``, or passed via the ``--config`` parameter.
 
diff --git a/docs/nlu/components.rst b/docs/nlu/components.rst
@@ -464,6 +464,9 @@ EmbeddingIntentClassifier
               ``inner`` for ``softmax``, ``cosine`` for ``margin``;
             - ``loss_type`` sets the type of the loss function,
               it should be either ``softmax`` or ``margin``;
+            - ``ranking_length`` defines the number of top confidences over
+              which to normalize ranking results if ``loss_type: "softmax"``;
+              to turn off normalization set it to 0
             - ``mu_pos`` controls how similar the algorithm should try
               to make embedding vectors for correct intent labels,
               used only if ``loss_type`` is set to ``margin``;
diff --git a/rasa/core/policies/embedding_policy.py b/rasa/core/policies/embedding_policy.py
@@ -74,6 +74,9 @@ class EmbeddingPolicy(Policy):
         "similarity_type": "auto",  # string 'auto' or 'cosine' or 'inner'
         # the type of the loss function
         "loss_type": "softmax",  # string 'softmax' or 'margin'
+        # number of top actions to normalize scores for softmax loss_type
+        # set to 0 to turn off normalization
+        "ranking_length": 10,
         # how similar the algorithm should try
         # to make embedding vectors for correct labels
         "mu_pos": 0.8,  # should be 0.0 < ... < 1.0 for 'cosine'
@@ -192,6 +195,7 @@ def _load_embedding_params(self, config: Dict[Text, Any]) -> None:
                 self.similarity_type = "inner"
             elif self.loss_type == "margin":
                 self.similarity_type = "cosine"
+        self.ranking_length = config["ranking_length"]
 
         self.mu_pos = config["mu_pos"]
         self.mu_neg = config["mu_neg"]
@@ -567,8 +571,12 @@ def predict_action_probabilities(
         tf_feed_dict = self.tf_feed_dict_for_prediction(tracker, domain)
 
         confidence = self.session.run(self.pred_confidence, feed_dict=tf_feed_dict)
+        confidence = confidence[0, -1, :]
 
-        return confidence[0, -1, :].tolist()
+        if self.loss_type == "softmax" and self.ranking_length > 0:
+            confidence = train_utils.normalize(confidence, self.ranking_length)
+
+        return confidence.tolist()
 
     def persist(self, path: Text) -> None:
         """Persists the policy to a storage."""
@@ -583,7 +591,11 @@ def persist(self, path: Text) -> None:
 
         self.featurizer.persist(path)
 
-        meta = {"priority": self.priority}
+        meta = {
+            "priority": self.priority,
+            "loss_type": self.loss_type,
+            "ranking_length": self.ranking_length,
+        }
 
         meta_file = os.path.join(path, "embedding_policy.json")
         rasa.utils.io.dump_obj_as_json_to_file(meta_file, meta)
@@ -665,7 +677,7 @@ def load(cls, path: Text) -> "EmbeddingPolicy":
 
         return cls(
             featurizer=featurizer,
-            priority=meta["priority"],
+            priority=meta.pop("priority"),
             graph=graph,
             session=session,
             user_placeholder=a_in,
@@ -677,4 +689,5 @@ def load(cls, path: Text) -> "EmbeddingPolicy":
             bot_embed=bot_embed,
             all_bot_embed=all_bot_embed,
             attention_weights=attention_weights,
+            **meta,
         )
diff --git a/rasa/core/processor.py b/rasa/core/processor.py
@@ -631,7 +631,7 @@ def _warn_about_new_slots(self, tracker, action_name, events) -> None:
                         warnings.warn(
                             f"Action '{action_name}' set a slot type '{e.key}' that "
                             f"it never set during the training. This "
-                            f"can throw of the prediction. Make sure to "
+                            f"can throw off the prediction. Make sure to "
                             f"include training examples in your stories "
                             f"for the different types of slots this "
                             f"action can return. Remember: you need to "
diff --git a/rasa/core/train.py b/rasa/core/train.py
@@ -173,7 +173,7 @@ async def do_compare_training(
 
 
 def do_interactive_learning(
-    args: argparse.Namespace, file_importer: TrainingDataImporter,
+    args: argparse.Namespace, file_importer: TrainingDataImporter
 ):
     from rasa.core.training import interactive
 
diff --git a/rasa/core/utils.py b/rasa/core/utils.py
@@ -471,13 +471,13 @@ def replace_floats_with_decimals(obj: Union[List, Dict], round_digits: int = 9)
     """
     if isinstance(obj, list):
         for i in range(len(obj)):
-            obj[i] = replace_floats_with_decimals(obj[i])
+            obj[i] = replace_floats_with_decimals(obj[i], round_digits)
         return obj
     elif isinstance(obj, dict):
         for j in obj:
-            obj[j] = replace_floats_with_decimals(obj[j])
+            obj[j] = replace_floats_with_decimals(obj[j], round_digits)
         return obj
-    elif isinstance(obj, float):
+    elif isinstance(obj, float) or isinstance(obj, Decimal):
         return round(Decimal(obj), round_digits)
     else:
         return obj
diff --git a/rasa/nlu/classifiers/embedding_intent_classifier.py b/rasa/nlu/classifiers/embedding_intent_classifier.py
@@ -93,6 +93,9 @@ class EmbeddingIntentClassifier(Component):
         "similarity_type": "auto",  # string 'auto' or 'cosine' or 'inner'
         # the type of the loss function
         "loss_type": "softmax",  # string 'softmax' or 'margin'
+        # number of top intents to normalize scores for softmax loss_type
+        # set to 0 to turn off normalization
+        "ranking_length": 10,
         # how similar the algorithm should try
         # to make embedding vectors for correct labels
         "mu_pos": 0.8,  # should be 0.0 < ... < 1.0 for 'cosine'
@@ -174,6 +177,7 @@ def _load_embedding_params(self, config: Dict[Text, Any]) -> None:
             elif self.loss_type == "margin":
                 self.similarity_type = "cosine"
 
+        self.ranking_length = config["ranking_length"]
         self.mu_pos = config["mu_pos"]
         self.mu_neg = config["mu_neg"]
         self.use_max_sim_neg = config["use_max_sim_neg"]
@@ -801,6 +805,10 @@ def _calculate_message_sim(
         message_sim = message_sim.flatten()  # sim is a matrix
 
         label_ids = message_sim.argsort()[::-1]
+
+        if self.loss_type == "softmax" and self.ranking_length > 0:
+            message_sim = train_utils.normalize(message_sim, self.ranking_length)
+
         message_sim[::-1].sort()
 
         # transform sim to python list for JSON serializing
@@ -838,8 +846,13 @@ def predict_label(
                 "confidence": message_sim[0],
             }
 
+            if self.ranking_length and 0 < self.ranking_length < LABEL_RANKING_LENGTH:
+                output_length = self.ranking_length
+            else:
+                output_length = LABEL_RANKING_LENGTH
+
             ranking = list(zip(list(label_ids), message_sim))
-            ranking = ranking[:LABEL_RANKING_LENGTH]
+            ranking = ranking[:output_length]
             label_ranking = [
                 {"name": self.inverted_label_dict[label_idx], "confidence": score}
                 for label_idx, score in ranking
diff --git a/rasa/nlu/selectors/embedding_response_selector.py b/rasa/nlu/selectors/embedding_response_selector.py
@@ -87,6 +87,9 @@ class ResponseSelector(EmbeddingIntentClassifier):
         "similarity_type": "auto",  # string 'auto' or 'cosine' or 'inner'
         # the type of the loss function
         "loss_type": "softmax",  # string 'softmax' or 'margin'
+        # number of top responses to normalize scores for softmax loss_type
+        # set to 0 to turn off normalization
+        "ranking_length": 10,
         # how similar the algorithm should try
         # to make embedding vectors for correct intent labels
         "mu_pos": 0.8,  # should be 0.0 < ... < 1.0 for 'cosine'
diff --git a/rasa/server.py b/rasa/server.py
@@ -1,3 +1,5 @@
+import asyncio
+import functools
 import logging
 import warnings
 import multiprocessing
@@ -736,9 +738,8 @@ async def add_message(request: Request, conversation_id: Text):
 
     @app.post("/model/train")
     @requires_auth(app, auth_token)
-    async def train(request: Request):
+    async def train(request: Request) -> HTTPResponse:
         """Train a Rasa Model."""
-        from rasa.train import train_async
 
         validate_request_body(
             request,
@@ -779,14 +780,25 @@ async def train(request: Request):
             with app.active_training_processes.get_lock():
                 app.active_training_processes.value += 1
 
-            model_path = await train_async(
+            info = dict(
                 domain=domain_path,
                 config=config_path,
                 training_files=temp_dir,
-                output_path=model_output_directory,
+                output=model_output_directory,
                 force_training=rjs.get("force", False),
             )
 
+            loop = asyncio.get_event_loop()
+
+            from rasa import train as train_model
+
+            # Declare `model_path` upfront to avoid pytype `name-error`
+            model_path: Optional[Text] = None
+            # pass `None` to run in default executor
+            model_path = await loop.run_in_executor(
+                None, functools.partial(train_model, **info)
+            )
+
             filename = os.path.basename(model_path) if model_path else None
 
             return await response.file(
diff --git a/rasa/train.py b/rasa/train.py
@@ -32,7 +32,11 @@ def train(
     loop: Optional[asyncio.AbstractEventLoop] = None,
 ) -> Optional[Text]:
     if loop is None:
-        loop = asyncio.get_event_loop()
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
 
     return loop.run_until_complete(
         train_async(
diff --git a/rasa/utils/train_utils.py b/rasa/utils/train_utils.py
@@ -1275,3 +1275,20 @@ def load_tensor(name: Text) -> Optional[Union["tf.Tensor", List["tf.Tensor"]]]:
         return tensor_list[0]
 
     return tensor_list
+
+
+def normalize(values: "np.ndarray", ranking_length: Optional[int] = 0) -> "np.ndarray":
+    """Normalizes an array of positive numbers over the top `ranking_length` values.
+
+    Other values will be set to 0.
+    """
+
+    new_values = values.copy()  # prevent mutation of the input
+    if 0 < ranking_length < len(new_values):
+        ranked = sorted(new_values, reverse=True)
+        new_values[new_values < ranked[ranking_length - 1]] = 0
+
+    if np.sum(new_values) > 0:
+        new_values = new_values / np.sum(new_values)
+
+    return new_values
diff --git a/tests/core/test_policies.py b/tests/core/test_policies.py
diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py
diff --git a/tests/nlu/classifiers/test_embedding_intent_classifier.py b/tests/nlu/classifiers/test_embedding_intent_classifier.py
diff --git a/tests/test_server.py b/tests/test_server.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Requests to ``/model/train`` do not longer block other requests to the Rasa server.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+Added a new configuration parameter, ``ranking_length`` to the ``EmbeddingPolicy``, ``EmbeddingIntentClassifier``,
	`2`	+and ``ResponseSelector`` classes.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+The ``EmbeddingPolicy``, ``EmbeddingIntentClassifier``, and ``ResponseSelector`` now by default normalize confidence
	`2`	+levels over the top 10 results. See :ref:`migration-to-rasa-1.7` for more details.