Merge pull request RasaHQ#5339 from RasaHQ/fix-flacky-test

Fix flacky test

Author: tabergma

CHANGELOG.rst

@@ -104,27 +104,6 @@ Improvements
   does not help for the case where messages are sent to an unavailable RabbitMQ instance.
 - `#5250 <https://github.com/rasahq/rasa/issues/5250>`_: Added support for mattermost connector to use bot accounts.
 - `#5266 <https://github.com/rasahq/rasa/issues/5266>`_: We updated our code to TensorFlow 2.
-
-  We added a new docker image for ConveRT.
-  The new images uses the following configuration
-
-  .. code-block:: yaml
-  
-      language: "en"
-
-      pipeline:
-      - name: ConveRTTokenizer
-      - name: ConveRTFeaturizer
-      - name: RegexFeaturizer
-      - name: LexicalSyntacticFeaturizer
-      - name: CountVectorsFeaturizer
-      - name: CountVectorsFeaturizer
-         analyzer: "char_wb"
-         min_ngram: 1
-         max_ngram: 4
-      - name: DIETClassifier
-      - name: EntitySynonymMapper
-      - name: ResponseSelector
 - `#5317 <https://github.com/rasahq/rasa/issues/5317>`_: Events exported using ``rasa export`` receive a message header if published through a
   ``PikaEventBroker``. The header is added to the message's ``BasicProperties.headers``
   under the ``rasa-export-process-id`` key

data/test_evaluations/end_to_end_story.md

@@ -1,29 +1,17 @@
 ## simple_story_with_only_start
-* greet: Hello
-    - utter_ask_howcanhelp
+* greet: /greet
+    - utter_greet
 
 ## simple_story_with_multiple_turns
-* greet: good morning
- - utter_ask_howcanhelp
-* inform: im looking for a [moderately](price:moderate) priced restaurant in the [east](location) part of town
- - utter_on_it
- - utter_ask_cuisine
+* greet: /greet
+ - utter_greet
+* default: /default
+ - utter_default
+ * goodbye: /goodbye
+ - utter_goodbye
 
  ## story_with_multiple_entities_correction_and_search
-* greet: hello
- - utter_ask_howcanhelp
-* inform: im looking for a [cheap](price:lo) restaurant which has [french](cuisine) food and is located in [bombay](location)
- - utter_on_it
- - utter_ask_numpeople
-* inform: for [six](people:6) please
- - utter_ask_moreupdates
-* inform: actually i need a [moderately](price:moderate) priced restaurant
- - utter_ask_moreupdates
-* deny: no
- - utter_ack_dosearch
- - action_search_restaurants
- - action_suggest
-
-## story_with_intent_restated
-* greet: /greet
- - utter_ask_howcanhelp
+* greet: /greet{"name": "Max"}
+ - utter_greet
+* default: /default
+ - utter_default

data/test_evaluations/form-end-to-end-stories.md

@@ -0,0 +1,37 @@
+## Happy path
+* greet: /greet
+    - utter_greet
+* request_restaurant: /request_restaurant
+    - restaurant_form
+    - form{"name": "restaurant_form"}
+    - form{"name": null}
+    - utter_slots_values
+* thankyou: /thankyou
+    - utter_noworries
+
+## Happy path with form prefix
+* greet: /greet
+    - utter_greet
+* request_restaurant: /request_restaurant
+    - restaurant_form
+    - form{"name": "restaurant_form"}
+* form: /inform{"cuisine": "afghan"} <!-- intent "inform" is ignored inside the form -->
+    - form: restaurant_form
+    - form{"name": null}
+    - utter_slots_values
+* thankyou: /thankyou
+    - utter_noworries
+ 
+## unhappy path
+* greet: /greet
+    - utter_greet
+* request_restaurant: /request_restaurant
+    - restaurant_form
+    - form{"name": "restaurant_form"}
+* chitchat: /chitchat
+    - utter_chitchat
+    - restaurant_form
+    - form{"name": null}
+    - utter_slots_values
+* thankyou: /thankyou
+    - utter_noworries

tests/core/conftest.py

@@ -8,21 +8,15 @@
 from typing import Text, Iterator
 
 import pytest
-from _pytest.tmpdir import TempdirFactory
 
 import rasa.utils.io
 from rasa.core.agent import Agent
 from rasa.core.channels.channel import CollectingOutputChannel, OutputChannel
-from rasa.core.domain import Domain, SessionConfig
+from rasa.core.domain import Domain
 from rasa.core.events import ReminderScheduled, UserUttered, ActionExecuted
-from rasa.core.interpreter import RegexInterpreter
 from rasa.core.nlg import TemplatedNaturalLanguageGenerator
-from rasa.core.policies.ensemble import PolicyEnsemble, SimplePolicyEnsemble
-from rasa.core.policies.memoization import (
-    AugmentedMemoizationPolicy,
-    MemoizationPolicy,
-    Policy,
-)
+from rasa.core.policies.ensemble import PolicyEnsemble
+from rasa.core.policies.memoization import Policy
 from rasa.core.processor import MessageProcessor
 from rasa.core.slots import Slot
 from rasa.core.tracker_store import InMemoryTrackerStore, MongoTrackerStore
@@ -229,23 +223,11 @@ def project() -> Text:
 
 
 @pytest.fixture
-async def restaurantbot(trained_async) -> Text:
-    restaurant_domain = os.path.join(RESTAURANTBOT_PATH, "domain.yml")
-    restaurant_config = os.path.join(RESTAURANTBOT_PATH, "config.yml")
-    restaurant_data = os.path.join(RESTAURANTBOT_PATH, "data/")
-
-    return await trained_async(restaurant_domain, restaurant_config, restaurant_data)
-
-
-@pytest.fixture
-async def form_bot(trained_async) -> Agent:
+async def form_bot_agent(trained_async, tmpdir_factory) -> Agent:
     zipped_model = await trained_async(
         domain="examples/formbot/domain.yml",
         config="examples/formbot/config.yml",
-        training_files=[
-            "examples/formbot/data/stories.md",
-            "examples/formbot/data/nlu.md",
-        ],
+        training_files=["examples/formbot/data/stories.md"],
     )
 
     return Agent.load_local_model(zipped_model)

tests/core/test_evaluation.py

@@ -1,10 +1,6 @@
 import os
-from typing import Text
-
 from pathlib import Path
 
-import pytest
-
 from rasa.core.test import _generate_trackers, collect_story_predictions, test
 
 # we need this import to ignore the warning...
@@ -34,63 +30,41 @@ async def test_evaluation_image_creation(tmpdir: Path, default_agent: Agent):
     assert os.path.isfile(stories_path)
 
 
-# TODO: this test is flaky - needs to be fixed
-@pytest.mark.xfail(strict=False)
-async def test_end_to_end_evaluation_script(restaurantbot: Text):
-    restaurantbot = Agent.load(restaurantbot)
+async def test_end_to_end_evaluation_script(default_agent: Agent):
     completed_trackers = await _generate_trackers(
-        END_TO_END_STORY_FILE, restaurantbot, use_e2e=True
+        END_TO_END_STORY_FILE, default_agent, use_e2e=True
     )
 
     story_evaluation, num_stories = collect_story_predictions(
-        completed_trackers, restaurantbot, use_e2e=True
+        completed_trackers, default_agent, use_e2e=True
     )
 
     serialised_store = [
-        "utter_ask_howcanhelp",
-        "action_listen",
-        "utter_ask_howcanhelp",
+        "utter_greet",
         "action_listen",
-        "utter_on_it",
-        "utter_ask_cuisine",
+        "utter_greet",
         "action_listen",
-        "utter_ask_howcanhelp",
+        "utter_default",
         "action_listen",
-        "utter_on_it",
-        "utter_ask_numpeople",
+        "utter_goodbye",
         "action_listen",
-        "utter_ask_moreupdates",
+        "utter_greet",
         "action_listen",
-        "utter_ask_moreupdates",
+        "utter_default",
         "action_listen",
-        "utter_ack_dosearch",
-        "action_search_restaurants",
-        "action_suggest",
-        "action_listen",
-        "utter_ask_howcanhelp",
-        "action_listen",
-        "greet",
         "greet",
-        "inform",
         "greet",
-        "inform",
-        "inform",
-        "inform",
-        "deny",
+        "default",
+        "goodbye",
         "greet",
-        "[moderately](price:moderate)",
-        "[east](location)",
-        "[cheap](price:lo)",
-        "[french](cuisine)",
-        "[bombay](location)",
-        "[six](people:6)",
-        "[moderately](price:moderate)",
+        "default",
+        '[{"name": "Max"}](name:Max)',
     ]
 
     assert story_evaluation.evaluation_store.serialise()[0] == serialised_store
     assert not story_evaluation.evaluation_store.has_prediction_target_mismatch()
     assert len(story_evaluation.failed_stories) == 0
-    assert num_stories == 4
+    assert num_stories == 3
 
 
 async def test_end_to_end_evaluation_script_unknown_entity(default_agent: Agent):
@@ -107,13 +81,13 @@ async def test_end_to_end_evaluation_script_unknown_entity(default_agent: Agent)
     assert num_stories == 1
 
 
-async def test_end_to_evaluation_with_forms(form_bot: Agent):
+async def test_end_to_evaluation_with_forms(form_bot_agent: Agent):
     test_stories = await _generate_trackers(
-        "examples/formbot/tests/end-to-end-stories.md", form_bot, use_e2e=True
+        "data/test_evaluations/form-end-to-end-stories.md", form_bot_agent, use_e2e=True
     )
 
     story_evaluation, num_stories = collect_story_predictions(
-        test_stories, form_bot, use_e2e=True
+        test_stories, form_bot_agent, use_e2e=True
     )
 
     assert not story_evaluation.evaluation_store.has_prediction_target_mismatch()