Include source in output of failed stories only

The file source was always included in the tracker's sender id.
This causes that persisted story to have 'different' trackers because the tracker's sender id will be different (because of the source).

To make sure the impact is minimal only failed stories will be exported with the source of the story file.
For this reason the tracker has been extended with an optional sender_source paramenter.

(cherry picked from commit 090e55134d9922d13ba626b1311eeecaa8e04915)

Author: cheemingli

rasa/core/test.py

@@ -348,7 +348,10 @@ def _predict_tracker_actions(
     events = list(tracker.events)
 
     partial_tracker = DialogueStateTracker.from_events(
-        tracker.sender_id, events[:1], agent.domain.slots
+        tracker.sender_id,
+        events[:1],
+        agent.domain.slots,
+        sender_source=tracker.sender_source,
     )
 
     tracker_actions = []
@@ -477,7 +480,7 @@ def log_failed_stories(failed, out_directory):
             f.write("<!-- All stories passed -->")
         else:
             for failure in failed:
-                f.write(failure.export_stories())
+                f.write(failure.export_stories(include_source=True))
                 f.write("\n\n")
 
 

rasa/core/trackers.py

@@ -98,8 +98,9 @@ def from_events(
         evts: List[Event],
         slots: Optional[List[Slot]] = None,
         max_event_history: Optional[int] = None,
+        sender_source: Optional[Text] = None,
     ):
-        tracker = cls(sender_id, slots, max_event_history)
+        tracker = cls(sender_id, slots, max_event_history, sender_source)
         for e in evts:
             tracker.update(e)
         return tracker
@@ -109,6 +110,7 @@ def __init__(
         sender_id: Text,
         slots: Optional[Iterable[Slot]],
         max_event_history: Optional[int] = None,
+        sender_source: Optional[Text] = None,
     ) -> None:
         """Initialize the tracker.
 
@@ -127,6 +129,8 @@ def __init__(
             self.slots = {slot.name: copy.deepcopy(slot) for slot in slots}
         else:
             self.slots = AnySlotDict()
+        # file source of the messages
+        self.sender_source = sender_source
 
         ###
         # current state of the tracker - MUST be re-creatable by processing
@@ -448,13 +452,18 @@ def update(self, event: Event, domain: Optional[Domain] = None) -> None:
             for e in domain.slots_for_entities(event.parse_data["entities"]):
                 self.update(e)
 
-    def export_stories(self, e2e: bool = False) -> Text:
+    def export_stories(self, e2e: bool = False, include_source: bool = False) -> Text:
         """Dump the tracker as a story in the Rasa Core story format.
 
         Returns the dumped tracker as a string."""
         from rasa.core.training.structures import Story
 
-        story = Story.from_events(self.applied_events(), self.sender_id)
+        story_name = (
+            f"{self.sender_id} ({self.sender_source})"
+            if include_source
+            else self.sender_id
+        )
+        story = Story.from_events(self.applied_events(), story_name)
         return story.as_story_string(flat=True, e2e=e2e)
 
     def export_stories_to_file(self, export_path: Text = "debug.md") -> None:

rasa/core/training/generator.py

@@ -81,7 +81,9 @@ def init_copy(self) -> "TrackerWithCachedStates":
             self.is_augmented,
         )
 
-    def copy(self, sender_id: Text = "") -> "TrackerWithCachedStates":
+    def copy(
+        self, sender_id: Text = "", sender_source: Text = ""
+    ) -> "TrackerWithCachedStates":
         """Creates a duplicate of this tracker.
 
         A new tracker will be created and all events
@@ -92,6 +94,7 @@ def copy(self, sender_id: Text = "") -> "TrackerWithCachedStates":
 
         tracker = self.init_copy()
         tracker.sender_id = sender_id
+        tracker.sender_source = sender_source
 
         for event in self.events:
             tracker.update(event, skip_states=True)
@@ -521,12 +524,12 @@ def _process_step(
                 # contribute to the trackers events
                 if tracker.sender_id:
                     if step.block_name not in tracker.sender_id.split(" > "):
-                        new_sender = f"{tracker.sender_id} > {step.block_name} > {step.source_name}"
+                        new_sender = tracker.sender_id + " > " + step.block_name
                     else:
                         new_sender = tracker.sender_id
                 else:
-                    new_sender = f"{step.block_name} > {step.source_name}"
-                trackers.append(tracker.copy(new_sender))
+                    new_sender = step.block_name
+                trackers.append(tracker.copy(new_sender, step.source_name))
 
         end_trackers = []
         for event in events:

tests/core/test_evaluation.py

@@ -1,6 +1,7 @@
 import os
 from pathlib import Path
 
+import rasa.utils.io
 from rasa.core.test import _generate_trackers, collect_story_predictions, test
 
 # we need this import to ignore the warning...
@@ -91,3 +92,22 @@ async def test_end_to_evaluation_with_forms(form_bot_agent: Agent):
     )
 
     assert not story_evaluation.evaluation_store.has_prediction_target_mismatch()
+
+
+async def test_source_in_failed_stories(tmpdir: Path, default_agent: Agent):
+    stories_path = str(tmpdir / "failed_stories.md")
+
+    await test(
+        stories=E2E_STORY_FILE_UNKNOWN_ENTITY,
+        agent=default_agent,
+        out_directory=str(tmpdir),
+        max_stories=None,
+        e2e=False,
+    )
+
+    failed_stories = rasa.utils.io.read_file(stories_path)
+
+    assert (
+        "## simple_story_with_unknown_entity (data/test_evaluations/story_unknown_entity.md)"
+        in failed_stories
+    )