Merge branch 'rasa-info-command' of github.com:RasaHQ/rasa into rasa-info-command

Author: koaning

.github/workflows/vulnerability-scan.yml

@@ -2,8 +2,8 @@ name: Vulnerability Scan
 
 on:
   schedule:
-    # Run once every day
-    - cron: '0 0 * * *'
+    # Run every third day
+    - cron: 0 0 * * */3
 
 jobs:
   scan:
@@ -35,10 +35,11 @@ jobs:
         echo "::set-env name=IMAGE_WITH_POETRY_LOCK::$IMAGE_NAME"
 
     - name: Scan image 🕵️‍♀️🕵️‍♂️
-      uses: homoluctus/gitrivy@v2.0.0
+      uses: wochinge/gitrivy@6bf026b
       with:
         # Needs the token so it can create an issue once a vulnerability was found
         token: ${{ secrets.GITHUB_TOKEN }}
         image: ${{ env.IMAGE_WITH_POETRY_LOCK }}
         ignore_unfixed: true
         issue_label: "tool:trivy,type:vulnerability"
+        fail_on_vulnerabilities: true

changelog/5737.bugfix.rst

@@ -0,0 +1,3 @@
+Pinned the library version for the Azure :ref:`cloud-storage` to ``2.1.0`` since the
+persistor is currently not compatible with later versions of the ``azure-storage-blob``
+library.

changelog/5756.improvement.rst

@@ -0,0 +1,8 @@
+To avoid the problem of our entity extractors predicting entity labels for just a part of the words,
+we introduced a cleaning method after the prediction was done.
+We should avoid the incorrect prediction in the first place.
+To achieve this we will not tokenize words into sub-words anymore.
+We take the mean feature vectors of the sub-words as the feature vector of the word.
+
+.. warning::
+    This change is model breaking. Please, retrain your models.

poetry.lock

@@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytype | .pytest_cache | build | dist))"
 
 [tool.poetry]
 name = "rasa"
-version = "1.10.0"
+version = "1.11.0a1"
 description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants"
 authors = [ "Rasa Technologies GmbH <[email protected]>",]
 maintainers = [ "Tom Bocklisch <[email protected]>",]
@@ -91,7 +91,7 @@ rocketchat_API = ">=0.6.31,<1.4.0"
 colorhash = "~1.0.2"
 pika = "~1.1.0"
 jsonschema = "~3.2"
-packaging = "~19.0"
+packaging = ">=19.0,<20.4"
 gevent = ">=1.4,<1.6"
 pytz = "^2019.1"
 rasa-sdk = "^1.10.0"
@@ -142,7 +142,7 @@ black = "^19.10b0"
 flake8 = "^3.7.9"
 pytype = "^2020.1.24"
 google-cloud-storage = "^1.25.0"
-azure-storage-blob = "^12.1.0"
+azure-storage-blob = "<=2.1.0"
 coveralls = "^2.0.0"
 towncrier = "^19.2.0"
 toml = "^0.10.0"

pyproject.toml

@@ -53,7 +53,7 @@
 CONFIG_MANDATORY_KEYS_NLU = ["language", "pipeline"]
 CONFIG_MANDATORY_KEYS = CONFIG_MANDATORY_KEYS_CORE + CONFIG_MANDATORY_KEYS_NLU
 
-MINIMUM_COMPATIBLE_VERSION = "1.10.0"
+MINIMUM_COMPATIBLE_VERSION = "1.11.0a1"
 
 GLOBAL_USER_CONFIG_PATH = os.path.expanduser("~/.config/rasa/global.yml")
 

rasa/constants.py

@@ -803,7 +803,6 @@ def _predict_entities(
         )
 
         entities = self.add_extractor_name(entities)
-        entities = self.clean_up_entities(message, entities)
         entities = message.get(ENTITIES, []) + entities
 
         return entities

rasa/nlu/classifiers/diet_classifier.py

@@ -37,6 +37,7 @@
 
 CLS_TOKEN = "__CLS__"
 POSITION_OF_CLS_TOKEN = -1
+NUMBER_OF_SUB_TOKENS = "number_of_sub_tokens"
 
 MESSAGE_ATTRIBUTES = [TEXT, INTENT, RESPONSE]
 

rasa/nlu/constants.py

@@ -195,7 +195,6 @@ def _update_crf_order(self, training_data: TrainingData):
     def process(self, message: Message, **kwargs: Any) -> None:
         entities = self.extract_entities(message)
         entities = self.add_extractor_name(entities)
-        entities = self.clean_up_entities(message, entities)
         message.set(ENTITIES, message.get(ENTITIES, []) + entities, add_to_output=True)
 
     def extract_entities(self, message: Message) -> List[Dict[Text, Any]]:

rasa/nlu/extractors/crf_entity_extractor.py

@@ -186,7 +186,6 @@ def process(self, message: Message, **kwargs: Any) -> None:
             )
 
         extracted = self.add_extractor_name(extracted)
-        extracted = self.clean_up_entities(message, extracted)
         message.set(ENTITIES, message.get(ENTITIES, []) + extracted, add_to_output=True)
 
     @classmethod