Added unit test to test Token comparisons

Author: hsm207

rasa/nlu/tokenizers/__init__.py

@@ -21,10 +21,10 @@ def get(self, prop, default=None):
 
     def __eq__(self, other):
         if not isinstance(other, Token):
-            return NotImplemented
+            raise NotImplementedError
         return self.text == other.text
 
     def __lt__(self, other):
         if not isinstance(other, Token):
-            return NotImplemented
+            raise NotImplementedError
         return self.text < other.text

tests/nlu/base/test_tokenizers.py

@@ -1,7 +1,26 @@
 from unittest.mock import patch
+
+import pytest
+
+from rasa.nlu import training_data
 from rasa.nlu.training_data import TrainingData, Message
 from tests.nlu import utilities
-from rasa.nlu import training_data
+
+
+def test_tokens_comparison():
+    from rasa.nlu.tokenizers import Token
+
+    x = Token("hello", 0)
+    y = Token("Hello", 0)
+
+    assert x == x
+    assert y < x
+
+    with pytest.raises(NotImplementedError):
+        assert x == 1
+
+    with pytest.raises(NotImplementedError):
+        assert y < "a"
 
 
 def test_whitespace():