Merge pull request RustPython#1977 from skinny121/isspace-unicode

Fix str.isspace to support unicode space characters

Author: coolreader18

Cargo.lock

@@ -204,8 +204,6 @@ def test_contains(self):
         self.assertIs(1 in {}, False)
         self.assertIs(1 in {1:1}, True)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_string(self):
         self.assertIs("xyz".endswith("z"), True)
         self.assertIs("xyz".endswith("x"), False)

Lib/test/test_bool.py

@@ -620,8 +620,6 @@ def test_istitle(self):
         for ch in ['\U00010429', '\U0001044E', '\U0001F40D', '\U0001F46F']:
             self.assertFalse(ch.istitle(), '{!a} is not title'.format(ch))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_isspace(self):
         super().test_isspace()
         self.checkequalnofix(True, '\u2000', 'isspace')

Lib/test/test_unicode.py

@@ -81,9 +81,9 @@ unicode_names2 = "0.4"
 # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939
 unicode-casing = "0.1"
 # update version all at the same time
-unic-bidi          = "0.9"
 unic-char-property = "0.9"
 unic-normal        = "0.9"
+unic-ucd-bidi      = "0.9"
 unic-ucd-category  = "0.9"
 unic-ucd-age       = "0.9"
 unic-ucd-ident     = "0.9"

vm/Cargo.toml

@@ -8,6 +8,7 @@ use std::string::ToString;
 use crossbeam_utils::atomic::AtomicCell;
 use itertools::Itertools;
 use num_traits::ToPrimitive;
+use unic_ucd_bidi::BidiClass;
 use unic_ucd_category::GeneralCategory;
 use unic_ucd_ident::{is_xid_continue, is_xid_start};
 use unicode_casing::CharExt;
@@ -751,11 +752,16 @@ impl PyString {
             .all(|c| c == '\u{0020}' || char_is_printable(c))
     }
 
-    // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty
-    // which is why isspace is using is_ascii_whitespace. Same for isupper & islower
     #[pymethod]
     fn isspace(&self) -> bool {
-        !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace())
+        if self.value.is_empty() {
+            return false;
+        }
+        use unic_ucd_bidi::bidi_class::abbr_names::*;
+        self.value.chars().all(|c| {
+            GeneralCategory::of(c) == GeneralCategory::SpaceSeparator
+                || matches!(BidiClass::of(c), WS | B | S)
+        })
     }
 
     // Return true if all cased characters in the string are lowercase and there is at least one cased character, false otherwise.

vm/src/obj/objstr.rs

@@ -9,10 +9,10 @@ use crate::pyobject::{PyClassImpl, PyObject, PyObjectRef, PyResult, PyValue};
 use crate::vm::VirtualMachine;
 
 use itertools::Itertools;
-use unic_bidi::BidiClass;
 use unic_char_property::EnumeratedCharProperty;
 use unic_normal::StrNormalForm;
 use unic_ucd_age::{Age, UnicodeVersion, UNICODE_VERSION};
+use unic_ucd_bidi::BidiClass;
 use unic_ucd_category::GeneralCategory;
 
 pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {