g:clever_f_mark_direct extended beyond ASCII (rhysd#74)

* g:clever_f_mark_direct extended beyond ASCII

`ch`, instead of just containing a byte (that is ignored when non-ASCII, e.g., coding utf8 or other encoding), now contains a possibly multibyte char.
`line` is changed to contain the list of these groupped non-ASCII chars.
Tested on utf8.

* Update autoload/clever_f.vim thanking rhysd

Co-authored-by: Linda_pp <[email protected]>

* Update autoload/clever_f.vim thanking rhysd

Co-authored-by: Linda_pp <[email protected]>

* Update autoload/clever_f.vim thanks rhysd

Co-authored-by: Linda_pp <[email protected]>

* patching multibyte test for clever_f#mark_direct

* updating description of the test

* don't return last char of multibyte under cursor 

CI spotted that bug. I did not yet run CI on my computer (I test on vim in termux app of google play).

* Fixed test, added backwards test

I don't know why getmatches() is 1-based and not 0-based

* all test pass with c-1 in line 161.

# tests 81
# passes 81

* skip start of current char getline('.')[c-1] for F

After `let [_, l, c, _] = getpos('.')`, first byte of char under cursor is `getline('.')[c-1]`. Hence don't include, and don't skip it, when going backwards.
Also, mark_direct has always been slow on very long lines. I commented about optimization, perfect for `f`, and imperfect for `F`.

* updated comments, please migrate them to an issue

I made better comments. After the pull request passed, I will migrate these comments to an issue (I don't know how to add a new option)

* range should stay positive in the future issue

* Update clever_f.vim

* Update clever_f.vim

Co-authored-by: Linda_pp <[email protected]>

autoload/clever_f.vim

@@ -156,11 +156,18 @@ function! clever_f#_mark_direct(forward, count) abort
 
     let char_count = {}
     let matches = []
-    let indices = a:forward ? range(c, len(line) - 1, 1) : range(c - 2, 0, -1)
-    for i in indices
-        let ch = line[i]
-        " only matches to ASCII
-        if ch !~# '^[\x00-\x7F]$' | continue | endif
+    if a:forward 
+        let line = split(line[c - 1 : ], '\zs') " adding a limit like c+4*termcol*termlines just after `:` is a free gift in a very long line, assuming all char are less than 4 bytes.
+        let i = c - 1 + len(line[0])
+        let line = line[1:] " skip char under cursor
+    else
+        let line = reverse(split(line[0 : c - 2], '\zs')) " split() is slow on very long lines, new option to use `max(0,c-4*termcol*termlines-1000)` instead of `0` would help on a very long line, with the drawback of seeking at an arbitrarily byte possibly in the middle of a multibyte char (which would be off screen)
+        let i = c - 1
+    endif
+    for ch in line
+        if !a:forward 
+            let i -= len(ch)
+        endif
         let ch_lower = tolower(ch)
 
         let char_count[ch] = get(char_count, ch, 0) + 1
@@ -176,6 +183,9 @@ function! clever_f#_mark_direct(forward, count) abort
             let m = matchaddpos('CleverFDirect', [[l, i + 1]])
             call add(matches, m)
         endif
+        if a:forward 
+            let i += len(ch)
+        endif
     endfor
     return matches
 endfunction

test/test.vimspec

@@ -1278,12 +1278,21 @@ Describe clever_f#_mark_direct()
         Assert Equals(GetHighlightedPositions(), s)
     End
 
-    It should not highlight multibyte characters
+    It should highlight multibyte characters to which the cursor can be moved directly
         normal! 2gg0
-        " ＃ムかわいいよzビムx
-        "               _    _
+        "        ＃ムかわいいよzビムx
+        let s = 'ビ＃＃＃＃い＃_＃ム_'
         call clever_f#_mark_direct(1, 1)
-        let cols = [22, 29]
+        let cols = filter(range(1,len(s)),'s[v:val-1]=="_"||s[v:val-1]=="＃"[0]')
+        Assert Equals(sort(map(getmatches(), 'v:val.pos1[1]'), 'n'), cols)
+    End
+
+    It should highlight multibyte characters to which the cursor can be moved backwards
+        normal! 2gg$h
+        "        ビムかわいいよzビ＃x
+        let s = 'ビ＃＃＃い＃＃_＃ムx'
+        call clever_f#_mark_direct(0, 1)
+        let cols = filter(range(1,len(s)),'s[v:val-1]=="_"||s[v:val-1]=="＃"[0]')
         Assert Equals(sort(map(getmatches(), 'v:val.pos1[1]'), 'n'), cols)
     End