feat: basic non ascii support (robertkrimen#423)

Adds basic non ascii support including substring, substr, slice functions,
which work with runes (prevent utf-8 length).

This includes adding runes compatible functions:
* indexOf
* lastIndexOf

.gitignore

@@ -3,3 +3,4 @@
 /otto/otto-*
 /test/test-*.js
 /test/tester
+.idea

builtin_string.go

@@ -70,12 +70,26 @@ func builtinString_concat(call FunctionCall) Value {
 	return toValue_string(value.String())
 }
 
+func lastIndexRune(s, substr string) int {
+	if i := strings.LastIndex(s, substr); i >= 0 {
+		return utf8.RuneCountInString(s[:i])
+	}
+	return -1
+}
+
+func indexRune(s, substr string) int {
+	if i := strings.Index(s, substr); i >= 0 {
+		return utf8.RuneCountInString(s[:i])
+	}
+	return -1
+}
+
 func builtinString_indexOf(call FunctionCall) Value {
 	checkObjectCoercible(call.runtime, call.This)
 	value := call.This.string()
 	target := call.Argument(0).string()
 	if 2 > len(call.ArgumentList) {
-		return toValue_int(strings.Index(value, target))
+		return toValue_int(indexRune(value, target))
 	}
 	start := toIntegerFloat(call.Argument(1))
 	if 0 > start {
@@ -86,7 +100,7 @@ func builtinString_indexOf(call FunctionCall) Value {
 		}
 		return toValue_int(-1)
 	}
-	index := strings.Index(value[int(start):], target)
+	index := indexRune(value[int(start):], target)
 	if index >= 0 {
 		index += int(start)
 	}
@@ -98,16 +112,16 @@ func builtinString_lastIndexOf(call FunctionCall) Value {
 	value := call.This.string()
 	target := call.Argument(0).string()
 	if 2 > len(call.ArgumentList) || call.ArgumentList[1].IsUndefined() {
-		return toValue_int(strings.LastIndex(value, target))
+		return toValue_int(lastIndexRune(value, target))
 	}
 	length := len(value)
 	if length == 0 {
-		return toValue_int(strings.LastIndex(value, target))
+		return toValue_int(lastIndexRune(value, target))
 	}
 	start := call.ArgumentList[1].number()
 	if start.kind == numberInfinity { // FIXME
 		// startNumber is infinity, so start is the end of string (start = length)
-		return toValue_int(strings.LastIndex(value, target))
+		return toValue_int(lastIndexRune(value, target))
 	}
 	if 0 > start.int64 {
 		start.int64 = 0
@@ -116,7 +130,7 @@ func builtinString_lastIndexOf(call FunctionCall) Value {
 	if end > length {
 		end = length
 	}
-	return toValue_int(strings.LastIndex(value[:end], target))
+	return toValue_int(lastIndexRune(value[:end], target))
 }
 
 func builtinString_match(call FunctionCall) Value {
@@ -389,23 +403,23 @@ func builtinString_slice(call FunctionCall) Value {
 	if end-start <= 0 {
 		return toValue_string("")
 	}
-	return toValue_string(target[start:end])
+	return toValue_string(string(target[start:end]))
 }
 
 func builtinString_substring(call FunctionCall) Value {
 	checkObjectCoercible(call.runtime, call.This)
-	target := call.This.string()
+	target := []rune(call.This.string())
 
 	length := int64(len(target))
 	start, end := rangeStartEnd(call.ArgumentList, length, true)
 	if start > end {
 		start, end = end, start
 	}
-	return toValue_string(target[start:end])
+	return toValue_string(string(target[start:end]))
 }
 
 func builtinString_substr(call FunctionCall) Value {
-	target := call.This.string()
+	target := []rune(call.This.string())
 
 	size := int64(len(target))
 	start, length := rangeStartLength(call.ArgumentList, size)
@@ -426,7 +440,7 @@ func builtinString_substr(call FunctionCall) Value {
 		length = size - start
 	}
 
-	return toValue_string(target[start : start+length])
+	return toValue_string(string(target[start : start+length]))
 }
 
 func builtinString_toLowerCase(call FunctionCall) Value {

builtin_test.go

@@ -8,6 +8,16 @@ func TestString_substr(t *testing.T) {
 	tt(t, func() {
 		test, _ := test()
 
+		test(`
+            [
+                "uñiçode".substr(0,1), // "u"
+                "uñiçode".substr(0,2), // "uñ"
+                "uñiçode".substr(0,3), // "uñi"
+                "uñiçode".substr(0,4), // "uñiç"
+                "uñiçode".substr(0,9), // "uñiçode"
+            ];
+        `, "u,uñ,uñi,uñiç,uñiçode")
+
 		test(`
             [
                 "abc".substr(0,1), // "a"

string_test.go

@@ -94,6 +94,12 @@ func TestString_indexOf(t *testing.T) {
 		test(`"abc".indexOf("a")`, 0)
 		test(`"abc".indexOf("bc")`, 1)
 		test(`"abc".indexOf("bc", 11)`, -1)
+
+		test(`"uñiçode".indexOf("ñ")`, 1)
+		test(`"uñiçode".indexOf("ñ", 11)`, -1)
+		test(`"uññiçode".indexOf("ç")`, 4)
+		test(`"uññiçode".indexOf("ç", 11)`, -1)
+
 		test(`"$$abcdabcd".indexOf("ab", function(){return -Infinity;}())`, 2)
 		test(`"$$abcdabcd".indexOf("ab", function(){return NaN;}())`, 2)
 
@@ -128,6 +134,11 @@ func TestString_lastIndexOf(t *testing.T) {
 		test(`"abc".lastIndexOf("abc", 2)`, 0)
 		test(`"abc".lastIndexOf("abc", 3)`, 0)
 
+		test(`"uñiçodeñ".lastIndexOf("ñ")`, 7)
+		test(`"uñiçode".lastIndexOf("ñ")`, 1)
+		test(`"uñiçodeñ".lastIndexOf("ç")`, 3)
+		test(`"uñiçodeñ".lastIndexOf("aç")`, -1)
+
 		test(`
             abc = new Object(true);
             abc.lastIndexOf = String.prototype.lastIndexOf;
@@ -299,6 +310,42 @@ func TestString_slice(t *testing.T) {
 	})
 }
 
+func TestString_length(t *testing.T) {
+	tt(t, func() {
+		test, _ := test()
+
+		test(`"abc".length`, 3)
+		test(`"uñiçode".length`, 7)
+	})
+}
+
+func TestString_slice_unicode(t *testing.T) {
+	tt(t, func() {
+		test, _ := test()
+
+		test(`"uñiçode".slice()`, "uñiçode")
+		test(`"uñiçode".slice(0)`, "uñiçode")
+		test(`"uñiçode".slice(0,11)`, "uñiçode")
+		test(`"uñiçode".slice(0,-1)`, "uñiçod")
+		test(`"uñiçode".slice(-1,11)`, "e")
+	})
+}
+
+func TestString_substring_unicode(t *testing.T) {
+	tt(t, func() {
+		test, _ := test()
+
+		test(`"uñiçode".substring()`, "uñiçode")
+		test(`"uñiçode".substring(0)`, "uñiçode")
+		test(`"uñiçode".substring(0,11)`, "uñiçode")
+		test(`"uñiçode".substring(11,0)`, "uñiçode")
+		test(`"uñiçode".substring(0,-1)`, "")
+		test(`"uñiçode".substring(-1,11)`, "uñiçode")
+		test(`"uñiçode".substring(1)`, "ñiçode")
+		test(`"uñiçode".substring(Infinity, Infinity)`, "")
+	})
+}
+
 func TestString_substring(t *testing.T) {
 	tt(t, func() {
 		test, _ := test()