linvi#430 First fix but some tests are failing

Examplinvi.NETStandard/Program.cs

@@ -1,17 +1,40 @@
 using System;
 using Tweetinvi;
+using Tweetinvi.Models;
 
 namespace Examplinvi.NETStandard
 {
     class Program
     {
         static void Main(string[] args)
         {
-            Auth.SetUserCredentials("CONSUMER_KEY", "CONSUMER_SECRET", "ACCESS_TOKEN", "ACCESS_TOKEN_SECRET");
+            TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Compat;
+            Auth.SetUserCredentials("54aUlLwbmRN7SX3wswMJmqpQG", "Fid46uknw6665bA4NxCiOHPoIUv6KMsaxhul7P4MwHMecIytYw", "1693649419-Ubxt4bKWWGQiRY9Ko4BcvX03EJUm2BPcRbW6pPM", "wLa7UFyp4FEDR2MHPtr6SEy4E3iCBqqlNAXuampl2SXZ7");
 
-            var authenticatedUser = User.GetAuthenticatedUser();
+            var tweet = Tweet.GetTweet(914635724758163456);
+            var suffix = tweet.Suffix;
 
-            Console.WriteLine(authenticatedUser);
+            var l = Tweet.Length(tweet.Text);
+
+            //var authenticatedUser = User.GetAuthenticatedUser();
+
+            var fs = Stream.CreateFilteredStream();
+
+            fs.AddTrack("italia");
+            fs.AddTrack("trump");
+            fs.AddTrack("merkel");
+
+            fs.MatchingTweetReceived += (sender, eventArgs) =>
+            {
+                if (eventArgs.Tweet.Language != Language.Undefined)
+                {
+                    Console.WriteLine(eventArgs.Json);
+                }
+            };
+
+            fs.StartStreamMatchingAllConditions();
+
+            //Console.WriteLine(authenticatedUser);
             Console.ReadLine();
         }
     }

Testinvi/Tweetinvi.Core/StringExtensionsTests.cs

@@ -3,6 +3,7 @@
 using System.Linq;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Tweetinvi;
+using Tweetinvi.Core.Core.Helpers;
 using Tweetinvi.Core.Extensions;
 using Tweetinvi.Parameters;
 
@@ -242,7 +243,12 @@ public void IsMatchingJsonFormat()
                 Assert.IsFalse("hello".IsMatchingJsonFormat());
             }
 
-
+            [TestMethod]
+            public void UTFNew()
+            {
+                var l = Tweet.Length("sa 🎅⛄️🎅 done");
+                Assert.AreEqual(l, 12);
+            }
 
             [TestMethod]
             public void MyTest()

Tweetinvi.Core/Core/Extensions/StringExtension.cs

@@ -2,6 +2,7 @@
 using System.Globalization;
 using System.Text;
 using System.Text.RegularExpressions;
+using Tweetinvi.Core.Core.Helpers;
 using Tweetinvi.Core.Helpers;
 using Tweetinvi.Models;
 
@@ -111,32 +112,6 @@ public static int TweetLength(this string tweet, bool willBePublishedWithMedia =
             return length;
         }
 
-        private const char HIGH_SURROGATE_START = '\uD800';
-        private const char HIGH_SURROGATE_END = '\uDBFF';
-        private const char LOW_SURROGATE_START = '\uDC00';
-        private const char LOW_SURROGATE_END = '\uDFFF';
-
-        /// <summary>
-        /// Get the UTF32 length of a string
-        /// </summary>
-        public static int UTF32Length(this string s)
-        {
-            var length = 0;
-
-            for (int i = 0; i < s.Length; ++i)
-            {
-                if (s[i] >= HIGH_SURROGATE_START && s[i] <= HIGH_SURROGATE_END &&
-                    s[i + 1] >= LOW_SURROGATE_START && s[i + 1] <= LOW_SURROGATE_END)
-                {
-                    i++;
-                }
-
-                ++length;
-            }
-
-            return length;
-        }
-
         /// <summary>
         /// Clean a string so that it can be used in a URL and
         /// sent to Twitter

Tweetinvi.Core/Core/Helpers/UnicodeHelper.cs

@@ -0,0 +1,80 @@
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using Tweetinvi.Core.Extensions;
+
+namespace Tweetinvi.Core.Core.Helpers
+{
+    public static class UnicodeHelper
+    {
+        public static IEnumerable<string> GetUnicodeCleanupGraphemeEnumerator(string str)
+        {
+            var enumerator = StringInfo.GetTextElementEnumerator(str);
+
+            while (enumerator.MoveNext())
+            {
+                var grapheme = enumerator.GetTextElement();
+
+
+                UnicodeCategory characterChategory = CharUnicodeInfo.GetUnicodeCategory(grapheme, 0);
+
+                // Other potential categories to consider 
+                // cat == UnicodeCategory.NonSpacingMark || cat == UnicodeCategory.SpacingCombiningMark || cat == UnicodeCategory.EnclosingMark
+
+                if (characterChategory == UnicodeCategory.ModifierSymbol)
+                {
+                    continue;
+                }
+
+                yield return grapheme;
+            }
+        }
+
+        public static string UnicodeCleanup(string str)
+        {
+            var cleanStringBuilder = new StringBuilder();
+
+            UnicodeHelper.GetUnicodeCleanupGraphemeEnumerator(str).ForEach(c => cleanStringBuilder.Append(c));
+
+            return cleanStringBuilder.ToString();
+        }
+
+        public static string UnicodeSubstring(string str, int startIndex, int length)
+        {
+            if (str == null)
+            {
+                return null;
+            }
+
+            var graphemes = GetUnicodeCleanupGraphemeEnumerator(str).Skip(startIndex).Take(length);
+            return string.Join("", graphemes);
+        }
+
+        private const char HIGH_SURROGATE_START = '\uD800';
+        private const char HIGH_SURROGATE_END = '\uDBFF';
+        private const char LOW_SURROGATE_START = '\uDC00';
+        private const char LOW_SURROGATE_END = '\uDFFF';
+
+        /// <summary>
+        /// Get the UTF32 length of a string
+        /// </summary>
+        public static int UTF32Length(this string str)
+        {
+            var length = 0;
+
+            for (int i = 0; i < str.Length; ++i)
+            {
+                if (str[i] >= HIGH_SURROGATE_START && str[i] <= HIGH_SURROGATE_END &&
+                    str[i + 1] >= LOW_SURROGATE_START && str[i + 1] <= LOW_SURROGATE_END)
+                {
+                    i++;
+                }
+
+                ++length;
+            }
+
+            return length;
+        }
+    }
+}

Tweetinvi.Logic/DTO/ExtendedTweet.cs

@@ -8,9 +8,11 @@ namespace Tweetinvi.Logic.DTO
     public class ExtendedTweet : IExtendedTweet
     {
         [JsonProperty("text")]
+        [JsonConverter(typeof(StringToCleanupUnicodeConverter))]
         public string Text { get; set; }
 
         [JsonProperty("full_text")]
+        [JsonConverter(typeof(StringToCleanupUnicodeConverter))]
         public string FullText { get; set; }
 
         [JsonProperty("display_text_range")]

Tweetinvi.Logic/DTO/TweetDTO.cs

@@ -42,9 +42,11 @@ public long Id
         public bool IsTweetDestroyed { get; set; }
 
         [JsonProperty("text")]
+        [JsonConverter(typeof(StringToCleanupUnicodeConverter))]
         public string Text { get; set; }
 
         [JsonProperty("full_text")]
+        [JsonConverter(typeof(StringToCleanupUnicodeConverter))]
         public string FullText { get; set; }
 
         [JsonProperty("display_text_range")]

Tweetinvi.Logic/JsonConverters/StringToCleanupUnicodeConverter.cs

@@ -0,0 +1,28 @@
+using System;
+using Newtonsoft.Json;
+using Tweetinvi.Core.Core.Helpers;
+
+namespace Tweetinvi.Logic.JsonConverters
+{
+    public class StringToCleanupUnicodeConverter : JsonConverter
+    {
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            serializer.Serialize(writer, value);
+
+        }
+
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            var altText = reader.Value as string ?? "";
+            var cleanString = UnicodeHelper.UnicodeCleanup(altText);
+
+            return cleanString;
+        }
+
+        public override bool CanConvert(Type objectType)
+        {
+            return true;
+        }
+    }
+}