Move bar aggregators to utils, add time overloads & add check of orde…

…rType for Kraken (QuantConnect#5969)

* Move Aggregate Quote and Trade Bars to utils

* Add decimal and long overload to UnixTimeStampToDateTime

* Add Kraken OrderType check in CanSubmitOrder

* PR !5969 review fixes

Common/Brokerages/KrakenBrokerageModel.cs

@@ -96,6 +96,16 @@ public override bool CanSubmitOrder(Security security, Order order, out Brokerag
 
                 return false;
             }
+
+            if (order.Type == OrderType.MarketOnClose || order.Type == OrderType.MarketOnOpen || order.Type == OrderType.OptionExercise)
+            {
+                message = new BrokerageMessageEvent(BrokerageMessageType.Warning, "NotSupported",
+                    StringExtensions.Invariant($"{order.Type} orders are not supported by Kraken.")
+                );
+
+                return false;
+            }
+
             return base.CanSubmitOrder(security, order, out message);
         }
 

Common/Time.cs

@@ -132,7 +132,8 @@ public DateTime LocalTime
         }
 
         private static readonly DateTime EpochTime = new DateTime(1970, 1, 1, 0, 0, 0, 0);
-
+        private const long SecondToMillisecond = 1000;
+
         /// <summary>
         /// Create a C# DateTime from a UnixTimestamp
         /// </summary>
@@ -153,12 +154,32 @@ public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
             }
             return time;
         }
+
+        /// <summary>
+        /// Create a C# DateTime from a UnixTimestamp
+        /// </summary>
+        /// <param name="unixTimeStamp">Decimal unix timestamp (Time since Midnight Jan 1 1970)</param>
+        /// <returns>C# date time object</returns>
+        public static DateTime UnixTimeStampToDateTime(decimal unixTimeStamp)
+        {
+            return UnixMillisecondTimeStampToDateTime(unixTimeStamp * SecondToMillisecond);
+        }
+
+        /// <summary>
+        /// Create a C# DateTime from a UnixTimestamp
+        /// </summary>
+        /// <param name="unixTimeStamp">Long unix timestamp (Time since Midnight Jan 1 1970)</param>
+        /// <returns>C# date time object</returns>
+        public static DateTime UnixTimeStampToDateTime(long unixTimeStamp)
+        {
+            return UnixTimeStampToDateTime(Convert.ToDecimal(unixTimeStamp));
+        }
 
         /// <summary>
         /// Create a C# DateTime from a UnixTimestamp
         /// </summary>
-        /// <param name="unixTimeStamp">Double unix timestamp (Time since Midnight Jan 1 1970) in milliseconds</param>
-        /// <returns>C# date timeobject</returns>
+        /// <param name="unixTimeStamp">Decimal unix timestamp (Time since Midnight Jan 1 1970) in milliseconds</param>
+        /// <returns>C# date time object</returns>
         public static DateTime UnixMillisecondTimeStampToDateTime(decimal unixTimeStamp)
         {
             DateTime time;
@@ -182,7 +203,7 @@ public static DateTime UnixMillisecondTimeStampToDateTime(decimal unixTimeStamp)
         /// Create a C# DateTime from a UnixTimestamp
         /// </summary>
         /// <param name="unixTimeStamp">Int64 unix timestamp (Time since Midnight Jan 1 1970) in nanoseconds</param>
-        /// <returns>C# date timeobject</returns>
+        /// <returns>C# date time object</returns>
         public static DateTime UnixNanosecondTimeStampToDateTime(long unixTimeStamp)
         {
             DateTime time;

Common/Util/LeanData.cs

@@ -1118,5 +1118,103 @@ private static List<string> SplitDataPath(string fileName)
             // split path into components
             return fileName.Split(pathSeparators, StringSplitOptions.RemoveEmptyEntries).ToList();
         }
+
+        /// <summary>
+        /// Aggregates a list of second/minute bars at the requested resolution
+        /// </summary>
+        /// <param name="bars">List of <see cref="TradeBar"/>s</param>
+        /// <param name="symbol">Symbol of all tradeBars</param>
+        /// <param name="resolution">Desired resolution for new <see cref="TradeBar"/>s</param>
+        /// <returns>List of aggregated <see cref="TradeBar"/>s</returns>
+        public static IEnumerable<TradeBar> AggregateTradeBars(IEnumerable<TradeBar> bars, Symbol symbol, TimeSpan resolution)
+        {
+            return
+                from b in bars
+                group b by b.Time.RoundDown(resolution)
+                into g
+                select new TradeBar
+                {
+                    Symbol = symbol,
+                    Time = g.Key,
+                    Open = g.First().Open,
+                    High = g.Max(b => b.High),
+                    Low = g.Min(b => b.Low),
+                    Close = g.Last().Close,
+                    Value = g.Last().Close,
+                    DataType = MarketDataType.TradeBar,
+                    Period = resolution
+                };
+        }
+
+        /// <summary>
+        /// Aggregates a list of second/minute bars at the requested resolution
+        /// </summary>
+        /// <param name="bars">List of <see cref="QuoteBar"/>s</param>
+        /// <param name="symbol">Symbol of all QuoteBars</param>
+        /// <param name="resolution">Desired resolution for new <see cref="QuoteBar"/>s</param>
+        /// <returns>List of aggregated <see cref="QuoteBar"/>s</returns>
+        public static IEnumerable<QuoteBar> AggregateQuoteBars(IEnumerable<QuoteBar> bars, Symbol symbol, TimeSpan resolution)
+        {
+            return
+                from b in bars
+                    group b by b.Time.RoundDown(resolution)
+                    into g
+                    select new QuoteBar
+                    {
+                        Symbol = symbol,
+                        Time = g.Key,
+                        Bid = new Bar
+                        {
+                            Open = g.First().Bid.Open,
+                            High = g.Max(b => b.Bid.High),
+                            Low = g.Min(b => b.Bid.Low),
+                            Close = g.Last().Bid.Close
+                        },
+                        Ask = new Bar
+                        {
+                            Open = g.First().Ask.Open,
+                            High = g.Max(b => b.Ask.High),
+                            Low = g.Min(b => b.Ask.Low),
+                            Close = g.Last().Ask.Close
+                        },
+                        Period = resolution
+                    };
+        }
+
+         /// <summary>
+         /// Aggregates a list of ticks at the requested resolution
+         /// </summary>
+         /// <param name="ticks">List of <see cref="QuoteBar"/>s</param>
+         /// <param name="symbol">Symbol of all QuoteBars</param>
+         /// <param name="resolution">Desired resolution for new <see cref="QuoteBar"/>s</param>
+         /// <returns>List of aggregated <see cref="QuoteBar"/>s</returns>
+         public static IEnumerable<QuoteBar> AggregateTicks(IEnumerable<Tick> ticks, Symbol symbol, TimeSpan resolution)
+         {
+             return
+                from t in ticks
+                    group t by t.Time.RoundDown(resolution)
+                    into g
+                    select new QuoteBar
+                    {
+                        Symbol = symbol,
+                        Time = g.Key,
+                        Bid = new Bar
+                        {
+                            Open = g.First().BidPrice,
+                            High = g.Max(b => b.BidPrice),
+                            Low = g.Min(b => b.BidPrice),
+                            Close = g.Last().BidPrice
+                        },
+                        Ask = new Bar
+                        {
+                            Open = g.First().AskPrice,
+                            High = g.Max(b => b.AskPrice),
+                            Low = g.Min(b => b.AskPrice),
+                            Close = g.Last().AskPrice
+                        },
+                        Period = resolution
+                    };
+        }
+
     }
 }

Tests/Common/TimeTests.cs

@@ -43,6 +43,25 @@ public void UnixTimeStampMillisecondsToDateTimeHasSubMillisecondPrecision()
             var time = Time.UnixMillisecondTimeStampToDateTime(stamp);
             Assert.AreEqual(expected, time);
         }
+
+
+        [Test]
+        public void UnixTimeStampSecondsToDateTimeSubMillisecondPrecision()
+        {
+            const decimal stamp = 1520711961.00055m;
+            var expected = new DateTime(2018, 3, 10, 19, 59, 21, 0).AddTicks(5500);
+            var time = Time.UnixTimeStampToDateTime(stamp);
+            Assert.AreEqual(expected, time);
+        }
+
+        [Test]
+        public void UnixTimeStampSecondsToDateTime()
+        {
+            const long stamp = 1520711961000;
+            var expected = new DateTime(2018, 3, 10, 19, 59, 21, 0);
+            var time = Time.UnixMillisecondTimeStampToDateTime(stamp);
+            Assert.AreEqual(expected, time);
+        }
 
         [Test]
         public void GetStartTimeForTradeBarsRoundsDown()

Tests/Common/Util/LeanDataTests.cs

@@ -444,6 +444,75 @@ public void FutureOptionSingleZipFileContainingMultipleFuturesOptionsContracts(O
             Assert.AreEqual("../../../Data/futureoption/comex/minute/og/20200428/20200105_quote_american.zip", optionZipFilePath);
             Assert.AreEqual($"20200105_og_minute_quote_american_{right.ToLower()}_{strike}0000_{expiry:yyyyMMdd}.csv", optionEntryFilePath);
         }
+
+        [Test, TestCaseSource(nameof(AggregateTradeBarsTestData))]
+        public void AggregateTradeBarsTest(TimeSpan resolution, TradeBar expectedFirstTradeBar)
+        {
+            var initialTime = new DateTime(2020, 1, 5, 12, 0, 0);
+            var symbol = Symbols.AAPL;
+            var initialBars = new[]
+            {
+                new TradeBar {Time = initialTime, Open = 10, High = 15, Low = 8, Close = 11, Volume = 50, Period = TimeSpan.FromSeconds(1), Symbol = symbol},
+                new TradeBar {Time = initialTime.Add(TimeSpan.FromSeconds(15)), Open = 13, High = 14, Low = 7, Close = 9, Volume = 150, Period = TimeSpan.FromSeconds(1), Symbol = symbol},
+                new TradeBar {Time = initialTime.Add(TimeSpan.FromMinutes(15)), Open = 11, High = 25, Low = 10, Close = 21, Volume = 90, Period = TimeSpan.FromMinutes(1), Symbol = symbol},
+                new TradeBar {Time = initialTime.Add(TimeSpan.FromHours(6)), Open = 17, High = 19, Low = 12, Close = 11, Volume = 20, Period = TimeSpan.FromMinutes(1), Symbol = symbol},
+            };
+
+            var aggregated = LeanData.AggregateTradeBars(initialBars, symbol, resolution).ToList();
+
+            Assert.True(aggregated.All(i => i.Period == resolution));
+
+            var firstBar = aggregated.First();
+
+            AssertBarsAreEqual(expectedFirstTradeBar, firstBar);
+
+        }
+
+        [Test, TestCaseSource(nameof(AggregateQuoteBarsTestData))]
+        public void AggregateQuoteBarsTest(TimeSpan resolution, QuoteBar expectedFirstBar)
+        {
+            var initialTime = new DateTime(2020, 1, 5, 12, 0, 0);
+            var symbol = Symbols.AAPL;
+            var initialBars = new[]
+            {
+                new QuoteBar {Time = initialTime, Ask = new Bar {Open = 10, High = 15, Low = 8, Close = 11}, Bid = {Open = 7, High = 14, Low = 5, Close = 10}, Period = TimeSpan.FromMinutes(1), Symbol = symbol},
+                new QuoteBar {Time = initialTime.Add(TimeSpan.FromSeconds(15)), Ask = new Bar {Open = 13, High = 14, Low = 7, Close = 9}, Bid = {Open = 10, High = 11, Low = 4, Close = 5}, Period = TimeSpan.FromMinutes(1), Symbol = symbol},
+                new QuoteBar {Time = initialTime.Add(TimeSpan.FromMinutes(15)), Ask = new Bar {Open = 11, High = 25, Low = 10, Close = 21}, Bid = {Open = 10, High = 22, Low = 9, Close = 20}, Period = TimeSpan.FromMinutes(1), Symbol = symbol},
+                new QuoteBar {Time = initialTime.Add(TimeSpan.FromHours(6)), Ask = new Bar {Open = 17, High = 19, Low = 12, Close = 11}, Bid = {Open = 16, High = 17, Low = 10, Close = 10}, Period = TimeSpan.FromMinutes(1), Symbol = symbol},
+            };
+
+            var aggregated = LeanData.AggregateQuoteBars(initialBars, symbol, resolution).ToList();
+
+            Assert.True(aggregated.All(i => i.Period == resolution));
+
+            var firstBar = aggregated.First();
+
+            AssertBarsAreEqual(expectedFirstBar.Ask, firstBar.Ask);
+            AssertBarsAreEqual(expectedFirstBar.Bid, firstBar.Bid);
+        }
+
+        [Test, TestCaseSource(nameof(AggregateTickTestData))]
+        public void AggregateTicksTest(TimeSpan resolution, QuoteBar expectedFirstBar)
+        {
+            var initialTime = new DateTime(2020, 1, 5, 12, 0, 0);
+            var symbol = Symbols.AAPL;
+            var initialTicks = new[]
+            {
+                new Tick(initialTime, symbol, string.Empty, string.Empty, 10, 11, 12, 13),
+                new Tick(initialTime.Add(TimeSpan.FromSeconds(1)), symbol, string.Empty, string.Empty, 14, 15, 16, 17),
+                new Tick(initialTime.Add(TimeSpan.FromSeconds(10)), symbol, string.Empty, string.Empty, 18, 19, 20, 21),
+                new Tick(initialTime.Add(TimeSpan.FromSeconds(61)), symbol, string.Empty, string.Empty, 22, 23, 24, 25),
+            };
+
+            var aggregated = LeanData.AggregateTicks(initialTicks, symbol, resolution).ToList();
+
+            Assert.True(aggregated.All(i => i.Period == resolution));
+
+            var firstBar = aggregated.First();
+
+            AssertBarsAreEqual(expectedFirstBar.Ask, firstBar.Ask);
+            AssertBarsAreEqual(expectedFirstBar.Bid, firstBar.Bid);
+        }
 
         private static void AssertBarsAreEqual(IBar expected, IBar actual)
         {
@@ -641,6 +710,44 @@ private static TestCaseData[] GetLeanDataLineTestParameters()
             }.Select(x => new TestCaseData(x).SetName(x.Name)).ToArray();
         }
 
+        private static TestCaseData[] AggregateTradeBarsTestData
+        {
+            get
+            {
+                return new[]
+                {
+                    new TestCaseData(TimeSpan.FromMinutes(1), new TradeBar {Open = 10, Close = 9, High = 15, Low = 7, Volume = 200, Period = TimeSpan.FromMinutes(1)}),
+                    new TestCaseData(TimeSpan.FromHours(1), new TradeBar {Open = 10, Close = 21, High = 25, Low = 7, Volume = 290, Period = TimeSpan.FromHours(1)}),
+                    new TestCaseData(TimeSpan.FromDays(1), new TradeBar {Open = 10, Close = 11, High = 25, Low = 7, Volume = 310, Period = TimeSpan.FromDays(1)}),
+                };
+            }
+        }
+
+        private static TestCaseData[] AggregateQuoteBarsTestData
+        {
+            get
+            {
+                return new[]
+                {
+                    new TestCaseData(TimeSpan.FromMinutes(1), new QuoteBar {Ask = new Bar {Open = 10, High = 15, Low = 7, Close = 9}, Bid = {Open = 7, High = 14, Low = 4, Close = 5}, Period = TimeSpan.FromMinutes(1)}),
+                    new TestCaseData(TimeSpan.FromHours(1), new QuoteBar {Ask = new Bar {Open = 10, High = 25, Low = 7, Close = 21}, Bid = {Open = 7, High = 22, Low = 4, Close = 20}, Period = TimeSpan.FromMinutes(1)}),
+                    new TestCaseData(TimeSpan.FromDays(1), new QuoteBar {Ask = new Bar {Open = 10, High = 25, Low = 7, Close = 11}, Bid = {Open = 7, High = 22, Low = 4, Close = 10}, Period = TimeSpan.FromMinutes(1)}),
+                };
+            }
+        }
+
+        private static TestCaseData[] AggregateTickTestData
+        {
+            get
+            {
+                return new[]
+                {
+                    new TestCaseData(TimeSpan.FromSeconds(1), new QuoteBar {Ask = new Bar {Open = 13, High = 13, Low = 13, Close = 13}, Bid = {Open = 11, High = 11, Low = 11, Close = 11}, Period = TimeSpan.FromSeconds(1)}),
+                    new TestCaseData(TimeSpan.FromMinutes(1), new QuoteBar {Ask = new Bar {Open = 13, High = 21, Low = 13, Close = 21}, Bid = {Open = 11, High = 19, Low = 11, Close = 19}, Period = TimeSpan.FromMinutes(1)}),
+                };
+            }
+        }
+
         public class LeanDataTestParameters
         {
             public readonly string Name;

ToolBox/BinanceDownloader/BinanceDataDownloader.cs

@@ -97,35 +97,6 @@ internal Symbol GetSymbol(string ticker)
             return _symbolMapper.GetLeanSymbol(ticker, SecurityType.Crypto, Market.Binance);
         }
 
-        /// <summary>
-        /// Aggregates a list of minute bars at the requested resolution
-        /// </summary>
-        /// <param name="symbol"></param>
-        /// <param name="bars"></param>
-        /// <param name="resolution"></param>
-        /// <returns></returns>
-        internal IEnumerable<TradeBar> AggregateBars(Symbol symbol, IEnumerable<TradeBar> bars, TimeSpan resolution)
-        {
-            return
-                (from b in bars
-                 group b by b.Time.RoundDown(resolution)
-                     into g
-                 select new TradeBar
-                 {
-                     Symbol = symbol,
-                     Time = g.Key,
-                     Open = g.First().Open,
-                     High = g.Max(b => b.High),
-                     Low = g.Min(b => b.Low),
-                     Close = g.Last().Close,
-                     Volume = g.Sum(b => b.Volume),
-                     Value = g.Last().Close,
-                     DataType = MarketDataType.TradeBar,
-                     Period = resolution,
-                     EndTime = g.Key.AddMilliseconds(resolution.TotalMilliseconds)
-                 });
-        }
-
         public void Dispose()
         {
             _brokerage.Disconnect();

ToolBox/BinanceDownloader/BinanceDownloaderProgram.cs

@@ -65,7 +65,7 @@ public static void DataDownloader(IList<string> tickers, string resolution, Date
                             // Save the data (other resolutions)
                             foreach (var res in new[] { Resolution.Hour, Resolution.Daily })
                             {
-                                var resData = downloader.AggregateBars(symbol, bars, res.ToTimeSpan());
+                                var resData = LeanData.AggregateTradeBars(bars, symbol, res.ToTimeSpan());
 
                                 writer = new LeanDataWriter(res, symbol, dataDirectory);
                                 writer.Write(resData);