Skip to content

Commit

Permalink
Removed reference to System.IO.Compression zip library in Compression…
Browse files Browse the repository at this point in the history 
….Unzip() and DataFileCacheProvider.cs, as in mono this namespace is implemented using SharpCompress. Reverted back to Ionic zip library.
  • Loading branch information
@quant1729
quant1729 committed Jan 21, 2017
1 parent 3e3cf7d commit 4382821
Show file tree
Hide file tree
Showing 5 changed files with 28 additions and 39 deletions.
17 changes: 6 additions & 11 deletions Compression/Compression.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,7 @@ public static void ZipFiles(string destination, IEnumerable<string> files)
/// <param name="filename">Location of the original zip file</param>
/// <param name="zip">The ZipFile instance to be returned to the caller</param>
/// <returns>Stream reader of the first file contents in the zip file</returns>
public static StreamReader Unzip(string filename, out ZipArchive zip)
public static StreamReader Unzip(string filename, out ZipFile zip)
{
return Unzip(filename, null, out zip);
}
Expand All @@ -485,9 +485,9 @@ public static StreamReader Unzip(string filename, out ZipArchive zip)
/// </summary>
/// <param name="filename">Location of the original zip file</param>
/// <param name="zipEntryName">The zip entry name to open a reader for. Specify null to access the first entry</param>
/// <param name="zip">The ZipArchive instance to be returned to the caller</param>
/// <param name="zip">The ZipFile instance to be returned to the caller</param>
/// <returns>Stream reader of the first file contents in the zip file</returns>
public static StreamReader Unzip(string filename, string zipEntryName, out ZipArchive zip)
public static StreamReader Unzip(string filename, string zipEntryName, out ZipFile zip)
{
StreamReader reader = null;
zip = null;
Expand All @@ -498,20 +498,15 @@ public static StreamReader Unzip(string filename, string zipEntryName, out ZipAr
{
try
{
var file = File.OpenRead(filename);
// reading up first two bytes
// http://george.chiramattel.com/blog/2007/09/deflatestream-block-length-does-not-match.html
file.ReadByte(); file.ReadByte();

zip = new ZipArchive(file);
var entry = zip.Entries.FirstOrDefault(x => zipEntryName == null || string.Compare(x.FullName, zipEntryName, StringComparison.OrdinalIgnoreCase) == 0);
zip = new ZipFile(filename);
var entry = zip.FirstOrDefault(x => zipEntryName == null || string.Compare(x.FileName, zipEntryName, StringComparison.OrdinalIgnoreCase) == 0);
if (entry == null)
{
// Unable to locate zip entry
return null;
}

reader = new StreamReader(entry.Open());
reader = new StreamReader(entry.OpenReader());
}
catch (Exception err)
{
Expand Down
23 changes: 9 additions & 14 deletions Engine/DataFeeds/DataFileCacheProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@
using System.IO;
using QuantConnect.Lean.Engine.DataFeeds.Transport;
using System.Collections.Concurrent;
using System.IO.Compression;
using QuantConnect.Logging;
using System.Linq;
using Ionic.Zip;

namespace QuantConnect.Lean.Engine.DataFeeds
{
// Cache entry tuple contains: date of the cache entry, and reference to .net zip archive
using CacheEntry = Tuple<DateTime, ZipArchive>;
using CacheEntry = Tuple<DateTime, ZipFile>;

/// <summary>
/// File provider implements optimized zip archives caching facility. Cache is thread safe.
Expand All @@ -35,7 +35,7 @@ public class DataFileCacheProvider : IDisposable
const int _cachePeriodBars = 10;

// ZipArchive cache used by the class
private ConcurrentDictionary<string, Lazy<CacheEntry>> _zipArchiveCache = new ConcurrentDictionary<string, Lazy<CacheEntry>>();
private ConcurrentDictionary<string, Lazy<CacheEntry>> _zipFileCache = new ConcurrentDictionary<string, Lazy<CacheEntry>>();
private DateTime lastDate = DateTime.MinValue;

/// <summary>
Expand Down Expand Up @@ -68,28 +68,23 @@ public IStreamReader Fetch(Symbol symbol, SubscriptionDataSource source, DateTim
if (lastDate == DateTime.MinValue || lastDate < date.Date)
{
// clean all items that that are older than _cachePeriodBars bars than the current date
foreach (var zip in _zipArchiveCache.Where(x => x.Value.Value.Item1 < date.Date.AddDays(-_cachePeriodBars)))
foreach (var zip in _zipFileCache.Where(x => x.Value.Value.Item1 < date.Date.AddDays(-_cachePeriodBars)))
{
// disposing zip archive
zip.Value.Value.Item2.Dispose();

// removing it from the cache
Lazy<CacheEntry> removed;
_zipArchiveCache.TryRemove(zip.Key, out removed);
_zipFileCache.TryRemove(zip.Key, out removed);
}

lastDate = date.Date;
}

_zipArchiveCache.AddOrUpdate(filename,
_zipFileCache.AddOrUpdate(filename,
x =>
{
var file = File.OpenRead(filename);
// reading up first two bytes
// http://george.chiramattel.com/blog/2007/09/deflatestream-block-length-does-not-match.html
file.ReadByte(); file.ReadByte();

var newItem = Tuple.Create(date.Date, new ZipArchive(file));
var newItem = Tuple.Create(date.Date, new ZipFile(filename));
reader = new LocalFileSubscriptionStreamReader(newItem.Item2, entryName);
return newItem;
},
Expand Down Expand Up @@ -121,12 +116,12 @@ public IStreamReader Fetch(Symbol symbol, SubscriptionDataSource source, DateTim
/// <filterpriority>2</filterpriority>
public void Dispose()
{
foreach (var zip in _zipArchiveCache)
foreach (var zip in _zipFileCache)
{
zip.Value.Value.Item2.Dispose();
}

_zipArchiveCache.Clear();
_zipFileCache.Clear();
}
}
}
18 changes: 9 additions & 9 deletions Engine/DataFeeds/Transport/LocalFileSubscriptionStreamReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Transport
public class LocalFileSubscriptionStreamReader : IStreamReader
{
private StreamReader _streamReader;
private readonly ZipArchive _zipArchive;
private readonly ZipFile _zipFile;

/// <summary>
/// Initializes a new instance of the <see cref="LocalFileSubscriptionStreamReader"/> class.
Expand All @@ -42,7 +42,7 @@ public LocalFileSubscriptionStreamReader(string source, string entryName = null)
{
// unzip if necessary
_streamReader = source.GetExtension() == ".zip"
? Compression.Unzip(source, entryName, out _zipArchive)
? Compression.Unzip(source, entryName, out _zipFile)
: new StreamReader(source);
}

Expand All @@ -57,7 +57,7 @@ public LocalFileSubscriptionStreamReader(string source, string entryName, long s
{
// unzip if necessary
_streamReader = source.GetExtension() == ".zip"
? Compression.Unzip(source, entryName, out _zipArchive)
? Compression.Unzip(source, entryName, out _zipFile)
: new StreamReader(source);

if (startingPosition != 0)
Expand All @@ -70,17 +70,17 @@ public LocalFileSubscriptionStreamReader(string source, string entryName, long s
/// <summary>
/// Initializes a new instance of the <see cref="LocalFileSubscriptionStreamReader"/> class.
/// </summary>
/// <param name="zipArchive">The local zip archive to be read</param>
/// <param name="zipFile">The local zip archive to be read</param>
/// <param name="entryName">Specifies the zip entry to be opened. Leave null if not applicable,
/// or to open the first zip entry found regardless of name</param>
public LocalFileSubscriptionStreamReader(ZipArchive zipArchive, string entryName = null)
public LocalFileSubscriptionStreamReader(ZipFile zipFile, string entryName = null)
{
_zipArchive = zipArchive;
var entry = _zipArchive.Entries.FirstOrDefault(x => entryName == null || string.Compare(x.FullName, entryName, StringComparison.OrdinalIgnoreCase) == 0);
_zipFile = zipFile;
var entry = _zipFile.Entries.FirstOrDefault(x => entryName == null || string.Compare(x.FileName, entryName, StringComparison.OrdinalIgnoreCase) == 0);
if (entry != null)
{
var stream = new MemoryStream();
entry.Open().CopyTo(stream);
entry.OpenReader().CopyTo(stream);
stream.Position = 0;
_streamReader = new StreamReader(stream);
}
Expand All @@ -93,7 +93,7 @@ public IEnumerable<string> EntryFileNames
{
get
{
return _zipArchive != null ? _zipArchive.Entries.Select(x => x.FullName).ToList() : Enumerable.Empty<string>();
return _zipFile != null ? _zipFile.Entries.Select(x => x.FileName).ToList() : Enumerable.Empty<string>();
}
}

Expand Down
6 changes: 3 additions & 3 deletions Tests/Compression/CompressionTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ public void ZipBytes()
public void ExtractsZipEntryByName()
{
var zip = Path.Combine("TestData", "multizip.zip");
ZipArchive zipArchive;
using (var entryStream = QuantConnect.Compression.Unzip(zip, "multizip/two.txt", out zipArchive))
using (zipArchive)
ZipFile zipFile;
using (var entryStream = QuantConnect.Compression.Unzip(zip, "multizip/two.txt", out zipFile))
using (zipFile)
{
var text = entryStream.ReadToEnd();
Assert.AreEqual("2", text);
Expand Down
3 changes: 1 addition & 2 deletions ToolBox/CoarseUniverseGenerator/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
using QuantConnect.Data.Auxiliary;
using QuantConnect.Util;
using Log = QuantConnect.Logging.Log;
using System.IO.Compression;

namespace QuantConnect.ToolBox.CoarseUniverseGenerator
{
Expand Down Expand Up @@ -203,7 +202,7 @@ public static ICollection<string> ProcessDailyFolder(string dailyFolder, string
}
}

ZipArchive zip;
ZipFile zip;
using (var reader = Compression.Unzip(file, out zip))
{
// 30 period EMA constant
Expand Down

0 comments on commit 4382821

Please sign in to comment.