Event profiler inspired in QSTK (pyalgotrade.eventprofiler).

CHANGELOG

@@ -1,4 +1,5 @@
 Version 0.14 (TBD)
+. [NEW] Event profiler inspired in QSTK (pyalgotrade.eventprofiler).
 . [NEW] Cumulative returns filter (pyalgotrade.technical.cumret.CumulativeReturn).
 . [NEW] Z-Score filter (pyalgotrade.technical.stats.ZScore).
 . [FIX] Fixed a bug with loggers when using the optimizer: https://github.com/gbeced/pyalgotrade/issues/10. 

doc/eventprofiler.rst

@@ -0,0 +1,43 @@
+Event profiler
+==============
+
+Inspired in QSTK (http://wiki.quantsoftware.org/index.php?title=QSTK_Tutorial_9), the **eventprofiler** module is a tool to analyze,
+statistically, how events affect future equity prices.
+The event profiler scans over historical data for a specified event and then calculates the impact of that event on the equity prices in the past
+and the future over a certain lookback period.
+
+**The goal of this tool is to help you quickly validate an idea, before moving forward with the backtesting process.**
+
+.. automodule:: pyalgotrade.eventprofiler
+    :members:
+    :member-order: bysource
+    :show-inheritance:
+
+Example
+-------
+
+The following example is inspired on the 'Buy-on-Gap Model' from Ernie Chan's book:
+'Algorithmic Trading: Winning Strategies and Their Rationale':
+
+ * The idea is to select a stock near the market open whose returns from their previous day's lows
+   to today's open are lower that one standard deviation. The standard deviation is computed using
+   the daily close-to-close returns of the last 90 days. These are the stocks that "gapped down".
+ * This is narrowed down by requiring the open price to be higher than the 20-day moving average
+   of the closing price.
+
+.. literalinclude:: ../samples/eventstudy.py
+
+The code is doing 4 things:
+
+ 1. Declaring a :class:`Predicate` that implements the 'Buy-on-Gap Model' event identification.
+ 2. Loading bars for some stocks.
+ 3. Running the analysis.
+ 4. Plotting the results.
+
+This is what the output should look like:
+
+.. image:: ../samples/eventstudy.png
+
+.. literalinclude:: ../samples/eventstudy.output
+
+Note that **Cummulative returns are normalized to the time of the event**.

doc/index.rst

@@ -15,6 +15,7 @@ Contents:
     tutorial
     code
     tools
+    eventprofiler
     talib
     googleappengine
     mtgox

pyalgotrade/barfeed/sqlitefeed.py

@@ -136,6 +136,9 @@ def __init__(self, dbFilePath, frequency, maxLen=dataseries.DEFAULT_MAX_LEN):
 		membf.Feed.__init__(self, frequency, maxLen)
 		self.__db = Database(dbFilePath)
 
+	def barsHaveAdjClose(self):
+		return True
+
 	def getDatabase(self):
 		return self.__db
 

pyalgotrade/eventstudy.py → pyalgotrade/eventprofiler.py

@@ -25,12 +25,27 @@
 from pyalgotrade import observer
 
 class Results:
-	def __init__(self, lookBack, lookForward):
+	"""Results from the profiler.""" 
+	def __init__(self, eventsDict, lookBack, lookForward):
 		assert(lookBack > 0)
 		assert(lookForward > 0)
 		self.__lookBack = lookBack
 		self.__lookForward = lookForward
 		self.__values = [[] for i in xrange(lookBack+lookForward+1)]
+		self.__eventCount = 0
+
+		# Process events.
+		for instrument, events in eventsDict.items():
+			for event in events:
+				# Skip events which are on the boundary.
+				if not event.onBoundary():
+					self.__eventCount += 1
+					# Compute cumulative returns: (1 + R1)*(1 + R2)*...*(1 + Rn)
+					values = np.cumprod(event.getValues() + 1)
+					# Normalize everything to the time of the event
+					values = values / values[event.getLookBack()]
+					for t in range(event.getLookBack()*-1, event.getLookForward()+1):
+						self.setValue(t, values[t+event.getLookBack()])
 
 	def __mapPos(self, t):
 		assert(t >= -1*self.__lookBack and t <= self.__lookForward)
@@ -52,8 +67,23 @@ def getLookBack(self):
 	def getLookForward(self):
 		return self.__lookForward
 
+	def getEventCount(self):
+		"""Returns the number of events occurred. Events that are on the boundary are skipped."""
+		return self.__eventCount
+
 class Predicate:
+	"""Base class for event identification. You should subclass this to implement
+	the event identification logic.""" 
+
 	def eventOccurred(self, instrument, bards):
+		"""Override (**mandatory**) to determine if an event took place in the last bar (bards[-1]).
+
+		:param instrument: Instrument identifier.
+		:type instrument: string.
+		:param bards: The BarDataSeries for the given instrument.
+		:type bards: :class:`pyalgotrade.dataseries.bards.BarDataSeries`.
+		:rtype: boolean.
+		"""
 		raise NotImplementedError()
 
 class Event:
@@ -92,6 +122,17 @@ def getValues(self):
 		return self.__values
 
 class Profiler:
+	"""This class is responsible for scanning over historical data and analyzing returns before
+	and after the events.
+
+	:param predicate: A :class:`Predicate` subclass responsible for identifying events.
+	:type predicate: :class:`Predicate`.
+	:param lookBack: The number of bars before the event to analyze. Must be > 0.
+	:type lookBack: int.
+	:param lookForward: The number of bars after the event to analyze. Must be > 0.
+	:type lookForward: int.
+	""" 
+
 	def __init__(self, predicate, lookBack, lookForward):
 		assert(lookBack > 0)
 		assert(lookForward > 0)
@@ -134,20 +175,21 @@ def __onBars(self, bars):
 				self.__futureRets[instrument].append((event, 1))
 
 	def getResults(self):
-		ret = Results(self.__lookBack, self.__lookForward)
-		for instrument, events in self.__events.items():
-			for event in events:
-				# Skip events which are on the boundary.
-				if not event.onBoundary():
-					# Compute cumulative returns: (1 + R1)*(1 + R2)*...*(1 + Rn)
-					values = np.cumprod(event.getValues() + 1)
-					# Normalize everything to the time of the event
-					values = values / values[event.getLookBack()]
-					for t in range(event.getLookBack()*-1, event.getLookForward()+1):
-						ret.setValue(t, values[t+event.getLookBack()])
-		return ret
+		"""Returns the results of the analysis.
+	
+		:rtype: :class:`Results`.
+		"""
+		return Results(self.__events, self.__lookBack, self.__lookForward)
 
 	def run(self, feed, useAdjustedCloseForReturns=True):
+		"""Runs the analysis using the bars supplied by the feed.
+
+		:param barFeed: The bar feed to use to run the analysis.
+		:type barFeed: :class:`pyalgotrade.barfeed.BarFeed`.
+		:param useAdjustedCloseForReturns: True if adjusted close values should be used to calculate returns.
+		:type useAdjustedCloseForReturns: boolean.
+		"""
+
 		try:
 			self.__feed = feed
 			self.__rets = {}
@@ -168,33 +210,37 @@ def run(self, feed, useAdjustedCloseForReturns=True):
 		finally:
 			feed.getNewBarsEvent().unsubscribe(self.__onBars)
 
-def build_plot(eventResults):
+def build_plot(profilerResults):
 	# Calculate each value.
 	x = []
 	y = []
 	std = []
-	for t in xrange(eventResults.getLookBack()*-1, eventResults.getLookForward()+1):
+	for t in xrange(profilerResults.getLookBack()*-1, profilerResults.getLookForward()+1):
 		x.append(t)
-		values = np.array(eventResults.getValues(t))
-		# This will fail if we don't have the same number of values on each window
-		# values = np.array(eventResults.getValues(0)) / np.array(eventResults.getValues(t))
+		values = np.array(profilerResults.getValues(t))
 		y.append(values.mean())
 		std.append(values.std())
 
 	# Plot
 	plt.clf()
 	plt.plot(x, y, color='#0000FF')
-	eventT = eventResults.getLookBack()
+	eventT = profilerResults.getLookBack()
 	# stdBegin = eventT + 1
 	# plt.errorbar(x[stdBegin:], y[stdBegin:], std[stdBegin:], alpha=0, ecolor='#AAAAFF')
 	plt.errorbar(x[eventT+1:], y[eventT+1:], std[eventT+1:], alpha=0, ecolor='#AAAAFF')
 	# plt.errorbar(x, y, std, alpha=0, ecolor='#AAAAFF')
-	plt.axhline(y=y[eventT],xmin=-1*eventResults.getLookBack(), xmax=eventResults.getLookForward(), color='#000000')
-	plt.xlim(eventResults.getLookBack()*-1-0.5, eventResults.getLookForward()+0.5)
+	plt.axhline(y=y[eventT],xmin=-1*profilerResults.getLookBack(), xmax=profilerResults.getLookForward(), color='#000000')
+	plt.xlim(profilerResults.getLookBack()*-1-0.5, profilerResults.getLookForward()+0.5)
 	plt.xlabel('Time')
 	plt.ylabel('Cumulative returns')
 
-def plot(eventResults):
-	build_plot(eventResults)
+def plot(profilerResults):
+	"""Plots the result of the analysis.
+
+	:param profilerResults: The result of the analysis
+	:type profilerResults: :class:`Results`.
+	"""
+
+	build_plot(profilerResults)
 	plt.show()
 

pyalgotrade/tools/yahoofinance.py

@@ -19,6 +19,10 @@
 """
 
 import urllib
+import os
+
+import pyalgotrade.logger
+from pyalgotrade.barfeed import yahoofeed
 
 def __adjust_month(month):
 	if month > 12 or month < 1:
@@ -73,3 +77,28 @@ def download_daily_bars(instrument, year, csvFile):
 	f.write(bars)
 	f.close()
 
+def build_feed(instruments, fromYear, toYear, storage, timezone=None, skipErrors=False):
+	logger = pyalgotrade.logger.getLogger("yahoofinance")
+	ret = yahoofeed.Feed()
+
+	if not os.path.exists(storage):
+		logger.info("Creating %s directory" % (storage))
+		os.mkdir(storage)
+
+	for year in range(fromYear, toYear+1):
+		for instrument in instruments:
+			fileName = os.path.join(storage, "%s-%d-yahoofinance.csv" % (instrument, year))
+			if not os.path.exists(fileName):
+				logger.info("Downloading %s %d to %s" % (instrument, year, fileName))
+				try: 
+					download_daily_bars(instrument, year, fileName)
+				except Exception, e:
+					if skipErrors:
+						logger.error(str(e))
+						continue
+					else:
+						raise e
+			ret.addBarsFromCSV(instrument, fileName)
+	return ret
+
+

release-checklist.txt

@@ -14,6 +14,7 @@ Release checklist
 [ ] Run samples/compinv-3.py using the installed lib.
 [ ] Run samples/tutorial_mtgox_2.py using the installed lib. Check disconnection detection.
 [ ] Run samples/tutorial_twitter_mtgox.py using the installed lib.
+[ ] Run samples/eventstudy.py using the installed lib.
 
 [ ] Update the website (doc + package).
 [ ] Build package and upload to sourceforge.