From ee4c3367e0000f271486fad952f1c629f348a062 Mon Sep 17 00:00:00 2001 From: brillliantz Date: Wed, 15 Nov 2017 22:19:24 +0800 Subject: [PATCH] pre-release 0.4.0 --- doc/backtest.md | 2 +- doc/realtime.md | 80 +++++++ doc/source/backtest.rst | 2 +- doc/source/jaqs.trade.rst | 26 +-- doc/source/realtime.rst | 80 +++++++ jaqs/__init__.py | 2 +- jaqs/data/dataservice.py | 9 +- jaqs/data/dataview.py | 13 +- .../{FamaFrench1993.py => FamaFrench.py} | 60 ++---- jaqs/example/alpha/Graham.py | 55 ++--- jaqs/example/alpha/ICCombine.py | 204 ++++++++++-------- jaqs/example/alpha/build_selection_rules.py | 33 --- ...ks_2.py => select_stocks_industry_head.py} | 54 ++--- ...stocks_1.py => select_stocks_pe_profit.py} | 59 ++--- jaqs/example/alpha/single_factor_weight.py | 54 ++--- jaqs/example/eventdriven/ctaStrategy.py | 82 ------- jaqs/example/eventdriven/doubleMaStrategy.py | 87 -------- jaqs/example/eventdriven/run_market_making.py | 101 --------- jaqs/research/signaldigger/digger.py | 14 +- jaqs/research/test_signal.py | 11 +- jaqs/trade/analyze/analyze.py | 62 +++--- jaqs/trade/analyze/report.py | 4 +- jaqs/trade/backtest.py | 48 ++--- jaqs/trade/model.py | 6 +- jaqs/trade/portfoliomanager.py | 8 + jaqs/trade/pubsub.py | 66 ------ jaqs/trade/{realinstance.py => realtime.py} | 55 +++-- jaqs/trade/strategy.py | 58 ++--- jaqs/trade/{gateway.py => tradegateway.py} | 12 +- jaqs/util/md2rst.py | 6 +- setup.py | 2 +- test_double_ma.py | 10 +- tests/test_align.py | 1 + tests/test_backtest.py | 17 +- tests/test_calendar.py | 2 +- tests/test_data_api.py | 4 +- tests/test_dataservice.py | 12 ++ tests/test_dataview.py | 6 +- tests/test_report.py | 4 +- tests/test_spread_trading.py | 96 --------- tests/test_strategy.py | 4 +- tests/test_trade_api.py | 4 +- tests/test_util.py | 10 +- 43 files changed, 575 insertions(+), 950 deletions(-) rename jaqs/example/alpha/{FamaFrench1993.py => FamaFrench.py} (63%) delete mode 100644 jaqs/example/alpha/build_selection_rules.py rename jaqs/example/alpha/{select_stocks_2.py => select_stocks_industry_head.py} (68%) rename jaqs/example/alpha/{select_stocks_1.py => select_stocks_pe_profit.py} (62%) delete mode 100644 jaqs/example/eventdriven/ctaStrategy.py delete mode 100644 jaqs/example/eventdriven/doubleMaStrategy.py delete mode 100644 jaqs/example/eventdriven/run_market_making.py delete mode 100644 jaqs/trade/pubsub.py rename jaqs/trade/{realinstance.py => realtime.py} (64%) rename jaqs/trade/{gateway.py => tradegateway.py} (99%) delete mode 100644 tests/test_spread_trading.py diff --git a/doc/backtest.md b/doc/backtest.md index c1e1c21..2462ebd 100644 --- a/doc/backtest.md +++ b/doc/backtest.md @@ -45,7 +45,7 @@ def test_alpha_strategy_dataview(): "position_ratio": 0.5, } - gateway = DailyStockSimGateway() + gateway = AlphaTradeApi() gateway.init_from_config(props) context = model.Context(dataview=dv, gateway=gateway) diff --git a/doc/realtime.md b/doc/realtime.md index 23b958c..ac3f310 100644 --- a/doc/realtime.md +++ b/doc/realtime.md @@ -1 +1,81 @@ ## Real Time Trading + +使用**JAQS**进行回测与实盘运行的代码具有高一致性，回测满意后，只需以下几点改动，即可接入实盘/模拟撮合： + +1. 使用实盘交易的交易接口：将`BacktestTradeApi`替换为`RealTimeTradeApi` +2. 使用实盘交易主程序：将`EventBacktestInstance`替换为`EventRealTimeInstance` +3. 在数据接口`RemoteDataService`中订阅所要交易品种的行情 +4. 在主程序最后添加`time.sleep(9999999)`. 保证在事件循环运行中，主程序不会提前终止 +5. 实时行情均为逐笔或Tick数据，即使订阅多个品种，行情数据仍会逐个到达`strategy.on_tick()`函数 + +这里我们以双均线策略为例，展示实盘运行的代码： + +```python +props = {'symbol': 'rb1801.SHF'} +tapi = RealTimeTradeApi() +ins = EventRealTimeInstance() + +tapi.use_strategy(3) + +ds = RemoteDataService() +strat = DoubleMaStrategy() +pm = PortfolioManager() + +context = model.Context(data_api=ds, trade_api=tapi, + instance=ins, strategy=strat, pm=pm) + +ins.init_from_config(props) +ds.subscribe(props['symbol']) + +ins.run() + +time.sleep(999999) + +``` + +程序运行后，行情、策略、交易会在不同的线程内运行，我们只需要在`on_tick`中进行下单，在`on_trade`, `on_order_status`中处理交易回报即可。 + +正常收到行情如下： +``` +rb1801.SHF 20171116-211319500 (BID) 229@3889.00 | 3891.00@12 (ASK) +Fast MA = 3889.89 Slow MA = 3889.81 +rb1801.SHF 20171116-211320000 (BID) 224@3889.00 | 3891.00@13 (ASK) +Fast MA = 3889.93 Slow MA = 3889.83 +rb1801.SHF 20171116-211320500 (BID) 223@3889.00 | 3891.00@5 (ASK) +Fast MA = 3889.89 Slow MA = 3889.85 +rb1801.SHF 20171116-211321000 (BID) 223@3889.00 | 3891.00@5 (ASK) +Fast MA = 3889.93 Slow MA = 3889.88 +rb1801.SHF 20171116-211321500 (BID) 24@3890.00 | 3891.00@5 (ASK) +Fast MA = 3890.00 Slow MA = 3889.92 +``` + +如果发生下单、成交，则会收到如下回报： + +``` + +rb1801.SHF 20171116-211319000 (BID) 230@3889.00 | 3891.00@6 (ASK) +Fast MA = 3889.93 Slow MA = 3889.79 + +Strategy on order status: +New | 20171115( 211319) Buy rb1801.SHF@3.000 size = 1 + +Strategy on order status: +Accepted | 20171115( 211319) Buy rb1801.SHF@3.000 size = 1 + +Strategy on trade: +20171115( 211319) Buy rb1801.SHF@3.000 size = 1 + +Strategy on order status: +Filled | 20171115( 211319) Buy rb1801.SHF@3.000 size = 1 + +Strategy on task ind: +task_id = 81116000001 | task_status = Stopped | task_algo = +task_msg = +DONE Execution Report (00'00"039): ++---------+------+----------+------+----+---------+-----+----------+----+---------+------------+-------+-------+--------+---------+ +|underlyer| ba_id| symbol|jzcode|side|is_filled|price|fill_price|size|fill_size|pending_size|cancels|rejects|entrusts| duration| ++---------+------+----------+------+----+---------+-----+----------+----+---------+------------+-------+-------+--------+---------+ +| 0|800000|rb1801.SHF| 27509|Long| Y| 3.0| 3.0000| 1| 1| 0| 0| 0| 1|00'00"033| ++---------+------+----------+------+----+---------+-----+----------+----+---------+------------+-------+-------+--------+---------+ + +``` diff --git a/doc/source/backtest.rst b/doc/source/backtest.rst index b60375e..a73d9bf 100644 --- a/doc/source/backtest.rst +++ b/doc/source/backtest.rst @@ -47,7 +47,7 @@ series)和信号（float series）权重 "position_ratio": 0.5, } - gateway = DailyStockSimGateway() + gateway = AlphaTradeApi() gateway.init_from_config(props) context = model.Context(dataview=dv, gateway=gateway) diff --git a/doc/source/jaqs.trade.rst b/doc/source/jaqs.trade.rst index 6354ec0..7c173de 100644 --- a/doc/source/jaqs.trade.rst +++ b/doc/source/jaqs.trade.rst @@ -29,14 +29,6 @@ jaqs\.trade\.common module :undoc-members: :show-inheritance: -jaqs\.trade\.gateway module ---------------------------- - -.. automodule:: jaqs.trade.gateway - :members: - :undoc-members: - :show-inheritance: - jaqs\.trade\.model module ------------------------- @@ -53,26 +45,26 @@ jaqs\.trade\.portfoliomanager module :undoc-members: :show-inheritance: -jaqs\.trade\.pubsub module --------------------------- +jaqs\.trade\.realtime module +---------------------------- -.. automodule:: jaqs.trade.pubsub +.. automodule:: jaqs.trade.realtime :members: :undoc-members: :show-inheritance: -jaqs\.trade\.realinstance module --------------------------------- +jaqs\.trade\.strategy module +---------------------------- -.. automodule:: jaqs.trade.realinstance +.. automodule:: jaqs.trade.strategy :members: :undoc-members: :show-inheritance: -jaqs\.trade\.strategy module ----------------------------- +jaqs\.trade\.tradegateway module +-------------------------------- -.. automodule:: jaqs.trade.strategy +.. automodule:: jaqs.trade.tradegateway :members: :undoc-members: :show-inheritance: diff --git a/doc/source/realtime.rst b/doc/source/realtime.rst index 253dad0..2eee241 100644 --- a/doc/source/realtime.rst +++ b/doc/source/realtime.rst @@ -1,2 +1,82 @@ Real Time Trading ----------------- + +使用\ **JAQS**\ 进行回测与实盘运行的代码具有高一致性，回测满意后，只需以下几点改动，即可接入实盘/模拟撮合： + +#. 使用实盘交易的交易接口：将\ ``BacktestTradeApi``\ 替换为\ ``RealTimeTradeApi`` +#. 使用实盘交易主程序：将\ ``EventBacktestInstance``\ 替换为\ ``EventRealTimeInstance`` +#. 在数据接口\ ``RemoteDataService``\ 中订阅所要交易品种的行情 +#. 在主程序最后添加\ ``time.sleep(9999999)``. + 保证在事件循环运行中，主程序不会提前终止 +#. 实时行情均为逐笔或Tick数据，即使订阅多个品种，行情数据仍会逐个到达\ ``strategy.on_tick()``\ 函数 + +这里我们以双均线策略为例，展示实盘运行的代码： + +.. code:: python + + props = {'symbol': 'rb1801.SHF'} + tapi = RealTimeTradeApi() + ins = EventRealTimeInstance() + + tapi.use_strategy(3) + + ds = RemoteDataService() + strat = DoubleMaStrategy() + pm = PortfolioManager() + + context = model.Context(data_api=ds, trade_api=tapi, + instance=ins, strategy=strat, pm=pm) + + ins.init_from_config(props) + ds.subscribe(props['symbol']) + + ins.run() + + time.sleep(999999) + +程序运行后，行情、策略、交易会在不同的线程内运行，我们只需要在\ ``on_tick``\ 中进行下单，在\ ``on_trade``, +``on_order_status``\ 中处理交易回报即可。 + +正常收到行情如下： + +:: + + rb1801.SHF 20171116-211319500 (BID) 229@3889.00 | 3891.00@12 (ASK) + Fast MA = 3889.89 Slow MA = 3889.81 + rb1801.SHF 20171116-211320000 (BID) 224@3889.00 | 3891.00@13 (ASK) + Fast MA = 3889.93 Slow MA = 3889.83 + rb1801.SHF 20171116-211320500 (BID) 223@3889.00 | 3891.00@5 (ASK) + Fast MA = 3889.89 Slow MA = 3889.85 + rb1801.SHF 20171116-211321000 (BID) 223@3889.00 | 3891.00@5 (ASK) + Fast MA = 3889.93 Slow MA = 3889.88 + rb1801.SHF 20171116-211321500 (BID) 24@3890.00 | 3891.00@5 (ASK) + Fast MA = 3890.00 Slow MA = 3889.92 + +如果发生下单、成交，则会收到如下回报： + +:: + + rb1801.SHF 20171116-211319000 (BID) 230@3889.00 | 3891.00@6 (ASK) + Fast MA = 3889.93 Slow MA = 3889.79 + + Strategy on order status: + New | 20171115( 211319) Buy rb1801.SHF@3.000 size = 1 + + Strategy on order status: + Accepted | 20171115( 211319) Buy rb1801.SHF@3.000 size = 1 + + Strategy on trade: + 20171115( 211319) Buy rb1801.SHF@3.000 size = 1 + + Strategy on order status: + Filled | 20171115( 211319) Buy rb1801.SHF@3.000 size = 1 + + Strategy on task ind: + task_id = 81116000001 | task_status = Stopped | task_algo = + task_msg = + DONE Execution Report (00'00"039): + +---------+------+----------+------+----+---------+-----+----------+----+---------+------------+-------+-------+--------+---------+ + |underlyer| ba_id| symbol|jzcode|side|is_filled|price|fill_price|size|fill_size|pending_size|cancels|rejects|entrusts| duration| + +---------+------+----------+------+----+---------+-----+----------+----+---------+------------+-------+-------+--------+---------+ + | 0|800000|rb1801.SHF| 27509|Long| Y| 3.0| 3.0000| 1| 1| 0| 0| 0| 1|00'00"033| + +---------+------+----------+------+----+---------+-----+----------+----+---------+------------+-------+-------+--------+---------+ diff --git a/jaqs/__init__.py b/jaqs/__init__.py index 7de6cbd..4b10a57 100644 --- a/jaqs/__init__.py +++ b/jaqs/__init__.py @@ -4,4 +4,4 @@ SOURCE_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) -__version__ = '0.3.0' +__version__ = '0.4.0' diff --git a/jaqs/data/dataservice.py b/jaqs/data/dataservice.py index 94f56da..b13dbae 100644 --- a/jaqs/data/dataservice.py +++ b/jaqs/data/dataservice.py @@ -5,8 +5,6 @@ import numpy as np import pandas as pd -from jaqs.util import fileio -# from jaqs.trade.pubsub import Publisher from jaqs.trade.event import EVENT_TYPE, Event from jaqs.data.dataapi import DataApi from jaqs.data import align @@ -259,7 +257,10 @@ def init_from_config(self, props=None): props = dict() if self.data_api is not None: - self.data_api.close() + if len(props) == 0: + return + else: + self.data_api.close() def get_from_list_of_dict(l, key, default=None): res = None @@ -271,7 +272,7 @@ def get_from_list_of_dict(l, key, default=None): res = default return res - props_default = fileio.read_json(fileio.join_relative_path('etc/data_config.json')) + props_default = jutil.read_json(jutil.join_relative_path('etc/data_config.json')) dic_list = [props, props_default] address = get_from_list_of_dict(dic_list, "remote.address", "") diff --git a/jaqs/data/dataview.py b/jaqs/data/dataview.py index 125f260..5275fba 100644 --- a/jaqs/data/dataview.py +++ b/jaqs/data/dataview.py @@ -13,8 +13,7 @@ import numpy as np import pandas as pd -import jaqs.util.fileio -from jaqs.util import dtutil +import jaqs.util as jutil from jaqs.data.align import align from jaqs.data.py_expression_eval import Parser @@ -405,8 +404,8 @@ def init_from_config(self, props, data_api): # initialize parameters self.start_date = props['start_date'] - self.extended_start_date_d = dtutil.shift(self.start_date, n_weeks=-8) # query more data - self.extended_start_date_q = dtutil.shift(self.start_date, n_weeks=-80) + self.extended_start_date_d = jutil.shift(self.start_date, n_weeks=-8) # query more data + self.extended_start_date_q = jutil.shift(self.start_date, n_weeks=-80) self.end_date = props['end_date'] self.all_price = props.get('all_price', True) self.freq = props.get('freq', 1) @@ -1326,7 +1325,7 @@ def load_dataview(self, folder_path='.'): Folder path to store hd5 file and meta data. """ - meta_data = jaqs.util.fileio.read_json(os.path.join(folder_path, 'meta_data.json')) + meta_data = jutil.read_json(os.path.join(folder_path, 'meta_data.json')) dic = self._load_h5(os.path.join(folder_path, 'data.hd5')) self.data_d = dic.get('/data_d', None) self.data_q = dic.get('/data_q', None) @@ -1357,7 +1356,7 @@ def save_dataview(self, folder_path): meta_data_to_store = {key: self.__dict__[key] for key in self.meta_data_list} print "\nStore data..." - jaqs.util.fileio.save_json(meta_data_to_store, meta_path) + jutil.save_json(meta_data_to_store, meta_path) self._save_h5(data_path, data_to_store) print ("Dataview has been successfully saved to:\n" @@ -1379,7 +1378,7 @@ def _save_h5(fp, dic): import warnings warnings.filterwarnings('ignore', category=pd.io.pytables.PerformanceWarning) - jaqs.util.fileio.create_dir(fp) + jutil.create_dir(fp) h5 = pd.HDFStore(fp, complevel=9, complib='blosc') for key, value in dic.viewitems(): h5[key] = value diff --git a/jaqs/example/alpha/FamaFrench1993.py b/jaqs/example/alpha/FamaFrench.py similarity index 63% rename from jaqs/example/alpha/FamaFrench1993.py rename to jaqs/example/alpha/FamaFrench.py index 818971d..b28b576 100644 --- a/jaqs/example/alpha/FamaFrench1993.py +++ b/jaqs/example/alpha/FamaFrench.py @@ -19,13 +19,14 @@ from jaqs.data.dataservice import RemoteDataService from jaqs.data.dataview import DataView from jaqs.trade import model +from jaqs.trade.portfoliomanager import PortfolioManager from jaqs.trade.backtest import AlphaBacktestInstance -from jaqs.trade.gateway import DailyStockSimGateway +from jaqs.trade.tradegateway import AlphaTradeApi from jaqs.trade.strategy import AlphaStrategy -from jaqs.util import fileio +import jaqs.util as jutil -dataview_dir_path = fileio.join_relative_path('../output/prepared/fama_french/dataview') -backtest_result_dir_path = fileio.join_relative_path('../output/fama_french') +dataview_dir_path = jutil.join_relative_path('../output/fama_french/dataview') +backtest_result_dir_path = jutil.join_relative_path('../output/fama_french') def test_save_dataview(sub_folder='test_dataview'): @@ -79,20 +80,21 @@ def test_alpha_strategy_dataview(): "position_ratio": 1.0, } - gateway = DailyStockSimGateway() - gateway.init_from_config(props) + trade_api = AlphaTradeApi() + trade_api.init_from_config(props) - context = model.Context(dataview=dv, gateway=gateway) - - stock_selector = model.StockSelector(context) + stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) - strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight', - # revenue_model=signal_model - ) + strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') + pm = PortfolioManager() bt = AlphaBacktestInstance() - bt.init_from_config(props, strategy, context=context) + + context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) + stock_selector.register_context(context) + + bt.init_from_config(props) bt.run_alpha() @@ -105,36 +107,8 @@ def test_backtest_analyze(): dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) - - print "process trades..." - ta.process_trades() - print "get daily stats..." - ta.get_daily() - ######################### - # ta.daily[symbol] # = df - - print "calc strategy return..." - ta.get_returns() - # position change info is huge! - # print "get position change..." - # ta.get_pos_change_info() - - selected_sec = [] # list(ta.universe)[:5] - if len(selected_sec) > 0: - print "Plot single securities PnL" - for symbol in selected_sec: - df_daily = ta.daily.get(symbol, None) - if df_daily is not None: - ana.plot_trades(df_daily, symbol=symbol, save_folder=backtest_result_dir_path) - - print "Plot strategy PnL..." - ta.plot_pnl(backtest_result_dir_path) - - print "generate report..." - static_folder = fileio.join_relative_path("trade/analyze/static") - ta.gen_report(source_dir=static_folder, template_fn='report_template.html', - out_folder=backtest_result_dir_path, - selected=selected_sec) + + ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3]) if __name__ == "__main__": diff --git a/jaqs/example/alpha/Graham.py b/jaqs/example/alpha/Graham.py index d7842bc..4f8eb27 100644 --- a/jaqs/example/alpha/Graham.py +++ b/jaqs/example/alpha/Graham.py @@ -24,12 +24,13 @@ from jaqs.data.dataview import DataView from jaqs.trade import model from jaqs.trade.backtest import AlphaBacktestInstance -from jaqs.trade.gateway import DailyStockSimGateway +from jaqs.trade.tradegateway import AlphaTradeApi +from jaqs.trade.portfoliomanager import PortfolioManager from jaqs.trade.strategy import AlphaStrategy -from jaqs.util import fileio +import jaqs.util as jutil -dataview_dir_path = fileio.join_relative_path('../output/prepared/Graham_dataview') -backtest_result_dir_path = fileio.join_relative_path('../output/Graham') +dataview_dir_path = jutil.join_relative_path('../output/Graham/dataview') +backtest_result_dir_path = jutil.join_relative_path('../output/Graham') def test_save_dataview(): @@ -107,24 +108,28 @@ def test_alpha_strategy_dataview(): "position_ratio": 1.0, } - gateway = DailyStockSimGateway() + gateway = AlphaTradeApi() gateway.init_from_config(props) context = model.Context(dataview=dv, gateway=gateway) - stock_selector = model.StockSelector(context) + stock_selector = model.StockSelector() stock_selector.add_filter(name='myselector', func=my_selector) - signal_model = model.FactorRevenueModel(context) + signal_model = model.FactorRevenueModel() signal_model.add_signal(name='signalsize', func=signal_size) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='factor_value_weight', - revenue_model=signal_model - ) + revenue_model=signal_model) + pm = PortfolioManager() bt = AlphaBacktestInstance() - bt.init_from_config(props, strategy, context=context) + context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) + for mdl in [signal_model, stock_selector]: + mdl.register_context(context) + + bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path) @@ -137,34 +142,8 @@ def test_backtest_analyze(): dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) - - print "process trades..." - ta.process_trades() - print "get daily stats..." - ta.get_daily() - - print "calc strategy return..." - ta.get_returns() - # position change info is huge! - # print "get position change..." - # ta.get_pos_change_info() - - selected_sec = [] # list(ta.universe)[:5] - if len(selected_sec) > 0: - print "Plot single securities PnL" - for symbol in selected_sec: - df_daily = ta.daily.get(symbol, None) - if df_daily is not None: - ana.plot_trades(df_daily, symbol=symbol, save_folder=backtest_result_dir_path) - - print "Plot strategy PnL..." - ta.plot_pnl(backtest_result_dir_path) - - print "generate report..." - static_folder = fileio.join_relative_path("trade/analyze/static") - ta.gen_report(source_dir=static_folder, template_fn='report_template.html', - out_folder=backtest_result_dir_path, - selected=selected_sec) + + ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3]) if __name__ == "__main__": diff --git a/jaqs/example/alpha/ICCombine.py b/jaqs/example/alpha/ICCombine.py index 00412e2..3da7d46 100644 --- a/jaqs/example/alpha/ICCombine.py +++ b/jaqs/example/alpha/ICCombine.py @@ -1,28 +1,35 @@ # -*- encoding: utf-8 -*- """ -Weekly rebalance +1. filter universe: separate helper functions +2. calc weights +3. generate trades -https://uqer.io/community/share/57b540ef228e5b79a4759398 +------------------------ -universe : hs300 -init_balance = 1e8 -start_date 20140101 -end_date 20170301 +- modify models: register function (with context parameter) +- modify AlphaStrategy: inheritate + +------------------------ + +suspensions and limit reachers: +1. deal with them in re_balance function, not in filter_universe +2. do not care about them when construct portfolio +3. subtract market value and re-normalize weights (positions) after (daily) market open, before sending orders """ import time - import numpy as np import numpy.linalg as nlg import pandas as pd import scipy.stats as stats - import jaqs.trade.analyze.analyze as ana + +from jaqs.trade.portfoliomanager import PortfolioManager from jaqs.data.dataservice import RemoteDataService from jaqs.data.dataview import DataView from jaqs.trade import model from jaqs.trade.backtest import AlphaBacktestInstance -from jaqs.trade.gateway import DailyStockSimGateway +from jaqs.trade.tradegateway import AlphaTradeApi from jaqs.trade.strategy import AlphaStrategy from jaqs.util import fileio @@ -33,29 +40,29 @@ custom_data_path = fileio.join_relative_path('../output/ICCombine', 'custom_date.json') -def test_save_dataview(): +def save_dataview(): ds = RemoteDataService() dv = DataView() - + props = {'start_date': 20150101, 'end_date': 20170930, 'universe': '000905.SH', 'fields': ('turnover,float_mv,close_adj,pe,pb'), 'freq': 1} - + dv.init_from_config(props, ds) dv.prepare_data() - + factor_formula = 'Cutoff(Standardize(turnover / 10000 / float_mv), 2)' dv.add_formula('TO', factor_formula, is_quarterly=False) - + factor_formula = 'Cutoff(Standardize(1/pb), 2)' dv.add_formula('BP', factor_formula, is_quarterly=False) - + factor_formula = 'Cutoff(Standardize(Return(close_adj, 20)), 2)' dv.add_formula('REVS20', factor_formula, is_quarterly=False) - + factor_formula = 'Cutoff(Standardize(Log(float_mv)), 2)' dv.add_formula('float_mv_factor', factor_formula, is_quarterly=False) - + factor_formula = 'Delay(Return(close_adj, 1), -1)' dv.add_formula('NextRet', factor_formula, is_quarterly=False) @@ -89,7 +96,7 @@ def get_ic(dv): for singleDate in dv.dates: singleSnapshot = dv.get_snapshot(singleDate) ICPanel[singleDate] = ic_calculation(singleSnapshot, factorList) - + ICPanel = pd.DataFrame(ICPanel).T return ICPanel @@ -151,59 +158,112 @@ def my_selector(context, user_options=None): t_date = context.trade_date current_ic_weight = np.mat(ic_weight.loc[t_date,]).reshape(-1, 1) factorList = context.factorList - + factorPanel = {} for factor in factorList: factorPanel[factor] = context.snapshot[factor] - + factorPanel = pd.DataFrame(factorPanel) factorResult = pd.DataFrame(np.mat(factorPanel) * np.mat(current_ic_weight), index=factorPanel.index) - + factorResult = factorResult.fillna(-9999) s = factorResult.sort_values(0)[::-1] - + critical = s.values[30] mask = factorResult > critical factorResult[mask] = 1.0 factorResult[~mask] = 0.0 - + return factorResult +def store_ic_weight(): + """ + Calculate IC weight and save it to file + """ + dv = DataView() + + dv.load_dataview(folder_path=dataview_dir_path) + + factorList = ['TO', 'BP', 'REVS20', 'float_mv_factor'] + + orthFactor_dic = {} + + for factor in factorList: + orthFactor_dic[factor] = {} + + # add the orthogonalized factor to dataview + for trade_date in dv.dates: + snapshot = dv.get_snapshot(trade_date) + factorPanel = snapshot[factorList] + factorPanel = factorPanel.dropna() + + if len(factorPanel) != 0: + orthfactorPanel = Schmidt(factorPanel) + orthfactorPanel.columns = [x + '_adj' for x in factorList] + + snapshot = pd.merge(left=snapshot, right=orthfactorPanel, + left_index=True, right_index=True, how='left') + + for factor in factorList: + orthFactor_dic[factor][trade_date] = snapshot[factor] + + for factor in factorList: + dv.append_df(pd.DataFrame(orthFactor_dic[factor]).T, field_name=factor + '_adj', is_quarterly=False) + dv.save_dataview(dataview_dir_path) + + factorList_adj = [x + '_adj' for x in factorList] + + fileio.save_json(factorList_adj, custom_data_path) + + w = get_ic_weight(dv) + + store = pd.HDFStore(ic_weight_hd5_path) + store['ic_weight'] = w + store.close() + + def test_alpha_strategy_dataview(): + dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) props = { "start_date": dv.start_date, "end_date": dv.end_date, - + "period": "week", "days_delay": 0, - - "init_balance": 1e8, - "position_ratio": 1.0, - } - - gateway = DailyStockSimGateway() - gateway.init_from_config(props) - - context = model.Context(dataview=dv, gateway=gateway) + "init_balance": 1e8, + "position_ratio": 0.7, + 'commission_rate': 0.0 + } + + trade_api = AlphaTradeApi() + bt = AlphaBacktestInstance() + + stock_selector = model.StockSelector() + stock_selector.add_filter(name='myselector', func=my_selector) + + strategy = AlphaStrategy(stock_selector=stock_selector, + pc_method='equal_weight') + pm = PortfolioManager() + + context = model.AlphaContext(dataview=dv, trade_api=trade_api, + instance=bt, strategy=strategy, pm=pm) + store = pd.HDFStore(ic_weight_hd5_path) factorList = fileio.read_json(custom_data_path) context.ic_weight = store['ic_weight'] context.factorList = factorList store.close() - - stock_selector = model.StockSelector(context) - stock_selector.add_filter(name='myselector', func=my_selector) - - strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') - - bt = AlphaBacktestInstance() - bt.init_from_config(props, strategy, context=context) - + + for mdl in [stock_selector]: + mdl.register_context(context) + + bt.init_from_config(props) + bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path) @@ -211,23 +271,23 @@ def test_alpha_strategy_dataview(): def test_backtest_analyze(): ta = ana.AlphaAnalyzer() + dv = DataView() dv.load_dataview(folder_path=dataview_dir_path) - - ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) + ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) + print "process trades..." ta.process_trades() print "get daily stats..." ta.get_daily() - print "calc strategy return..." - ta.get_returns(consider_commission=False) + ta.get_returns(consider_commission=True) # position change info is huge! # print "get position change..." # ta.get_pos_change_info() - selected_sec = [] # list(ta.universe)[:5] + selected_sec = list(ta.universe)[:2] if len(selected_sec) > 0: print "Plot single securities PnL" for symbol in selected_sec: @@ -245,59 +305,13 @@ def test_backtest_analyze(): selected=selected_sec) -def store_ic_weight(): - """ - Calculate IC weight and save it to file - """ - dv = DataView() - - dv.load_dataview(folder_path=dataview_dir_path) - - factorList = ['TO', 'BP', 'REVS20', 'float_mv_factor'] - - orthFactor_dic = {} - - for factor in factorList: - orthFactor_dic[factor] = {} - - # add the orthogonalized factor to dataview - for trade_date in dv.dates: - snapshot = dv.get_snapshot(trade_date) - factorPanel = snapshot[factorList] - factorPanel = factorPanel.dropna() - - if len(factorPanel) != 0: - orthfactorPanel = Schmidt(factorPanel) - orthfactorPanel.columns = [x + '_adj' for x in factorList] - - snapshot = pd.merge(left=snapshot, right=orthfactorPanel, - left_index=True, right_index=True, how='left') - - for factor in factorList: - orthFactor_dic[factor][trade_date] = snapshot[factor] - - for factor in factorList: - dv.append_df(pd.DataFrame(orthFactor_dic[factor]).T, field_name=factor + '_adj', is_quarterly=False) - dv.save_dataview(dataview_dir_path) - - factorList_adj = [x + '_adj' for x in factorList] - - fileio.save_json(factorList_adj, custom_data_path) - - w = get_ic_weight(dv) - - store = pd.HDFStore(ic_weight_hd5_path) - store['ic_weight'] = w - store.close() - - if __name__ == "__main__": t_start = time.time() - - test_save_dataview() + save_dataview() store_ic_weight() test_alpha_strategy_dataview() test_backtest_analyze() t3 = time.time() - t_start print "\n\n\nTime lapsed in total: {:.1f}".format(t3) + diff --git a/jaqs/example/alpha/build_selection_rules.py b/jaqs/example/alpha/build_selection_rules.py deleted file mode 100644 index d0080dc..0000000 --- a/jaqs/example/alpha/build_selection_rules.py +++ /dev/null @@ -1,33 +0,0 @@ -# encoding: utf-8 - -import numpy as np -import pandas as pd - -from jaqs.data.dataview import DataView -from jaqs.data.dataservice import RemoteDataService -from jaqs.util import fileio - - -def build_stock_selection_factor(): - ds = RemoteDataService() - dv = DataView() - - props = {'start_date': 20120101, 'end_date': 20170901, 'universe': '000300.SH', - # 'symbol': 'rb1710.SHF,rb1801.SHF', - 'fields': ('open,high,low,close,vwap,volume,turnover,' - # + 'pb,net_assets,' - + 's_fa_eps_basic,oper_exp,tot_profit,int_income' - ), - 'freq': 1} - - dv.init_from_config(props, ds) - dv.prepare_data() - - dv.add_formula('eps_ret', 'Return(s_fa_eps_basic, 4)', is_quarterly=True) - dv.add_formula('rule1', '(eps_ret > 0.2) && (Delay(eps_ret, 1) > 0.2)', is_quarterly=True) - dv.add_formula('rule2', 'close > Ts_Max(close, 120)', is_quarterly=False) - # dv.add_formula('ytan', 'rule1 && rule2', is_quarterly=False) - - dv.add_formula('ret20', 'Delay(Return(close_adj, 20), -20)', is_quarterly=False) - - dv.save_dataview(folder_path=fileio.join_relative_path('../output/prepared')) diff --git a/jaqs/example/alpha/select_stocks_2.py b/jaqs/example/alpha/select_stocks_industry_head.py similarity index 68% rename from jaqs/example/alpha/select_stocks_2.py rename to jaqs/example/alpha/select_stocks_industry_head.py index 7280fb2..a39c7e1 100644 --- a/jaqs/example/alpha/select_stocks_2.py +++ b/jaqs/example/alpha/select_stocks_industry_head.py @@ -14,20 +14,22 @@ """ import time +import pandas as pd + from jaqs.data.dataservice import RemoteDataService from jaqs.trade.backtest import AlphaBacktestInstance -from jaqs.util import fileio +import jaqs.util as jutil +from jaqs.trade.portfoliomanager import PortfolioManager import jaqs.trade.analyze.analyze as ana from jaqs.trade.strategy import AlphaStrategy -from jaqs.trade.gateway import DailyStockSimGateway +from jaqs.trade.tradegateway import AlphaTradeApi from jaqs.trade import model from jaqs.data.dataview import DataView -import pandas as pd -dataview_dir_path = fileio.join_relative_path('../output/prepared/select_stocks2') -backtest_result_dir_path = fileio.join_relative_path('../output/select_stocks2') +dataview_dir_path = jutil.join_relative_path('../output/prepared/select_stocks2') +backtest_result_dir_path = jutil.join_relative_path('../output/select_stocks2') def test_save_dataview(): @@ -86,19 +88,22 @@ def test_alpha_strategy_dataview(): "position_ratio": 1.0, } - gateway = DailyStockSimGateway() - gateway.init_from_config(props) + trade_api = AlphaTradeApi() - context = model.Context(dataview=dv, gateway=gateway) + context = model.Context(dataview=dv, gateway=trade_api) stock_selector = model.StockSelector(context) stock_selector.add_filter(name='myrank', func=my_selector) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') + pm = PortfolioManager() bt = AlphaBacktestInstance() - bt.init_from_config(props, strategy, context=context) - + + context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) + stock_selector.register_context(context) + + bt.init_from_config(props) bt.run_alpha() bt.save_results(folder_path=backtest_result_dir_path) @@ -110,33 +115,8 @@ def test_backtest_analyze(): dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) - - print "process trades..." - ta.process_trades() - print "get daily stats..." - ta.get_daily() - print "calc strategy return..." - ta.get_returns() - # position change info is huge! - print "get position change..." - ta.get_pos_change_info() - - selected_sec = list(ta.universe)[:3] - if len(selected_sec) > 0: - print "Plot single securities PnL" - for symbol in selected_sec: - df_daily = ta.daily.get(symbol, None) - if df_daily is not None: - ana.plot_trades(df_daily, symbol=symbol, save_folder=backtest_result_dir_path) - - print "Plot strategy PnL..." - ta.plot_pnl(backtest_result_dir_path) - - print "generate report..." - static_folder = fileio.join_relative_path("trade/analyze/static") - ta.gen_report(source_dir=static_folder, template_fn='report_template.html', - out_folder=backtest_result_dir_path, - selected=selected_sec) + + ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3]) if __name__ == "__main__": diff --git a/jaqs/example/alpha/select_stocks_1.py b/jaqs/example/alpha/select_stocks_pe_profit.py similarity index 62% rename from jaqs/example/alpha/select_stocks_1.py rename to jaqs/example/alpha/select_stocks_pe_profit.py index e756e29..5179154 100644 --- a/jaqs/example/alpha/select_stocks_1.py +++ b/jaqs/example/alpha/select_stocks_pe_profit.py @@ -14,15 +14,16 @@ from jaqs.data.dataservice import RemoteDataService from jaqs.trade.backtest import AlphaBacktestInstance -from jaqs.util import fileio +import jaqs.util as jutil import jaqs.trade.analyze.analyze as ana +from jaqs.trade.portfoliomanager import PortfolioManager from jaqs.trade.strategy import AlphaStrategy -from jaqs.trade.gateway import DailyStockSimGateway +from jaqs.trade.tradegateway import AlphaTradeApi from jaqs.trade import model from jaqs.data.dataview import DataView -dataview_dir_path = fileio.join_relative_path('../output/prepared/test_backtest') -backtest_result_dir_path = fileio.join_relative_path('../output/select_stocks') +dataview_dir_path = jutil.join_relative_path('../output/prepared/test_backtest') +backtest_result_dir_path = jutil.join_relative_path('../output/select_stocks') def test_save_dataview(): @@ -62,10 +63,8 @@ def test_alpha_strategy_dataview(): "position_ratio": 1.0, } - gateway = DailyStockSimGateway() - gateway.init_from_config(props) - - context = model.Context(dataview=dv, gateway=gateway) + trade_api = AlphaTradeApi() + trade_api.init_from_config(props) def selector_growth(context, user_options=None): growth_rate = context.snapshot['net_profit_growth'] @@ -75,17 +74,21 @@ def selector_pe(context, user_options=None): pe_ttm = context.snapshot['pe_ttm'] return (pe_ttm >= 10) & (pe_ttm <= 20) - stock_selector = model.StockSelector(context) + stock_selector = model.StockSelector() stock_selector.add_filter(name='net_profit_growth', func=selector_growth) stock_selector.add_filter(name='pe', func=selector_pe) strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='equal_weight') + pm = PortfolioManager() bt = AlphaBacktestInstance() - bt.init_from_config(props, strategy, context=context) - + + context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) + stock_selector.register_context(context) + + bt.init_from_config(props) bt.run_alpha() - + bt.save_results(folder_path=backtest_result_dir_path) @@ -95,38 +98,14 @@ def test_backtest_analyze(): dv.load_dataview(folder_path=dataview_dir_path) ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) - - print "process trades..." - ta.process_trades() - print "get daily stats..." - ta.get_daily() - print "calc strategy return..." - ta.get_returns() - # position change info is huge! - # print "get position change..." - # ta.get_pos_change_info() - - selected_sec = list(ta.universe)[:3] - if len(selected_sec) > 0: - print "Plot single securities PnL" - for symbol in selected_sec: - df_daily = ta.daily.get(symbol, None) - if df_daily is not None: - ana.plot_trades(df_daily, symbol=symbol, save_folder=backtest_result_dir_path) - - print "Plot strategy PnL..." - ta.plot_pnl(backtest_result_dir_path) - - print "generate report..." - static_folder = fileio.join_relative_path("trade/analyze/static") - ta.gen_report(source_dir=static_folder, template_fn='report_template.html', - out_folder=backtest_result_dir_path, - selected=selected_sec) + + ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3]) + if __name__ == "__main__": t_start = time.time() - test_save_dataview() + # test_save_dataview() test_alpha_strategy_dataview() test_backtest_analyze() diff --git a/jaqs/example/alpha/single_factor_weight.py b/jaqs/example/alpha/single_factor_weight.py index 6533df9..3c9b151 100644 --- a/jaqs/example/alpha/single_factor_weight.py +++ b/jaqs/example/alpha/single_factor_weight.py @@ -14,18 +14,17 @@ from jaqs.data.dataservice import RemoteDataService from jaqs.trade.backtest import AlphaBacktestInstance +from jaqs.trade.portfoliomanager import PortfolioManager -from jaqs.util import fileio +import jaqs.util as jutil import jaqs.trade.analyze.analyze as ana from jaqs.trade.strategy import AlphaStrategy -from jaqs.trade.gateway import DailyStockSimGateway +from jaqs.trade.tradegateway import AlphaTradeApi from jaqs.trade import model from jaqs.data.dataview import DataView -import pandas as pd - -dataview_dir_path = fileio.join_relative_path('../output/prepared/single_factor_weight') -backtest_result_dir_path = fileio.join_relative_path('../output/single_factor_weight') +dataview_dir_path = jutil.join_relative_path('../output/prepared/single_factor_weight') +backtest_result_dir_path = jutil.join_relative_path('../output/single_factor_weight') def test_save_dataview(): @@ -45,7 +44,6 @@ def test_save_dataview(): dv.save_dataview(folder_path=dataview_dir_path) - def test_alpha_strategy_dataview(): dv = DataView() @@ -65,23 +63,26 @@ def test_alpha_strategy_dataview(): "position_ratio": 1.0, } - gateway = DailyStockSimGateway() - gateway.init_from_config(props) - - context = model.Context(dataview=dv, gateway=gateway) + trade_api = AlphaTradeApi() def singal_gq30(context, user_options=None): import numpy as np res = np.power(context.snapshot['gq30'], 8) return res - signal_model = model.FactorRevenueModel(context) + signal_model = model.FactorRevenueModel() signal_model.add_signal('signal_gq30', singal_gq30) strategy = AlphaStrategy(revenue_model=signal_model, pc_method='factor_value_weight') + pm = PortfolioManager() bt = AlphaBacktestInstance() - bt.init_from_config(props, strategy, context=context) + + context = model.Context(dataview=dv, instance=bt, strategy=strategy, trade_api=trade_api, pm=pm) + + signal_model.register_context(context) + + bt.init_from_config(props) bt.run_alpha() @@ -95,32 +96,7 @@ def test_backtest_analyze(): ta.initialize(dataview=dv, file_folder=backtest_result_dir_path) - print "process trades..." - ta.process_trades() - print "get daily stats..." - ta.get_daily() - print "calc strategy return..." - ta.get_returns() - # position change info is huge! - # print "get position change..." - # ta.get_pos_change_info() - - selected_sec = list(ta.universe)[:5] - if len(selected_sec) > 0: - print "Plot single securities PnL" - for symbol in selected_sec: - df_daily = ta.daily.get(symbol, None) - if df_daily is not None: - ana.plot_trades(df_daily, symbol=symbol, save_folder=backtest_result_dir_path) - - print "Plot strategy PnL..." - ta.plot_pnl(backtest_result_dir_path) - - print "generate report..." - static_folder = fileio.join_relative_path("trade/analyze/static") - ta.gen_report(source_dir=static_folder, template_fn='report_template.html', - out_folder=backtest_result_dir_path, - selected=selected_sec) + ta.do_analyze(result_dir=backtest_result_dir_path, selected_sec=list(ta.universe)[:3]) if __name__ == "__main__": diff --git a/jaqs/example/eventdriven/ctaStrategy.py b/jaqs/example/eventdriven/ctaStrategy.py deleted file mode 100644 index 20b235a..0000000 --- a/jaqs/example/eventdriven/ctaStrategy.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- encoding: utf-8 -*- - -import json - - -from jaqs.util import fileio -from jaqs.trade import model -from jaqs.data.basic.order import Order -from jaqs.trade import common -from jaqs.data.dataservice import RemoteDataService -from jaqs.trade.strategy import EventDrivenStrategy -from jaqs.trade.backtest import EventBacktestInstance -from jaqs.trade.gateway import BacktestTradeApi - - -class CtaStrategy(EventDrivenStrategy): - def __init__(self): - EventDrivenStrategy.__init__(self) - self.symbol = '' - - def init_from_config(self, props): - self.symbol = props.get('symbol') - self.init_balance = props.get('init_balance') - - def on_cycle(self): - pass - - def on_quote(self, quote): - # quote.show() - - time = quote.time - if time == 100000: - order = Order() - order.entrust_action = common.ORDER_ACTION.BUY - order.order_type = common.ORDER_TYPE.LIMIT - order.entrust_date = quote.trade_date - order.entrust_time = quote.time - order.symbol = quote.symbol - order.entrust_size = 10000 - order.entrust_price = quote.close - self.ctx.gateway.send_order(order, '', '') - print 'send order %s: %s %s %f' % (order.entrust_no, order.symbol, order.entrust_action, order.entrust_price) - if time == 140000: - order = Order() - order.entrust_action = common.ORDER_ACTION.SELL - order.order_type = common.ORDER_TYPE.LIMIT - order.entrust_date = quote.trade_date - order.entrust_time = quote.time - order.symbol = quote.symbol - order.entrust_size = 5000 - order.entrust_price = quote.close - self.ctx.gateway.send_order(order, '', '') - print 'send order %s: %s %s %f' % (order.entrust_no, order.symbol, order.entrust_action, order.entrust_price) - - -def test_cta(): - prop_file_path = fileio.join_relative_path("etc/backtest.json") - print prop_file_path - prop_file = open(prop_file_path, 'r') - - props = json.load(prop_file) - - enum_props = {'bar_type': common.QUOTE_TYPE} - for k, v in enum_props.iteritems(): - props[k] = v.to_enum(props[k]) - - strategy = CtaStrategy() - gateway = BacktestTradeApi() - data_service = RemoteDataService() - - context = model.Context() - - backtest = EventBacktestInstance() - backtest.init_from_config(props, strategy, context=context) - - backtest.run() - report = backtest.generate_report(output_format="plot") - - -if __name__ == "__main__": - test_cta() - print "test success." diff --git a/jaqs/example/eventdriven/doubleMaStrategy.py b/jaqs/example/eventdriven/doubleMaStrategy.py deleted file mode 100644 index f0f3ad9..0000000 --- a/jaqs/example/eventdriven/doubleMaStrategy.py +++ /dev/null @@ -1,87 +0,0 @@ -import numpy as np - -from jaqs.trade import common -from jaqs.trade.strategy import EventDrivenStrategy -from jaqs.data.basic.order import Order - - -class DoubleMaStrategy(EventDrivenStrategy): - """""" - def __init__(self): - EventDrivenStrategy.__init__(self) - self.symbol = '' - self.fastN = 3 - self.slowN = 8 - self.bar = None - self.bufferCount = 0 - self.bufferSize = 20 - self.closeArray = np.zeros(self.bufferSize) - self.fastMa = 0 - self.slowMa = 0 - self.pos = 0 - - def init_from_config(self, props): - super(DoubleMaStrategy, self).init_from_config(props) - self.symbol = props.get('symbol') - self.init_balance = props.get('init_balance') - - def on_cycle(self): - pass - - def create_order(self, quote, price, size): - order = Order.new_order(quote.symbol, "", price, size, quote.trade_date, quote.time, - order_type=common.ORDER_TYPE.LIMIT) - return order - - def buy(self, quote, price, size): - order = self.create_order(quote, price, size) - order.entrust_action = common.ORDER_ACTION.BUY - self.ctx.gateway.send_order(order, '', '') - - def sell(self, quote, price, size): - order = self.create_order(quote, price, size) - order.entrust_action = common.ORDER_ACTION.SELL - self.ctx.gateway.send_order(order, '', '') - - def cover(self, quote, price, size): - order = self.create_order(quote, price, size) - order.entrust_action = common.ORDER_ACTION.BUY - self.ctx.gateway.send_order(order, '', '') - - def short(self, quote, price, size): - order = self.create_order(quote, price, size) - order.entrust_action = common.ORDER_ACTION.SELL - self.ctx.gateway.send_order(order, '', '') - - def on_quote(self, quote_dic): - quote = quote_dic.values()[0] - quote_date = quote.trade_date - p = self.ctx.pm.get_position(quote.symbol) - if p is None: - self.pos = 0 - else: - self.pos = p.current_size - - self.closeArray[0:self.bufferSize - 1] = self.closeArray[1:self.bufferSize] - self.closeArray[-1] = quote.close - self.bufferCount += 1 - - if self.bufferCount <= self.bufferSize: - return - self.fastMa = self.closeArray[-self.fastN - 1:-1].mean() - self.slowMa = self.closeArray[-self.slowN - 1:-1].mean() - - if self.fastMa > self.slowMa: - if self.pos == 0: - self.buy(quote, quote.close + 3, 1) - - elif self.pos < 0: - self.cover(quote, quote.close + 3, 1) - self.buy(quote, quote.close + 3, 1) - - elif self.fastMa < self.slowMa: - if self.pos == 0: - self.short(quote, quote.close - 3, 1) - elif self.pos > 0: - self.sell(quote, quote.close - 3, 1) - self.short(quote, quote.close - 3, 1) diff --git a/jaqs/example/eventdriven/run_market_making.py b/jaqs/example/eventdriven/run_market_making.py deleted file mode 100644 index 6166265..0000000 --- a/jaqs/example/eventdriven/run_market_making.py +++ /dev/null @@ -1,101 +0,0 @@ -# encoding: utf-8 - -import time -from jaqs.data.dataservice import RemoteDataService -from jaqs.trade import model, common -from jaqs.trade.realinstance import RealInstance -from jaqs.trade.backtest import EventBacktestInstance -from jaqs.example.eventdriven.market_making import RealStrategy -from jaqs.trade.gateway import RealTimeTradeApi, BacktestTradeApi -from jaqs.trade.portfoliomanager import PortfolioManager -import jaqs.util as jutil -import jaqs.trade.analyze.analyze as ana - -result_dir_path = jutil.join_relative_path('../output/test_consistency') -is_backtest = False - - -def consis(): - - if is_backtest: - props = {"symbol": "rb1710.SHF,hc1710.SHF", - "start_date": 20170510, - "end_date": 20170930, - "bar_type": "1M", # '1d' - "init_balance": 2e4, - "future_commission_rate": 0.00002, - "stock_commission_rate": 0.0001, - "stock_tax_rate": 0.0000} - - tapi = BacktestTradeApi() - ins = EventBacktestInstance() - - else: - props = {'symbol': 'rb1801.SHF,IC1712.CFE'} - tapi = RealTimeTradeApi() - ins = RealInstance() - - tapi.use_strategy(3) - - ds = RemoteDataService() - strat = RealStrategy() - pm = PortfolioManager() - - context = model.Context(data_api=ds, trade_api=tapi, gateway=None, instance=ins, - strategy=strat, pm=pm) - - ins.init_from_config(props) - if not is_backtest: - ds.subscribe(props['symbol']) - - ins.run() - if not is_backtest: - time.sleep(1000) - ins.save_results(folder_path=result_dir_path) - - # order dependent is not good! we should make it not dependent or hard-code the order - # props = {'symbol': '000001.SZ,600001.SH'} - # props = {'symbol': 'CFCICA.JZ,600001.SH'} - - # ds.subscribe('CFCICA.JZ,000001.SZ') - # ds.subscribe('rb1710.SHF') - - -def test_backtest_analyze(): - ta = ana.EventAnalyzer() - - ds = RemoteDataService() - - ta.initialize(data_server_=ds, file_folder=result_dir_path) - - print "process trades..." - ta.process_trades() - print "get daily stats..." - ta.get_daily() - print "calc strategy return..." - ta.get_returns(consider_commission=False) - # position change info is huge! - # print "get position change..." - # ta.get_pos_change_info() - - selected_sec = list(ta.universe)[:2] - if len(selected_sec) > 0: - print "Plot single securities PnL" - for symbol in selected_sec: - df_daily = ta.daily.get(symbol, None) - if df_daily is not None: - ana.plot_trades(df_daily, symbol=symbol, save_folder=result_dir_path) - - print "Plot strategy PnL..." - ta.plot_pnl(result_dir_path) - - print "generate report..." - static_folder = jutil.join_relative_path("trade/analyze/static") - ta.gen_report(source_dir=static_folder, template_fn='report_template.html', - out_folder=result_dir_path, - selected=selected_sec) - - -if __name__ == "__main__": - consis() - test_backtest_analyze() diff --git a/jaqs/research/signaldigger/digger.py b/jaqs/research/signaldigger/digger.py index 2bad948..bc65e94 100644 --- a/jaqs/research/signaldigger/digger.py +++ b/jaqs/research/signaldigger/digger.py @@ -7,7 +7,7 @@ from . import performance as pfm from . import plotting -from jaqs.util import fileio, pdutil +import jaqs.util as jutil class SignalDigger(object): @@ -88,12 +88,12 @@ def process_signal_before_analysis(self, if mask is not None: assert np.all(signal.index == mask.index) assert np.all(signal.columns == mask.columns) - mask = pdutil.fillinf(mask) + mask = jutil.fillinf(mask) mask = mask.astype(int).fillna(0).astype(bool) else: mask = pd.DataFrame(index=signal.index, columns=signal.columns, data=False) - signal = pdutil.fillinf(signal) - data = pdutil.fillinf(data) + signal = jutil.fillinf(signal) + data = jutil.fillinf(data) # ---------------------------------------------------------------------- # save data @@ -132,7 +132,7 @@ def process_signal_before_analysis(self, # calculate quantile signal_masked = signal.copy() signal_masked = signal_masked[~mask] - df_quantile = pdutil.to_quantile(signal_masked, n_quantiles=n_quantiles) + df_quantile = jutil.to_quantile(signal_masked, n_quantiles=n_quantiles) # ---------------------------------------------------------------------- # stack @@ -175,11 +175,11 @@ def show_fig(self, fig, file_name): if self.output_format in ['pdf', 'png', 'jpg']: fp = os.path.join(self.output_folder, '.'.join([file_name, self.output_format])) - fileio.create_dir(fp) + jutil.create_dir(fp) fig.savefig(fp) print "Figure saved: {}".format(fp) elif self.output_format == 'base64': - fig_b64 = fileio.fig2base64(fig, 'png') + fig_b64 = jutil.fig2base64(fig, 'png') self.fig_data[file_name] = fig_b64 print "Base64 data of figure {} will be stored in dictionary.".format(file_name) elif self.output_format == 'plot': diff --git a/jaqs/research/test_signal.py b/jaqs/research/test_signal.py index c7896cc..6ab9252 100644 --- a/jaqs/research/test_signal.py +++ b/jaqs/research/test_signal.py @@ -6,12 +6,13 @@ from jaqs.data.dataview import DataView from jaqs.data.dataservice import RemoteDataService from jaqs.research import signaldigger -from jaqs.util import fileio +import jaqs.util as jutil -dataview_folder = fileio.join_relative_path('../output/prepared', 'test_signal') +dataview_folder = jutil.join_relative_path('../output/prepared', 'test_signal') -def save_dataview(data_folder_name): +def save_dataview(): ds = RemoteDataService() + ds.init_from_config() dv = DataView() props = {'start_date': 20140101, 'end_date': 20171001, 'universe': '000300.SH', @@ -67,7 +68,7 @@ def analyze_signal(): # Step.4 analyze! my_period = 5 - obj = signaldigger.digger.SignalDigger(output_folder=fileio.join_relative_path('../output'), + obj = signaldigger.digger.SignalDigger(output_folder=jutil.join_relative_path('../output'), output_format='pdf') obj.process_signal_before_analysis(signal, price=price, mask=mask_all, @@ -81,5 +82,5 @@ def analyze_signal(): if __name__ == "__main__": - save_dataview() + # save_dataview() analyze_signal() diff --git a/jaqs/trade/analyze/analyze.py b/jaqs/trade/analyze/analyze.py index 1fa4ed5..c8931aa 100644 --- a/jaqs/trade/analyze/analyze.py +++ b/jaqs/trade/analyze/analyze.py @@ -4,9 +4,9 @@ import json from collections import OrderedDict -import matplotlib.pyplot as plt import numpy as np import pandas as pd +import matplotlib.pyplot as plt from matplotlib.ticker import Formatter from jaqs.trade.analyze.report import Report @@ -14,8 +14,7 @@ from jaqs.data.basic.instrument import InstManager import jaqs.util as jutil - -# sys.path.append(os.path.abspath("..")) +STATIC_FOLDER = jutil.join_relative_path("trade/analyze/static") class MyFormatter(Formatter): @@ -57,6 +56,8 @@ def __init__(self): self.inst_map = dict() + self.metrics = dict() + @property def trades(self): """Read-only attribute""" @@ -118,7 +119,7 @@ def _init_inst_data(self): self.inst_map = data_inst.to_dict(orient='index') elif self.data_api is not None: inst_mgr = InstManager(symbol=symbol_str, data_api=self.data_api) - self.inst_map = inst_mgr.inst_map + self.inst_map = {k: v.__dict__ for k, v in inst_mgr.inst_map.items()} else: raise ValueError("no dataview or dataapi provided.") @@ -373,7 +374,32 @@ def gen_report(self, source_dir, template_fn, out_folder='.', selected=None): r.generate_html() r.output_html('report.html') - # r.output_pdf('report.pdf') + + def do_analyze(self, result_dir, selected_sec=None): + if selected_sec is None: + selected_sec = [] + + print "process trades..." + self.process_trades() + print "get daily stats..." + self.get_daily() + print "calc strategy return..." + self.get_returns(consider_commission=False) + + if len(selected_sec) > 0: + print "Plot single securities PnL" + for symbol in selected_sec: + df_daily = self.daily.get(symbol, None) + if df_daily is not None: + plot_trades(df_daily, symbol=symbol, save_folder=result_dir) + + print "Plot strategy PnL..." + self.plot_pnl(result_dir) + + print "generate report..." + self.gen_report(source_dir=STATIC_FOLDER, template_fn='report_template.html', + out_folder=result_dir, + selected=selected_sec) class EventAnalyzer(BaseAnalyzer): @@ -428,6 +454,7 @@ def _to_pct_return(arr, cumulative=False): r[1:] = arr[1:] / arr[:-1] - 1 return r + ''' def get_returns_OLD(self, compound_return=True, consider_commission=True): profit_col_name = 'CumProfitComm' if consider_commission else 'CumProfit' vp_list = {sec: df_profit.loc[:, profit_col_name] for sec, df_profit in self.daily.viewitems()} @@ -462,26 +489,7 @@ def get_returns_OLD(self, compound_return=True, consider_commission=True): # df_returns = df_returns.join(bt_strat_mv, how='right') self.returns = df_returns -def calc_uat_metrics(t1, symbol): - cump1 = t1.loc[:, 'CumProfit'].values - profit1 = cump1[-1] - - n_trades = t1.loc[:, 'CumVolume'].values[-1] / 2. # signle - avg_trade = profit1 / n_trades - print "profit without commission = {} \nprofit with commission {}".format(profit1, profit1) - print "avg_trade = {:.3f}".format(avg_trade) - - fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=(16, 8)) - ax1.plot(cump1, label='inst1') - ax1.set_title("{} PnL in price".format(symbol)) - ax1.legend(loc='upper left') - ax1.axhline(0, color='k', lw=1, ls='--') - ax2.plot(t1.loc[:, 'position'].values) - ax2.set_title("Position") - ax2.axhline(0, color='k', lw=1, ls='--') - - plt.show() - return + ''' def plot_trades(df, symbol="", save_folder="."): @@ -491,8 +499,8 @@ def plot_trades(df, symbol="", save_folder="."): bv, sv = df.loc[:, 'BuyVolume'].values, df.loc[:, 'SellVolume'].values profit = df.loc[:, 'CumProfit'].values avgpx = df.loc[:, 'AvgPosPrice'] - bv *= .1 - sv *= .1 + bv *= .05 + sv *= .05 fig = plt.figure(figsize=(14, 10), dpi=300) ax1 = plt.subplot2grid((4, 1), (0, 0), rowspan=3) diff --git a/jaqs/trade/analyze/report.py b/jaqs/trade/analyze/report.py index 6e2d14f..425b636 100644 --- a/jaqs/trade/analyze/report.py +++ b/jaqs/trade/analyze/report.py @@ -2,7 +2,7 @@ import os import jinja2 -from jaqs.util import fileio +import jaqs.util as jutil # from weasyprint import HTML @@ -46,7 +46,7 @@ def generate_html(self): def output_html(self, fn='test_out.html'): path = os.path.abspath(os.path.join(self.out_folder, fn)) - fileio.create_dir(path) + jutil.create_dir(path) with open(path, 'w') as f: f.write(self.html) diff --git a/jaqs/trade/backtest.py b/jaqs/trade/backtest.py index 2d3fb71..e118e6a 100644 --- a/jaqs/trade/backtest.py +++ b/jaqs/trade/backtest.py @@ -1,21 +1,15 @@ # encoding: utf-8 -import os - import numpy as np import pandas as pd from jaqs.trade import common -from jaqs.trade.analyze.pnlreport import PnlManager -from jaqs.trade.pubsub import Subscriber from jaqs.data.basic.marketdata import Bar from jaqs.data.basic.trade import Trade -from jaqs.util import dtutil -from jaqs.util import fileio import jaqs.util as jutil -class BacktestInstance(Subscriber): +class BacktestInstance(object): """ Attributes ---------- @@ -34,8 +28,6 @@ def __init__(self): self.ctx = None - self.commission_rate = 0.0 - def init_from_config(self, props): """ @@ -64,10 +56,6 @@ def init_from_config(self, props): if obj is not None: obj.init_from_config(props) - def calc_commission(self, trade_ind): - to = abs(trade_ind.fill_price * trade_ind.fill_size) - res = to * self.commission_rate - return res ''' @@ -88,7 +76,7 @@ def _is_trade_date(self, start, end, date, data_server): def go_next_rebalance_day(self): """update self.ctx.trade_date and last_date.""" if self.ctx.gateway.match_finished: - next_period_day = dtutil.get_next_period_day(self.ctx.trade_date, + next_period_day = jutil.get_next_period_day(self.ctx.trade_date, self.ctx.strategy.period, self.ctx.strategy.days_delay) # update current_date: next_period_day is a workday, but not necessarily a trade date if self.ctx.calendar.is_trade_date(next_period_day): @@ -344,7 +332,7 @@ def re_balance_plan_after_open(self): self.ctx.strategy.on_after_rebalance(cash_available + market_value_frozen) def run_alpha(self): - gateway = self.ctx.trade_api + tapi = self.ctx.trade_api self.ctx.trade_date = self.start_date while True: @@ -355,7 +343,7 @@ def run_alpha(self): print "\n=======new day {}".format(self.ctx.trade_date) # match uncome orders or re-balance - if gateway.match_finished: + if tapi.match_finished: # Step1. # position adjust according to dividend, cash paid, de-list actions during the last period # two adjust must in order @@ -379,7 +367,7 @@ def run_alpha(self): # Deal with trade indications # results = gateway.match(self.univ_price_dic) - results = gateway.match_and_callback(self.univ_price_dic) + results = tapi.match_and_callback(self.univ_price_dic) for trade_ind, order_status_ind in results: self.ctx.strategy.cash -= trade_ind.commission @@ -406,7 +394,7 @@ def go_next_rebalance_day(self): """update self.ctx.trade_date and last_date.""" current_date = self.ctx.trade_date if self.ctx.trade_api.match_finished: - next_period_day = dtutil.get_next_period_day(current_date, self.ctx.strategy.period, + next_period_day = jutil.get_next_period_day(current_date, self.ctx.strategy.period, n=self.ctx.strategy.n_periods, extra_offset=self.ctx.strategy.days_delay) # update current_date: next_period_day is a workday, but not necessarily a trade date @@ -478,10 +466,10 @@ def save_results(self, folder_path='.'): trades_fn = os.path.join(folder_path, 'trades.csv') configs_fn = os.path.join(folder_path, 'configs.json') - fileio.create_dir(trades_fn) + jutil.create_dir(trades_fn) df_trades.to_csv(trades_fn) - fileio.save_json(self.props, configs_fn) + jutil.save_json(self.props, configs_fn) print ("Backtest results has been successfully saved to:\n" + folder_path) @@ -501,25 +489,13 @@ class EventBacktestInstance(BacktestInstance): def __init__(self): super(EventBacktestInstance, self).__init__() - # self.pnlmgr = None - self.bar_type = 1 + self.bar_type = "" - self.commission_rate = 1E-4 - def init_from_config(self, props): super(EventBacktestInstance, self).init_from_config(props) - # TODO should be consistent with tradeAPI - # self.ctx.gateway.register_callback('portfolio manager', strategy.pm) + self.bar_type = props.get("bar_type", "1d") - self.bar_type = props.get("bar_type") - - self.commission_rate = props.get('commission_rate', 20E-4) - - # self.pnlmgr = PnlManager() - # self.pnlmgr.setStrategy(strategy) - # self.pnlmgr.initFromConfig(props, self.ctx.data_api) - def go_next_trade_date(self): next_dt = self.ctx.calendar.get_next_trade_date(self.ctx.trade_date) @@ -694,10 +670,10 @@ def save_results(self, folder_path='.'): trades_fn = os.path.join(folder_path, 'trades.csv') configs_fn = os.path.join(folder_path, 'configs.json') - fileio.create_dir(trades_fn) + jutil.create_dir(trades_fn) df_trades.to_csv(trades_fn) - fileio.save_json(self.props, configs_fn) + jutil.save_json(self.props, configs_fn) print ("Backtest results has been successfully saved to:\n" + folder_path) diff --git a/jaqs/trade/model.py b/jaqs/trade/model.py index 32a02f4..ea68942 100644 --- a/jaqs/trade/model.py +++ b/jaqs/trade/model.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd from jaqs.data.dataservice import Calendar -from jaqs.util import fileio +import jaqs.util as jutil class RegisteredFunction(object): @@ -71,10 +71,10 @@ def __init__(self, data_api=None, trade_api=None, gateway=None, obj.ctx = self def save_store(self, path): - fileio.save_pickle(self.storage, path) + jutil.save_pickle(self.storage, path) def load_store(self, path): - s = fileio.load_pickle(path) + s = jutil.load_pickle(path) if s is not None: self.storage = s diff --git a/jaqs/trade/portfoliomanager.py b/jaqs/trade/portfoliomanager.py index 4900d2c..5616fc1 100644 --- a/jaqs/trade/portfoliomanager.py +++ b/jaqs/trade/portfoliomanager.py @@ -136,6 +136,14 @@ def get_position(self, symbol): pos_key = self._make_position_key(symbol) position = self.positions.get(pos_key, None) return position + + def get_pos(self, symbol): + pos_key = self._make_position_key(symbol) + position = self.positions.get(pos_key, None) + if position is None: + return 0 + else: + return position.current_size def get_task(self, task_id): """ diff --git a/jaqs/trade/pubsub.py b/jaqs/trade/pubsub.py deleted file mode 100644 index 689d645..0000000 --- a/jaqs/trade/pubsub.py +++ /dev/null @@ -1,66 +0,0 @@ -# encoding: UTF-8 - -from abc import abstractmethod -from collections import defaultdict - - -class EventTemp(object): - def __init__(self, topic="unknown", data=None): - self.data = data - self.topic = topic - - -class Publisher(object): - def __init__(self): - self.subscribers = defaultdict(list) # use empty list as default-factory - - def add_subscriber(self, subscriber, topic): - self.subscribers[topic].append(subscriber) - - def publish(self, event): - """ - Publish an event to all its subscribers. - - Parameters - ---------- - event : Event object - - Returns - ------- - - """ - sub_list = self.subscribers.get(event.topic, None) - if sub_list is None: - return - - for subscriber in sub_list: - subscriber.on_event(event) - - def get_topics(self): - return self.subscribers.keys() - - -class Subscriber(object): - def __init__(self): - pass - - @abstractmethod - def subscribe(self, publisher, topic): - pass - - @abstractmethod - def on_event(self, event): - """ - Process an event. - - Parameters - ---------- - event : Event object - - """ - pass - - -if __name__ == "__main__": - p = Publisher() - p.add_subscriber(lambda x: x, 'topic1') diff --git a/jaqs/trade/realinstance.py b/jaqs/trade/realtime.py similarity index 64% rename from jaqs/trade/realinstance.py rename to jaqs/trade/realtime.py index 687551c..4d84c4b 100644 --- a/jaqs/trade/realinstance.py +++ b/jaqs/trade/realtime.py @@ -2,9 +2,10 @@ from jaqs.trade.event import EventEngine, Event, EVENT_TYPE from jaqs.data.basic import Quote +import jaqs.util as jutil -class RealInstance(EventEngine): +class EventRealTimeInstance(EventEngine): """ Attributes ---------- @@ -13,7 +14,7 @@ class RealInstance(EventEngine): """ def __init__(self): - super(RealInstance, self).__init__() + super(EventRealTimeInstance, self).__init__() self.start_date = 0 self.end_date = 0 @@ -92,18 +93,40 @@ def on_order_status(self, event): def on_task_status(self, event): ind = event.dic['ind'] self.ctx.strategy.on_task_status(ind) - - - - - - - - - - - - - - + # --------------------------------------------------------- + # Save Results + def save_results(self, folder_path='.'): + import os + import pandas as pd + folder_path = os.path.abspath(folder_path) + + trades = self.ctx.pm.trades + + type_map = {'task_id': str, + 'entrust_no': str, + 'entrust_action': str, + 'symbol': str, + 'fill_price': float, + 'fill_size': float, + 'fill_date': int, + 'fill_time': int, + 'fill_no': str, + 'commission': float} + # keys = trades[0].__dict__.keys() + ser_list = dict() + for key in type_map.keys(): + v = [t.__getattribute__(key) for t in trades] + ser = pd.Series(data=v, index=None, dtype=type_map[key], name=key) + ser_list[key] = ser + df_trades = pd.DataFrame(ser_list) + df_trades.index.name = 'index' + + trades_fn = os.path.join(folder_path, 'trades.csv') + configs_fn = os.path.join(folder_path, 'configs.json') + jutil.create_dir(trades_fn) + + df_trades.to_csv(trades_fn) + jutil.save_json(self.props, configs_fn) + + print ("Backtest results has been successfully saved to:\n" + folder_path) diff --git a/jaqs/trade/strategy.py b/jaqs/trade/strategy.py index 620d32d..27ad1a4 100644 --- a/jaqs/trade/strategy.py +++ b/jaqs/trade/strategy.py @@ -3,19 +3,15 @@ import abc from abc import abstractmethod from six import with_metaclass -from collections import defaultdict import numpy as np -from jaqs.trade.portfoliomanager import PortfolioManager -from jaqs.data.basic.order import * from jaqs.data.basic.position import GoalPosition from jaqs.util.sequence import SequenceGenerator -from jaqs.util import fileio +# import jaqs.util as jutil from jaqs.trade import model from jaqs.trade import common -from jaqs.trade.event import EVENT_TYPE, Event class Strategy(with_metaclass(abc.ABCMeta)): @@ -61,6 +57,7 @@ def _get_next_num(self, key): """used to generate id for orders and trades.""" return str(np.int64(self.ctx.trade_date) * 10000 + self.seq_gen.get_next(key)) + ''' # ------------------------------------------------------------------------------------------- # Order @@ -91,7 +88,7 @@ def place_order(self, symbol, action, price, size, algo="", algo_param=None): """ pass - + def cancel_order(self, task_id): """Cancel all uncome orders of a task according to its task ID. @@ -108,24 +105,11 @@ def cancel_order(self, task_id): err_msg : str """ - ''' - entrust_no_list = self.task_id_map.get(task_id, None) - if entrust_no_list is None: - return False, "No task id {}".format(task_id) - - err_msgs = [] - for entrust_no in entrust_no_list: - err_msg = self.ctx.gateway.cancel_order(entrust_no) - err_msgs.append(err_msg) - if any(map(lambda s: bool(s), err_msgs)): - return False, ','.join(err_msgs) - else: - return True, "" - ''' + pass # ------------------------------------------------------------------------------------------- # Query - + def query_account(self): """ @@ -135,7 +119,7 @@ def query_account(self): """ pass - + def query_universe(self): """ @@ -145,7 +129,7 @@ def query_universe(self): """ pass - + def query_position(self, mode="all", symbols=""): """ Parameters @@ -160,7 +144,7 @@ def query_position(self, mode="all", symbols=""): """ pass - + def query_portfolio(self): """ Return net positions of all securities in the strategy universe (including zero positions). @@ -187,6 +171,7 @@ def query_task(self, task_id=-1): pd.DataFrame """ + def query_order(self, task_id=-1): """ Query order information of current day. @@ -256,7 +241,7 @@ def stop_portfolio(self): """ pass - + def place_batch_order(self, orders, algo="", algo_param=None): """Send a batch of orders to the system together. @@ -295,6 +280,7 @@ def basket_order(self, orders, algo="", algo_param=None): """ pass + ''' # ------------------------------------------------------------------------------------------- # Callback Indications & Responses @@ -723,9 +709,29 @@ def __init__(self): def on_quote(self, quote): pass - @abstractmethod + def on_tick(self, quote): + pass + def on_cycle(self): pass def initialize(self): pass + + def cancel_all_orders(self): + for task_id, task in self.ctx.pm.tasks.items(): + if not task.is_finished: + self.ctx.trade_api.cancel_order(task_id) + + def liquidate(self, quote, n, tick_size=1.0, pos=0): + if pos == 0: + return + + ref_price = quote.close + if pos < 0: + action = common.ORDER_ACTION.BUY + price = ref_price + n * tick_size + else: + action = common.ORDER_ACTION.SELL + price = ref_price - n * tick_size + self.ctx.trade_api.place_order(quote.symbol, action, price, abs(pos)) diff --git a/jaqs/trade/gateway.py b/jaqs/trade/tradegateway.py similarity index 99% rename from jaqs/trade/gateway.py rename to jaqs/trade/tradegateway.py index d0da77a..93c5dd4 100644 --- a/jaqs/trade/gateway.py +++ b/jaqs/trade/tradegateway.py @@ -375,7 +375,7 @@ def get_from_list_of_dict(l, key, default=None): } # ------------------------------------------------------------------------------------------- - # On TradeAPI Callback: put a corresponding event to RealInstance + # On TradeAPI Callback: put a corresponding event to EventRealTimeInstance def set_trade_api_callbacks(self, trade_api): trade_api.set_task_status_callback(self.on_task_status) @@ -741,8 +741,8 @@ def on_task_status(self, ind_dic): self.ctx.strategy.on_task_status(ind) @staticmethod - def _check_task_id(task_id): - return not (task_id is None) or (task_id == 0) + def _is_failed_task(task_id): + return (task_id is None) or (task_id == 0) def place_order(self, symbol, action, price, size, algo="", algo_param={}, userdata=""): # Generate Task @@ -750,7 +750,7 @@ def place_order(self, symbol, action, price, size, algo="", algo_param={}, userd order_type=common.ORDER_TYPE.LIMIT) task_id, msg = super(RealTimeTradeApi, self).place_order(symbol, action, price, size, algo, algo_param, userdata) - if not self._check_task_id(task_id): + if self._is_failed_task(task_id): return task_id, msg task = Task(task_id, @@ -765,9 +765,9 @@ def place_order(self, symbol, action, price, size, algo="", algo_param={}, userd # --------------------------------------------- # For Alpha Strategy -class DailyStockSimGateway(BaseTradeApi): +class AlphaTradeApi(BaseTradeApi): def __init__(self): - super(DailyStockSimGateway, self).__init__() + super(AlphaTradeApi, self).__init__() self.ctx = None self._simulator = DailyStockSimulator() diff --git a/jaqs/util/md2rst.py b/jaqs/util/md2rst.py index 221abd5..4c7ea61 100644 --- a/jaqs/util/md2rst.py +++ b/jaqs/util/md2rst.py @@ -2,13 +2,13 @@ import os from os.path import join -from jaqs.util import fileio +import jaqs.util as jutil import subprocess def md2rst(): - input_dir = fileio.join_relative_path('../doc') - output_dir = fileio.join_relative_path('../doc/source') + input_dir = jutil.join_relative_path('../doc') + output_dir = jutil.join_relative_path('../doc/source') for dir_path, dir_names, file_names in os.walk(input_dir): for fn in file_names: diff --git a/setup.py b/setup.py index 58f2fe1..6cea51d 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def readme(): setup( name='jaqs', - version='0.2.0', + version='0.4.0', description='Open source quantitative research&trading framework.', long_description = readme(), install_requires=[ diff --git a/test_double_ma.py b/test_double_ma.py index 442fc64..0dfd65a 100644 --- a/test_double_ma.py +++ b/test_double_ma.py @@ -3,17 +3,17 @@ import json -from jaqs.util import fileio +import jaqs.util as jutil from jaqs.trade import model from jaqs.trade import common from jaqs.data.dataservice import RemoteDataService -from jaqs.example.eventdriven.doubleMaStrategy import DoubleMaStrategy +from jaqs.example.eventdriven.ma import DoubleMaStrategy from jaqs.trade.backtest import EventBacktestInstance -from jaqs.trade.gateway import BacktestTradeApi +from jaqs.trade.tradegateway import BacktestTradeApi def test_double_ma(): - prop_file_path = fileio.join_relative_path("etc/backtest.json") + prop_file_path = jutil.join_relative_path("etc/backtest.json") print prop_file_path prop_file = open(prop_file_path, 'r') @@ -28,7 +28,7 @@ def test_double_ma(): gateway = BacktestTradeApi() data_service = RemoteDataService() - context = model.Context(data_api=data_service, gateway=gateway) + context = model.Context(data_api=data_service) backtest = EventBacktestInstance() backtest.init_from_config(props, strategy, context=context) diff --git a/tests/test_align.py b/tests/test_align.py index 98bc29a..08f769e 100644 --- a/tests/test_align.py +++ b/tests/test_align.py @@ -9,6 +9,7 @@ def test_align(): # ------------------------------------------------------------------------------------- # input and pre-process demo data ds = RemoteDataService() + ds.init_from_config() raw, msg = ds.query_lb_fin_stat('income', '600000.SH', 20151225, 20170501, 'oper_rev') assert msg == '0,' diff --git a/tests/test_backtest.py b/tests/test_backtest.py index 61a13c4..9f896f2 100644 --- a/tests/test_backtest.py +++ b/tests/test_backtest.py @@ -22,20 +22,21 @@ from jaqs.data.dataservice import RemoteDataService from jaqs.trade.strategy import AlphaStrategy -from jaqs.util import fileio +import jaqs.util as jutil import jaqs.trade.analyze.analyze as ana from jaqs.trade.backtest import AlphaBacktestInstance from jaqs.trade.portfoliomanager import PortfolioManager -from jaqs.trade.gateway import DailyStockSimGateway +from jaqs.trade.tradegateway import AlphaTradeApi from jaqs.trade import model from jaqs.data.dataview import DataView -dataview_dir_path = fileio.join_relative_path('../output/prepared/test_backtest') -backtest_result_dir_path = fileio.join_relative_path('../output/test_backtest') +dataview_dir_path = jutil.join_relative_path('../output/prepared/test_backtest') +backtest_result_dir_path = jutil.join_relative_path('../output/test_backtest') def save_dataview(): ds = RemoteDataService() + ds.init_from_config() dv = DataView() props = {'start_date': 20170101, 'end_date': 20171030, 'universe': '000300.SH', @@ -101,7 +102,7 @@ def test_alpha_strategy_dataview(): 'commission_rate': 0.0 } - gateway = DailyStockSimGateway() + trade_api = AlphaTradeApi() bt = AlphaBacktestInstance() risk_model = model.FactorRiskModel() @@ -122,7 +123,7 @@ def test_alpha_strategy_dataview(): # strategy = AlphaStrategy(stock_selector=stock_selector, pc_method='market_value_weight') # strategy = AlphaStrategy() - context = model.AlphaContext(dataview=dv, gateway=gateway, trade_api=gateway, + context = model.AlphaContext(dataview=dv, trade_api=trade_api, instance=bt, strategy=strategy, pm=pm) for mdl in [risk_model, signal_model, cost_model, stock_selector]: mdl.register_context(context) @@ -164,7 +165,7 @@ def test_backtest_analyze(): ta.plot_pnl(backtest_result_dir_path) print "generate report..." - static_folder = fileio.join_relative_path("trade/analyze/static") + static_folder = jutil.join_relative_path("trade/analyze/static") ta.gen_report(source_dir=static_folder, template_fn='report_template.html', out_folder=backtest_result_dir_path, selected=selected_sec) @@ -173,7 +174,7 @@ def test_backtest_analyze(): if __name__ == "__main__": t_start = time.time() - # test_alpha_strategy_dataview() + test_alpha_strategy_dataview() test_backtest_analyze() t3 = time.time() - t_start diff --git a/tests/test_calendar.py b/tests/test_calendar.py index 19f6337..7bd5993 100644 --- a/tests/test_calendar.py +++ b/tests/test_calendar.py @@ -2,7 +2,7 @@ import datetime -from jaqs.data.calendar import Calendar +from jaqs.data.dataservice import Calendar from jaqs.util import dtutil diff --git a/tests/test_data_api.py b/tests/test_data_api.py index 91409de..0f270bf 100644 --- a/tests/test_data_api.py +++ b/tests/test_data_api.py @@ -1,12 +1,12 @@ # encoding: UTF-8 -from jaqs.util import fileio +import jaqs.util as jutil from jaqs.data.dataapi import DataApi from jaqs.trade import common def test_data_api(): - dic = fileio.read_json(fileio.join_relative_path('etc/data_config.json')) + dic = jutil.read_json(jutil.join_relative_path('etc/data_config.json')) address = dic.get("remote.address", None) username = dic.get("remote.username", None) password = dic.get("remote.password", None) diff --git a/tests/test_dataservice.py b/tests/test_dataservice.py index e3e46db..8274cea 100644 --- a/tests/test_dataservice.py +++ b/tests/test_dataservice.py @@ -6,6 +6,7 @@ def test_remote_data_service_daily(): ds = RemoteDataService() + ds.init_from_config() # test daily res, msg = ds.daily('rb1710.SHF,600662.SH', fields="", @@ -24,6 +25,7 @@ def test_remote_data_service_daily(): def test_remote_data_service_daily_quited(): ds = RemoteDataService() + ds.init_from_config() # test daily res, msg = ds.daily('600832.SH', fields="", @@ -35,6 +37,7 @@ def test_remote_data_service_daily_quited(): def test_remote_data_service_bar(): ds = RemoteDataService() + ds.init_from_config() # test bar res2, msg2 = ds.bar('rb1710.SHF,600662.SH', start_time=200000, end_time=160000, trade_date=20170831, fields="") @@ -53,6 +56,7 @@ def test_remote_data_service_bar(): def test_remote_data_service_lb(): ds = RemoteDataService() + ds.init_from_config() # test lb.secDailyIndicator fields = "pb,pe,free_share,net_assets,limit_status" @@ -78,6 +82,7 @@ def test_remote_data_service_lb(): def test_remote_data_service_daily_ind_performance(): ds = RemoteDataService() + ds.init_from_config() hs300 = ds.get_index_comp('000300.SH', 20140101, 20170101) hs300_str = ','.join(hs300) @@ -92,6 +97,7 @@ def test_remote_data_service_daily_ind_performance(): def test_remote_data_service_components(): ds = RemoteDataService() + ds.init_from_config() res = ds.get_index_comp_df(index='000300.SH', start_date=20140101, end_date=20170505) assert res.shape == (814, 430) @@ -104,6 +110,7 @@ def test_remote_data_service_industry(): import pandas as pd ds = RemoteDataService() + ds.init_from_config() arr = ds.get_index_comp(index='000300.SH', start_date=20130101, end_date=20170505) df = ds.get_industry_raw(symbol=','.join(arr), type_='ZZ') df = df.astype(dtype={'in_date': int}) @@ -139,6 +146,7 @@ def test_remote_data_service_industry_df(): cal = Calendar() ds = RemoteDataService() + ds.init_from_config() arr = ds.get_index_comp(index='000300.SH', start_date=20130101, end_date=20170505) symbol_arr = ','.join(arr) @@ -161,6 +169,7 @@ def test_remote_data_service_industry_df(): def test_remote_data_service_fin_indicator(): ds = RemoteDataService() + ds.init_from_config() symbol = '000008.SZ' filter_argument = ds._dic2url({'symbol': symbol}) @@ -171,6 +180,7 @@ def test_remote_data_service_fin_indicator(): def test_remote_data_service_adj_factor(): ds = RemoteDataService() + ds.init_from_config() arr = ds.get_index_comp(index='000300.SH', start_date=20130101, end_date=20170505) symbol_arr = ','.join(arr) @@ -182,6 +192,7 @@ def test_remote_data_service_adj_factor(): def test_remote_data_service_inst_info(): ds = RemoteDataService() + ds.init_from_config() sec = '000001.SZ' res = ds.query_inst_info(sec, fields='status,selllot,buylot,pricetick,multiplier,product') @@ -192,6 +203,7 @@ def test_remote_data_service_inst_info(): def test_remote_data_service_index_weight(): ds = RemoteDataService() + ds.init_from_config() df = ds.get_index_weights(index='000300.SH', trade_date=20140101) assert df.shape[0] == 300 assert abs(df['weight'].sum() - 100) < 1.0 diff --git a/tests/test_dataview.py b/tests/test_dataview.py index 8e363cc..a8e15c3 100644 --- a/tests/test_dataview.py +++ b/tests/test_dataview.py @@ -1,10 +1,10 @@ # encoding: utf-8 from jaqs.data.dataview import DataView -from jaqs.util import fileio +import jaqs.util as jutil -daily_path = fileio.join_relative_path('../output/tests/test_dataview_d') -quarterly_path = fileio.join_relative_path('../output/tests/test_dataview_q') +daily_path = jutil.join_relative_path('../output/tests/test_dataview_d') +quarterly_path = jutil.join_relative_path('../output/tests/test_dataview_q') def test_write(): diff --git a/tests/test_report.py b/tests/test_report.py index 5b3e35f..9df269c 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,10 +1,10 @@ # encoding: utf-8 from jaqs.trade.analyze.report import Report -from jaqs.util import fileio +import jaqs.util as jutil def test_output(): - static_folder = fileio.join_relative_path('trade/analyze/static') + static_folder = jutil.join_relative_path('trade/analyze/static') r = Report({'mytitle': 'Test Title', 'mytable': 'Hello World!'}, source_dir=static_folder, diff --git a/tests/test_spread_trading.py b/tests/test_spread_trading.py deleted file mode 100644 index faf7577..0000000 --- a/tests/test_spread_trading.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- encoding: utf-8 -*- - -import json - - -from jaqs.util import fileio -from jaqs.trade import model -from jaqs.trade import common -import jaqs.trade.analyze.analyze as ana -from jaqs.data.dataservice import RemoteDataService -from jaqs.example.eventdriven.spread import SpreadStrategy -from jaqs.trade.backtest import EventBacktestInstance -from jaqs.trade.gateway import BacktestTradeApi - -backtest_result_dir_path = fileio.join_relative_path('../output/event_driven') - - -def test_spread_trading(): - props = {"symbol": "rb1710.SHF,hc1710.SHF", - "start_date": 20170510, - "end_date": 20170930, - "bar_type": "MIN", - "init_balance": 2e4, - "future_commission_rate": 0.00002, - "stock_commission_rate": 0.0001, - "stock_tax_rate": 0.0000} - - # props['bar_type'] = 'DAILY' - - enum_props = {'bar_type': common.QUOTE_TYPE} - for k, v in enum_props.iteritems(): - props[k] = v.to_enum(props[k]) - - bt_tapi = BacktestTradeApi() - data_service = RemoteDataService() - - strategy = SpreadStrategy() - from jaqs.trade.portfoliomanager import PortfolioManager - pm = PortfolioManager(strategy=strategy) - - bt = EventBacktestInstance() - - context = model.Context(data_api=data_service, trade_api=bt_tapi, gateway=bt_tapi, instance=bt) - context.pm = pm - - bt.init_from_config(props, strategy, context=context) - - bt.run() - - bt.save_results(folder_path=backtest_result_dir_path) - - # report = bt.generate_report(output_format="plot") - # print report.trades[:100] - # for pnl in report.daily_pnls: - # print pnl.date, pnl.trade_pnl, pnl.hold_pnl,pnl.total_pnl, pnl.positions.get('600030.SH') - - -def test_backtest_analyze(): - ta = ana.EventAnalyzer() - - ds = RemoteDataService() - - ta.initialize(data_server_=ds, file_folder=backtest_result_dir_path) - - print "process trades..." - ta.process_trades() - print "get daily stats..." - ta.get_daily() - print "calc strategy return..." - ta.get_returns(consider_commission=False) - # position change info is huge! - # print "get position change..." - # ta.get_pos_change_info() - - selected_sec = list(ta.universe)[:2] - if len(selected_sec) > 0: - print "Plot single securities PnL" - for symbol in selected_sec: - df_daily = ta.daily.get(symbol, None) - if df_daily is not None: - ana.plot_trades(df_daily, symbol=symbol, save_folder=backtest_result_dir_path) - - print "Plot strategy PnL..." - ta.plot_pnl(backtest_result_dir_path) - - print "generate report..." - static_folder = fileio.join_relative_path("trade/analyze/static") - ta.gen_report(source_dir=static_folder, template_fn='report_template.html', - out_folder=backtest_result_dir_path, - selected=selected_sec) - - -if __name__ == "__main__": - test_spread_trading() - test_backtest_analyze() - print "test success." diff --git a/tests/test_strategy.py b/tests/test_strategy.py index 473728b..a342ae4 100644 --- a/tests/test_strategy.py +++ b/tests/test_strategy.py @@ -1,13 +1,13 @@ # encoding: utf-8 from jaqs.trade import model -from jaqs.util import fileio +import jaqs.util as jutil import random def test_context(): r = random.random() - path = fileio.join_relative_path('../output/tests/storage{:.6f}.pic'.format(r)) + path = jutil.join_relative_path('../output/tests/storage{:.6f}.pic'.format(r)) context = model.Context() context.load_store(path) assert len(context.storage) == 0 diff --git a/tests/test_trade_api.py b/tests/test_trade_api.py index 7cac1d4..11297bd 100644 --- a/tests/test_trade_api.py +++ b/tests/test_trade_api.py @@ -1,11 +1,11 @@ # encoding: UTF-8 -from jaqs.util import fileio +import jaqs.util as jutil from jaqs.trade.tradeapi import TradeApi import pandas as pd def test_trade_api(): - dic = fileio.read_json(fileio.join_relative_path('etc/trade_config.json')) + dic = jutil.read_json(jutil.join_relative_path('etc/trade_config.json')) address = dic.get("remote.address", None) username = dic.get("remote.username", None) password = dic.get("remote.password", None) diff --git a/tests/test_util.py b/tests/test_util.py index 3fd49e3..14e2c01 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,17 +1,17 @@ # encoding: utf-8 -from jaqs.util import fileio +import jaqs.util as jutil def test_read_save_pickle(): - fp = fileio.join_relative_path('../output/tests/test_read_save_pickle.pic') + fp = jutil.join_relative_path('../output/tests/test_read_save_pickle.pic') d = {'a': 1.0, 'b': 2, 'c': True, 'd': list()} - fileio.save_pickle(d, fp) + jutil.save_pickle(d, fp) - d2 = fileio.load_pickle(fp) + d2 = jutil.load_pickle(fp) assert d2['b'] == 2 - d3 = fileio.load_pickle('a_non_exits_file_blabla.pic') + d3 = jutil.load_pickle('a_non_exits_file_blabla.pic') assert d3 is None