Skip to content

Commit

Permalink
class_08
Browse files Browse the repository at this point in the history
  • Loading branch information
lin committed Nov 22, 2020
1 parent c17ca5e commit 7ada212
Show file tree
Hide file tree
Showing 5 changed files with 413 additions and 9 deletions.
18 changes: 18 additions & 0 deletions class_07/class_07.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
from howtrader.gateway.binance.binance_gateway import BinanceGateway

from howtrader.gateway.binances.binances_gateway import BinancesGateway
from howtrader.event.engine import EventEngine, Event, EVENT_TIMER

from howtrader.trader.constant import Status, OrderType
from howtrader.trader.constant import Exchange, Interval
# from howtrader.trader.object import Exchange, Interval
from howtrader.trader.object import TickData



# Exchange.BINANCE
# Interval.MINUTE

tick = TickData()
# tick.ask_volume_1
# tick.bid_price_1
9 changes: 0 additions & 9 deletions class_07/class_7.md
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,3 @@
#第七课: howtrader的常见类和配置

## 数据库


## 配置文件


## 技术指标库


##
168 changes: 168 additions & 0 deletions class_08/class_08.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
# 第八课: VNPY数据库配置和数据爬取


## 币安邀请码
没有开通币安或者交易所的可以通过下面链接去开通，有手续费减免:
**https://www.binancezh.pro/cn/futures/ref/51bitquant**
合约邀请码：**51bitquant**


## 答疑: 安装vnpy的错误
1. MacOS系统，要记得安装Xcode开发工具，主要是Xcode会帮你把一些构建工具和库安装好。
2. 安装通过pip install git+https://github.com/ramoslin02/howtrader.git
3. 如果没有安装git工具的，记得去安装，具体百度下。
4. 如果出现一些get_localzone的问题，是因为系统时区的问题。具体可以百度学习下。

## 支持的数据库类型
1. 支持sqlite3数据库(默认使用)
2. mongodb数据库
3. mysql

## 配置文件

1. mongodb 需要安装 先启动数据库
> mongod --dbpath /Users/wanglin/mongodb/data/db(替换成你的路径, replace
> with your own database path)

配置文件的详细信息请参考: howtrader/trader/setting.py文件
```python

SETTINGS: Dict[str, Any] = {
"font.family": "Arial",
"font.size": 12,

"log.active": True,
"log.level": CRITICAL,
"log.console": True,
"log.file": True,

"email.server": "smtp.qq.com",
"email.port": 465,
"email.username": "",
"email.password": "",
"email.sender": "",
"email.receiver": "",

"database.timezone": get_localzone().zone,
"database.driver": "sqlite", # see database.Driver
"database.database": "database.db", # for sqlite, use this as filepath
"database.host": "localhost",
"database.port": 3306,
"database.user": "root",
"database.password": "",
"database.authentication_source": "admin", # for mongodb
}

```

``` json
{
"font.family": "Arial",
"font.size": 12,

"log.active": true,
"log.level": "CRITICAL",
"log.console": true,
"log.file": true,

"email.server": "smtp.qq.com",
"email.port": 465,
"email.username": "",
"email.password": "",
"email.sender": "",
"email.receiver": "",

"database.driver": "mongodb",
"database.database": "howtrader",
"database.host": "localhost",
"database.port": 3306,
"database.user": "root",
"database.password": "",
"database.authentication_source": "admin"
}


```
1. database.driver : 填写的值有: sqlite, mysql, mongodb
2. database.database: 数据库名称,
howtrader,随便填写，如果默认用sqlite.db就不用修改。
3. host 主机, user用户， passwordm密码

默认的sqlite的配置
```json

{
"database.driver": "sqlite",
"database.database": "database.db"
}


```
使用mongodb的配置
``` json
{
"database.driver": "mongodb",
"database.database": "howtrader",
"database.host": "localhost",
"database.port": 27017,
"database.authentication_source": "admin"
}

```




可能的报错：
```
pymongo.errors.ServerSelectionTimeoutError: Got response
id 3158574 but expected 996497972, Timeout: 30s, Topology Description:
<TopologyDescription id: 5fb9f86c773b82991c1df7fb, topology_type:
Single, servers:
[<ServerDescription ('localhost', 3306) server_type: Unknown, rtt: None, error=ProtocolError('Got response id 3158574 but expected 996497972')>]>
```


## 关于连接交易所的配置

1. 现货

```json
{
"key": "xxxxxx",
"secret": "xxx",
"session_number": 3,
"proxy_host": "127.0.0.1",
"proxy_port": 1087
}

```

2. 合约

```json
{
"key": "xxxx",
"secret": "xxxx",
"会话数": 3,
"服务器": "REAL",
"合约模式": "正向",
"代理地址": "127.0.0.1",
"代理端口": 1087
}

```

记得合约代理的proxy_host或者代理地址:
只能写ip地址，不能写http://xxx.xxx.xxx.xxx, 不包含http或者https协议头


## 数据批量爬取数据
具体参考crawl_data.py脚本,修改成自己需要的交易对，填写具体的交易对就可以了。记得修改对应的爬取时间
1. download_spot(), 这个方法是爬取现货的数据.

2. download_future(), 这是合约的数据

记得下载好BTCUSDT和ETHUSDT的数据，后面的课程会用到。
169 changes: 169 additions & 0 deletions class_08/crawl_data.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
"""
we use the ccxt to crawl data then save it to csv file.
you need to install ccxt by running firstly:
"""

import pandas as pd
import time
from datetime import datetime
import requests
import pytz
from howtrader.trader.database import database_manager

pd.set_option('expand_frame_repr', False) #
from howtrader.trader.object import BarData, Interval, Exchange

BINANCE_SPOT_LIMIT = 1000
BINANCE_FUTURE_LIMIT = 1500

CHINA_TZ = pytz.timezone("Asia/Shanghai")
from threading import Thread


def generate_datetime(timestamp: float) -> datetime:
"""
:param timestamp:
:return:
"""
dt = datetime.fromtimestamp(timestamp / 1000)
dt = CHINA_TZ.localize(dt)
return dt


def get_binance_data(symbol: str, exchanges: str, start_time: str, end_time: str):
"""
爬取币安交易所的数据
:param symbol: BTCUSDT.
:param exchanges: 现货、USDT合约, 或者币币合约.
:param start_time: 格式如下:2020-1-1 或者2020-01-01
:param end_time: 格式如下:2020-1-1 或者2020-01-01
:return:
"""

api_url = ''
save_symbol = symbol
gate_way = 'BINANCES'

if exchanges == 'spot':
print("spot")
limit = BINANCE_SPOT_LIMIT
save_symbol = symbol.lower()
gate_way = 'BINANCE'
api_url = f'https://api.binance.com/api/v3/klines?symbol={symbol}&interval=1m&limit={limit}'

elif exchanges == 'future':
print('future')
limit = BINANCE_FUTURE_LIMIT
api_url = f'https://fapi.binance.com/fapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'

elif exchanges == 'coin_future':
print("coin_future")
limit = BINANCE_FUTURE_LIMIT
f'https://dapi.binance.com/dapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'
pass

else:
raise Exception('交易所名称请输入以下其中一个：spot, future, coin_future')

start_time = int(datetime.strptime(start_time, '%Y-%m-%d').timestamp() * 1000)
end_time = int(datetime.strptime(end_time, '%Y-%m-%d').timestamp() * 1000)

while True:
try:
print(start_time)
url = f'{api_url}&startTime={start_time}'
print(url)

data = requests.get(url=url).json()

"""
[
[
1591258320000, // 开盘时间
"9640.7", // 开盘价
"9642.4", // 最高价
"9640.6", // 最低价
"9642.0", // 收盘价(当前K线未结束的即为最新价)
"206", // 成交量
1591258379999, // 收盘时间
"2.13660389", // 成交额(标的数量)
48, // 成交笔数
"119", // 主动买入成交量
"1.23424865", // 主动买入成交额(标的数量)
"0" // 请忽略该参数
]
"""

buf = []

for l in data:
bar = BarData(
symbol=save_symbol,
exchange=Exchange.BINANCE,
datetime=generate_datetime(l[0]),
interval=Interval.MINUTE,
volume=float(l[5]),
open_price=float(l[1]),
high_price=float(l[2]),
low_price=float(l[3]),
close_price=float(l[4]),
gateway_name=gate_way
)
buf.append(bar)

database_manager.save_bar_data(buf)

# 到结束时间就退出, 后者收盘价大于当前的时间.
if (data[-1][0] > end_time) or data[-1][6] >= (int(time.time() * 1000) - 60 * 1000):
break

start_time = data[-1][0]

except Exception as error:
print(error)
time.sleep(10)


def download_spot():
"""
下载现货数据的方法.
:return:
"""
t1 = Thread(target=get_binance_data, args=('BTCUSDT', 'spot', "2018-1-1", "2019-1-1"))

t2 = Thread(target=get_binance_data, args=('BTCUSDT', 'spot', "2019-1-1", "2020-1-1"))

t3 = Thread(target=get_binance_data, args=('BTCUSDT', 'spot', "2020-1-1", "2020-11-16"))

t1.start()
t2.start()
t3.start()

t1.join()
t2.join()
t3.join()


def download_future():
"""
下载合约数据的方法。
:return:
"""
t1 = Thread(target=get_binance_data, args=('BTCUSDT', 'future', "2019-9-10", "2020-3-1"))
t2 = Thread(target=get_binance_data, args=('BTCUSDT', 'future', "2019-3-1", "2020-11-16"))

t1.start()
t2.start()

t1.join()
t2.join()


if __name__ == '__main__':
# download_spot() # 下载现货的数据.

download_future() # 下载合约的数据


Loading

0 comments on commit 7ada212

Please sign in to comment.