c10.py

# -*- encoding:utf-8 -*-
from __future__ import print_function
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd
# import warnings

# noinspection PyUnresolvedReferences
import abu_local_env
import abupy
from abupy import abu
from abupy import ABuSymbolPd

import sklearn.preprocessing as preprocessing

# warnings.filterwarnings('ignore')
sns.set_context(rc={'figure.figsize': (14, 7)})
# 使用沙盒数据，目的是和书中一样的数据环境
abupy.env.enable_example_env_ipython()

"""
    第10章 量化系统——机器学习•猪老三

    abu量化系统github地址：https://github.com/bbfamily/abu (您的star是我的动力！)
    abu量化文档教程ipython notebook：https://github.com/bbfamily/abu/tree/master/abupy_lecture
"""

"""
    10.2 猪老三世界中的量化环境
"""

"""
    是否开启date_week噪音, 开启这个的目的是让分类结果正确率降低，接近真实
"""
g_with_date_week_noise = False


def _gen_another_word_price(kl_another_word):
    """
    生成股票在另一个世界中的价格
    :param kl_another_word:
    :return:
    """
    for ind in np.arange(2, kl_another_word.shape[0]):
        # 前天数据
        bf_yesterday = kl_another_word.iloc[ind - 2]
        # 昨天
        yesterday = kl_another_word.iloc[ind - 1]
        # 今天
        today = kl_another_word.iloc[ind]
        # 生成今天的收盘价格
        kl_another_word.close[ind] = _gen_another_word_price_rule(
            yesterday.close, yesterday.volume,
            bf_yesterday.close, bf_yesterday.volume,
            today.volume, today.date_week)


def _gen_another_word_price_rule(yesterday_close, yesterday_volume,
                                 bf_yesterday_close,
                                 bf_yesterday_volume,
                                 today_volume, date_week):
    """
        通过前天收盘量价，昨天收盘量价，今天的量，构建另一个世界中的价格模型
    """
    # 昨天收盘价格与前天收盘价格的价格差
    price_change = yesterday_close - bf_yesterday_close
    # 昨天成交量与前天成交量的量差
    volume_change = yesterday_volume - bf_yesterday_volume

    # 如果量和价变动一致，今天价格涨，否则跌
    # 即量价齐涨－>涨, 量价齐跌－>涨，量价不一致－>跌
    sign = 1.0 if price_change * volume_change > 0 else -1.0

    # 通过date_week生成噪音，否则之后分类100%分对
    if g_with_date_week_noise:
        # 针对sign生成噪音，噪音的生效的先决条件是今天的量是这三天最大的
        gen_noise = today_volume > np.max(
            [yesterday_volume, bf_yesterday_volume])
        # 如果量是这三天最大 且是周五，下跌
        if gen_noise and date_week == 4:
            sign = -1.0
        # 如果量是这三天最大，如果是周一，上涨
        elif gen_noise and date_week == 0:
            sign = 1.0

    # 今天的涨跌幅度基础是price_change（昨天前天的价格变动）
    price_base = abs(price_change)
    # 今天的涨跌幅度变动因素：量比，
    # 今天的成交量/昨天的成交量 和 今天的成交量/前天的成交量 的均值
    price_factor = np.mean([today_volume / yesterday_volume,
                            today_volume / bf_yesterday_volume])

    if abs(price_base * price_factor) < yesterday_close * 0.10:
        # 如果 量比 * price_base 没超过10%，今天价格计算
        today_price = yesterday_close + \
                      sign * price_base * price_factor
    else:
        # 如果涨跌幅度超过10%，限制上限，下限为10%
        today_price = yesterday_close + sign * yesterday_close * 0.10
    return today_price


def change_real_to_another_word(symbol):
    """
    将原始真正的股票数据价格列只保留前两天数据，成交量，周几列完全保留
    价格列其他数据使用_gen_another_word_price变成另一个世界价格
    :param symbol:
    :return:
    """
    kl_pd = ABuSymbolPd.make_kl_df(symbol)
    if kl_pd is not None:
        # 原始股票数据也只保留价格，周几，成交量
        kl_pig_three = kl_pd.filter(['close', 'date_week', 'volume'])
        # 只保留原始头两天的交易收盘价格，其他的的都赋予nan
        kl_pig_three['close'][2:] = np.nan
        # 将其他nan价格变成猪老三世界中价格使用_gen_another_word_price
        _gen_another_word_price(kl_pig_three)
        return kl_pig_three


def sample_102(show=True):
    """
    10.2 生成猪老三的世界中的映射股票数据
    :return:
    """
    choice_symbols = ['usNOAH', 'usSFUN', 'usBIDU', 'usAAPL', 'usGOOG',
                      'usTSLA', 'usWUBA', 'usVIPS']
    another_word_dict = {}
    real_dict = {}
    for symbol in choice_symbols:
        # 猪老三世界的股票走势字典
        another_word_dict[symbol] = change_real_to_another_word(symbol)
        # 真实世界的股票走势字典，这里不考虑运行效率问题
        real_dict[symbol] = ABuSymbolPd.make_kl_df(symbol)
    if show:
        # 表10-1所示
        print('another_word_dict[usNOAH].head():\n', another_word_dict['usNOAH'].head())

        print('real_dict[usNOAH].head():\n', real_dict['usNOAH'].head().filter(['close', 'date_week', 'volume']))

        import itertools
        # 4 ＊ 2
        _, axs = plt.subplots(nrows=4, ncols=2, figsize=(20, 15))
        # 将画布序列拉平
        axs_list = list(itertools.chain.from_iterable(axs))

        for symbol, ax in zip(choice_symbols, axs_list):
            # 绘制猪老三世界的股价走势
            another_word_dict[symbol].close.plot(ax=ax)
            # 同样的股票在真实世界的股价走势
            real_dict[symbol].close.plot(ax=ax)
            ax.set_title(symbol)
        plt.show()
    return another_word_dict


"""
    10.3 有监督机器学习
"""


def gen_pig_three_feature(kl_another_word):
    """
    猪老三构建特征模型函数
    :param kl_another_word: 即上一节使用_gen_another_word_price
    生成的dataframe有收盘价，周几，成交量列
    :return:
    """
    # y值使用close.pct_change即涨跌幅度
    kl_another_word['regress_y'] = kl_another_word.close.pct_change()
    # 前天收盘价格
    kl_another_word['bf_yesterday_close'] = 0
    # 昨天收盘价格
    kl_another_word['yesterday_close'] = 0
    # 昨天收盘成交量
    kl_another_word['yesterday_volume'] = 0
    # 前天收盘成交量
    kl_another_word['bf_yesterday_volume'] = 0

    # 对齐特征，前天收盘价格即与今天的收盘错2个时间单位，[2:] = [:-2]
    kl_another_word['bf_yesterday_close'][2:] = \
        kl_another_word['close'][:-2]
    # 对齐特征，前天成交量
    kl_another_word['bf_yesterday_volume'][2:] = \
        kl_another_word['volume'][:-2]
    # 对齐特征，昨天收盘价与今天的收盘错1个时间单位，[1:] = [:-1]
    kl_another_word['yesterday_close'][1:] = \
        kl_another_word['close'][:-1]
    # 对齐特征，昨天成交量
    kl_another_word['yesterday_volume'][1:] = \
        kl_another_word['volume'][:-1]

    # 特征1: 价格差
    kl_another_word['feature_price_change'] = \
        kl_another_word['yesterday_close'] - \
        kl_another_word['bf_yesterday_close']

    # 特征2: 成交量差
    kl_another_word['feature_volume_Change'] = \
        kl_another_word['yesterday_volume'] - \
        kl_another_word['bf_yesterday_volume']

    # 特征3: 涨跌sign
    kl_another_word['feature_sign'] = np.sign(
        kl_another_word['feature_price_change'] * kl_another_word[
            'feature_volume_Change'])

    # 特征4: 周几
    kl_another_word['feature_date_week'] = kl_another_word[
        'date_week']

    """
        构建噪音特征, 因为猪老三也不可能全部分析正确真实的特征因素
        这里引入一些噪音特征
    """
    # 成交量乘积
    kl_another_word['feature_volume_noise'] = \
        kl_another_word['yesterday_volume'] * \
        kl_another_word['bf_yesterday_volume']

    # 价格乘积
    kl_another_word['feature_price_noise'] = \
        kl_another_word['yesterday_close'] * \
        kl_another_word['bf_yesterday_close']

    # 将数据标准化
    scaler = preprocessing.StandardScaler()
    kl_another_word['feature_price_change'] = scaler.fit_transform(
        kl_another_word['feature_price_change'].values.reshape(-1, 1))
    kl_another_word['feature_volume_Change'] = scaler.fit_transform(
        kl_another_word['feature_volume_Change'].values.reshape(-1, 1))
    kl_another_word['feature_volume_noise'] = scaler.fit_transform(
        kl_another_word['feature_volume_noise'].values.reshape(-1, 1))
    kl_another_word['feature_price_noise'] = scaler.fit_transform(
        kl_another_word['feature_price_noise'].values.reshape(-1, 1))

    # 只筛选feature_开头的特征和regress_y，抛弃前两天数据，即[2:]
    kl_pig_three_feature = kl_another_word.filter(
        regex='regress_y|feature_*')[2:]
    return kl_pig_three_feature


def sample_103_0(show=True):
    """
    10.3 生成猪老三的训练集特征示例
    :return:
    """
    another_word_dict = sample_102(show=False)
    pig_three_feature = None
    for symbol in another_word_dict:
        # 首先拿出对应的走势数据
        kl_another_word = another_word_dict[symbol]
        # 通过走势数据生成训练集特征通过gen_pig_three_feature
        kl_feature = gen_pig_three_feature(kl_another_word)
        # 将每个股票的特征数据都拼接起来，形成训练集
        pig_three_feature = kl_feature if pig_three_feature is None \
            else pig_three_feature.append(kl_feature)

    # Dataframe -> matrix
    feature_np = pig_three_feature.as_matrix()
    # x特征矩阵
    train_x = feature_np[:, 1:]
    # 回归训练的连续值y
    train_y_regress = feature_np[:, 0]
    # 分类训练的离散值y，之后分类技术使用
    # noinspection PyTypeChecker
    train_y_classification = np.where(train_y_regress > 0, 1, 0)

    if show:
        print('pig_three_feature.shape:', pig_three_feature.shape)
        print('pig_three_feature.tail():\n', pig_three_feature.tail())
        print('train_x[:5], train_y_regress[:5], train_y_classification[:5]:\n', train_x[:5], train_y_regress[:5],
              train_y_classification[:5])

    return train_x, train_y_regress, train_y_classification, pig_three_feature


"""
    猪老三使用回归预测股价
"""


def sample_1031_1():
    """
    10.3.1_1 猪老三使用回归预测股价：生成训练集数据和测试集数据
    :return:
    """

    # noinspection PyShadowingNames
    def gen_feature_from_symbol(symbol):
        """
        封装由一个symbol转换为特征矩阵序列函数
        :param symbol:
        :return:
        """
        # 真实世界走势数据转换到老三的世界
        kl_another_word = change_real_to_another_word(symbol)
        # 由走势转换为特征dataframe通过gen_pig_three_feature
        kl_another_word_feature_test = gen_pig_three_feature(kl_another_word)
        # 转换为matrix
        feature_np_test = kl_another_word_feature_test.as_matrix()
        # 从matrix抽取y回归
        test_y_regress = feature_np_test[:, 0]
        # y回归 －> y分类
        # noinspection PyTypeChecker
        test_y_classification = np.where(test_y_regress > 0, 1, 0)
        # 从matrix抽取x特征矩阵
        test_x = feature_np_test[:, 1:]
        return test_x, test_y_regress, test_y_classification, kl_another_word_feature_test

    # 生成训练集数据
    train_x, train_y_regress, train_y_classification, pig_three_feature = sample_103_0(show=False)
    # 生成测试集数据
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = gen_feature_from_symbol('usFB')

    print('训练集：{}, 测试集：{}'.format(pig_three_feature.shape[0], kl_another_word_feature_test.shape[0]))

    return train_x, train_y_regress, train_y_classification, pig_three_feature, \
           test_x, test_y_regress, test_y_classification, kl_another_word_feature_test


def regress_process(estimator, train_x, train_y_regress, test_x,
                    test_y_regress):
    # 训练训练集数据
    estimator.fit(train_x, train_y_regress)
    # 使用训练好的模型预测测试集对应的y，即根据usFB的走势特征预测股价涨跌幅度
    test_y_prdict_regress = estimator.predict(test_x)

    # 绘制usFB实际股价涨跌幅度
    plt.plot(test_y_regress.cumsum())
    # 绘制通过模型预测的usFB股价涨跌幅度
    plt.plot(test_y_prdict_regress.cumsum())

    # 针对训练集数据做交叉验证
    from abupy import cross_val_score
    from abupy.CoreBu.ABuFixes import mean_squared_error_scorer
    scores = cross_val_score(estimator, train_x,
                             train_y_regress, cv=10,
                             scoring=mean_squared_error_scorer)
    # mse开方 -> rmse
    mean_sc = -np.mean(np.sqrt(-scores))
    print('{} RMSE: {}'.format(estimator.__class__.__name__, mean_sc))


def sample_1031_2():
    """
    10.3.1_2 猪老三使用回归预测股价：LinearRegressio
    :return:
    """
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    # 实例化线性回归对象estimator
    from sklearn.linear_model import LinearRegression
    estimator = LinearRegression()
    # 将回归模型对象，训练集x，训练集连续y值，测试集x，测试集连续y传入
    regress_process(estimator, train_x, train_y_regress, test_x,
                    test_y_regress)
    plt.show()

    from abupy import ABuMLExecute
    ABuMLExecute.plot_learning_curve(estimator, train_x, train_y_regress, cv=10)


def sample_1031_3():
    """
    10.3.1_3 猪老三使用回归预测股价：PolynomialFeatures
    :return:
    """
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    from sklearn.pipeline import make_pipeline
    from sklearn.preprocessing import PolynomialFeatures
    from sklearn.linear_model import LinearRegression

    # pipeline套上 degree=3 ＋ LinearRegression
    estimator = make_pipeline(PolynomialFeatures(degree=3),
                              LinearRegression())
    # 继续使用regress_process，区别是estimator变了
    regress_process(estimator, train_x, train_y_regress, test_x,
                    test_y_regress)
    plt.show()


def sample_1031_4():
    """
    10.3.1_4 猪老三使用回归预测股价：使用集成学习算法预测股价AdaBoost与RandomForest
    :return:
    """
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    # AdaBoost
    from sklearn.ensemble import AdaBoostRegressor

    estimator = AdaBoostRegressor(n_estimators=100)
    regress_process(estimator, train_x, train_y_regress, test_x,
                    test_y_regress)
    plt.show()
    # RandomForest
    from sklearn.ensemble import RandomForestRegressor

    estimator = RandomForestRegressor(n_estimators=100)
    regress_process(estimator, train_x, train_y_regress, test_x, test_y_regress)
    plt.show()


"""
    10.3.2 猪老三使用分类预测股票涨跌
"""


def classification_process(estimator, train_x, train_y_classification,
                           test_x, test_y_classification):
    from sklearn import metrics
    # 训练数据，这里分类要所以要使用y_classification
    estimator.fit(train_x, train_y_classification)
    # 使用训练好的分类模型预测测试集对应的y，即根据usFB的走势特征预测涨跌
    test_y_prdict_classification = estimator.predict(test_x)
    # 通过metrics.accuracy_score度量预测涨跌的准确率
    print("{} accuracy = {:.2f}".format(
        estimator.__class__.__name__,
        metrics.accuracy_score(test_y_classification,
                               test_y_prdict_classification)))

    from abupy import cross_val_score
    # 针对训练集数据做交叉验证scoring='accuracy'，cv＝10
    scores = cross_val_score(estimator, train_x,
                             train_y_classification,
                             cv=10,
                             scoring='accuracy')
    # 所有交叉验证的分数取平均值
    mean_sc = np.mean(scores)
    print('cross validation accuracy mean: {:.2f}'.format(mean_sc))


def sample_1032_1():
    """
    10.3.2_1 猪老三使用分类预测股票涨跌：LogisticRegression
    :return:
    """
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    # 无噪音分类正确100%
    from sklearn.linear_model import LogisticRegression
    estimator = LogisticRegression(C=1.0, penalty='l1', tol=1e-6)
    # 将分类器，训练集x，训练集y分类，测试集，测试集y分别传入函数
    classification_process(estimator, train_x, train_y_classification,
                           test_x, test_y_classification)

    # 开启噪音，再来一遍，有噪音正确率93%, 之后的都开启g_with_date_week_noise
    global g_with_date_week_noise
    g_with_date_week_noise = True
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
    classification_process(estimator, train_x, train_y_classification,
                           test_x, test_y_classification)


def sample_1032_2():
    """
    10.3.2_2 猪老三使用分类预测股票涨跌：svm
    :return:
    """
    global g_with_date_week_noise
    g_with_date_week_noise = True

    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    from sklearn.svm import SVC

    estimator = SVC(kernel='rbf')
    classification_process(estimator, train_x, train_y_classification,
                           test_x, test_y_classification)


def sample_1032_3():
    """
    10.3.2_3 猪老三使用分类预测股票涨跌：RandomForestClassifier
    :return:
    """
    global g_with_date_week_noise
    g_with_date_week_noise = True

    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    from sklearn.ensemble import RandomForestClassifier

    estimator = RandomForestClassifier(n_estimators=100)
    classification_process(estimator, train_x, train_y_classification,
                           test_x, test_y_classification)


def sample_1032_4(show=True):
    """
    10.3.2_4 猪老三使用分类预测股票涨跌：train_test_split
    :return:
    """
    from sklearn import metrics
    from abupy import train_test_split

    # noinspection PyShadowingNames
    def train_test_split_xy(estimator, x, y, test_size=0.5,
                            random_state=0):
        # 通过train_test_split将原始训练集随机切割为新训练集与测试集
        train_x, test_x, train_y, test_y = \
            train_test_split(x, y, test_size=test_size,
                             random_state=random_state)

        if show:
            print(x.shape, y.shape)
            print(train_x.shape, train_y.shape)
            print(test_x.shape, test_y.shape)

        clf = estimator.fit(train_x, train_y)
        predictions = clf.predict(test_x)

        if show:
            # 度量准确率
            print("accuracy = %.2f" %
                  (metrics.accuracy_score(test_y, predictions)))

            # 度量查准率
            print("precision_score = %.2f" %
                  (metrics.precision_score(test_y, predictions)))

            # 度量回收率
            print("recall_score = %.2f" %
                  (metrics.recall_score(test_y, predictions)))

        return test_y, predictions

    global g_with_date_week_noise
    g_with_date_week_noise = True
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    from sklearn.ensemble import RandomForestClassifier
    estimator = RandomForestClassifier(n_estimators=100)

    test_y, predictions = train_test_split_xy(estimator, train_x, train_y_classification)
    return estimator, train_x, train_y_classification, test_y, predictions


def sample_1032_5():
    """
    10.3.2_5 猪老三使用分类预测股票涨跌：混淆矩阵和roc曲线
    :return:
    """

    from sklearn import metrics

    # noinspection PyShadowingNames
    def confusion_matrix_with_report(test_y, predictions):
        confusion_matrix = metrics.confusion_matrix(test_y, predictions)
        # print("Confusion Matrix ", confusion_matrix)
        print("          Predicted")
        print("         |  0  |  1  |")
        print("         |-----|-----|")
        print("       0 | %3d | %3d |" % (confusion_matrix[0, 0],
                                          confusion_matrix[0, 1]))
        print("Actual   |-----|-----|")
        print("       1 | %3d | %3d |" % (confusion_matrix[1, 0],
                                          confusion_matrix[1, 1]))
        print("         |-----|-----|")

        print(metrics.classification_report(test_y, predictions))

    estimator, train_x, train_y_classification, test_y, predictions = sample_1032_4(show=False)
    confusion_matrix_with_report(test_y, predictions)
    from abupy import ABuMLExecute
    ABuMLExecute.plot_roc_estimator(estimator, train_x, train_y_classification)


def sample_1033_1():
    """
    10.3.3 通过决策树分类，绘制出决策图
    这里需要安装dot graphviz，才能通过os.system("dot -T png graphviz.dot -o graphviz.png")生成png
    :return:
    """
    from sklearn.tree import DecisionTreeClassifier
    from sklearn import tree
    import os

    estimator = DecisionTreeClassifier(max_depth=2, random_state=1)

    # noinspection PyShadowingNames
    def graphviz_tree(estimator, features, x, y):
        if not hasattr(estimator, 'tree_'):
            print('only tree can graphviz!')
            return

        estimator.fit(x, y)
        # 将决策模型导出graphviz.dot文件
        tree.export_graphviz(estimator.tree_, out_file='graphviz.dot',
                             feature_names=features)
        # 通过dot将模型绘制决策图，保存png
        os.system("dot -T png graphviz.dot -o graphviz.png")

    global g_with_date_week_noise
    g_with_date_week_noise = True
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    # 这里会使用到特征的名称列pig_three_feature.columns[1:]
    graphviz_tree(estimator, pig_three_feature.columns[1:], train_x,
                  train_y_classification)

    import PIL.Image
    PIL.Image.open('graphviz.png').show()


def sample_1033_2():
    """
    10.3.3 特征的重要性排序及支持度评级
    :return:
    """
    global g_with_date_week_noise
    g_with_date_week_noise = True
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    # noinspection PyShadowingNames
    def importances_coef_pd(estimator):
        """
            特征的重要性
        """
        if hasattr(estimator, 'feature_importances_'):
            # 有feature_importances_的通过sort_values排序
            return pd.DataFrame(
                {'feature': list(pig_three_feature.columns[1:]),
                 'importance': estimator.feature_importances_}).sort_values('importance')

        elif hasattr(estimator, 'coef_'):
            # 有coef_的通过coef排序
            return pd.DataFrame(
                {"columns": list(pig_three_feature.columns)[1:], "coef": list(estimator.coef_.T)}).sort_values('coef')
        else:
            print('estimator not hasattr feature_importances_ or coef_!')

    # 使用随机森林分类器
    from sklearn.ensemble import RandomForestClassifier
    estimator = RandomForestClassifier(n_estimators=100)
    # 训练数据模型
    estimator.fit(train_x, train_y_classification)
    # 对训练后的模型特征的重要度进行判定，重要程度由小到大，表10-4所示
    print('importances_coef_pd(estimator):\n', importances_coef_pd(estimator))

    from sklearn.feature_selection import RFE

    # noinspection PyShadowingNames
    def feature_selection(estimator, x, y):
        """
            支持度评级
        """
        selector = RFE(estimator)
        selector.fit(x, y)
        print('RFE selection')
        print(pd.DataFrame(
            {'support': selector.support_, 'ranking': selector.ranking_},
            index=pig_three_feature.columns[1:]))

    print('feature_selection(estimator, train_x, train_y_classification):\n',
          feature_selection(estimator, train_x, train_y_classification))


"""
    10.4 无监督机器学习
"""


def sample_1041():
    """
    10.4.1 使用降维可视化数据
    :return:
    """
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    from sklearn.decomposition import PCA
    from abupy import ABuMLExecute

    # noinspection PyShadowingNames
    def plot_decision_function(estimator, x, y):
        # pca进行降维，只保留2个特征序列
        pca_2n = PCA(n_components=2)
        x = pca_2n.fit_transform(x)

        # 进行训练
        estimator.fit(x, y)
        plt.scatter(x[:, 0], x[:, 1], c=y, s=50, cmap='spring')
        ABuMLExecute.plot_decision_boundary(
            lambda p_x: estimator.predict(p_x), x, y)

    from sklearn.ensemble import RandomForestClassifier
    estimator = RandomForestClassifier(n_estimators=100)
    plot_decision_function(estimator, train_x, train_y_classification)


# noinspection PyTypeChecker
def sample_1042():
    """
    10.4.2 猪老三使用聚类算法提高正确率
    :return:
    """
    global g_with_date_week_noise
    g_with_date_week_noise = True
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    # 使用随机森林作为分类器
    from sklearn.ensemble import RandomForestClassifier
    estimator = RandomForestClassifier(n_estimators=100)
    estimator.fit(train_x, train_y_classification)
    test_y_prdict_classification = estimator.predict(test_x)

    from sklearn import metrics
    print("accuracy = %.2f" % (
        metrics.accuracy_score(test_y_classification,
                               test_y_prdict_classification)))

    # 测试集feature即usFB的kl feature
    pig_three_kmean_feature = kl_another_word_feature_test
    # 测试集真实的涨跌结果test_y_classification
    pig_three_kmean_feature['y'] = test_y_classification
    # 使用刚刚的随机森林作为分类器的预测涨跌结果test_y_prdict_classification
    pig_three_kmean_feature['y_prdict'] = test_y_prdict_classification
    # 即生成一列新数据记录预测是否正确
    pig_three_kmean_feature['y_same'] = np.where(
        pig_three_kmean_feature['y'] ==
        pig_three_kmean_feature['y_prdict'], 1, 0)
    # 将feature中只保留刚刚得到的y_same
    pig_three_kmean_feature = pig_three_kmean_feature.filter(['y_same'])

    from sklearn.cluster import KMeans

    # 使用刚刚得到的只有y_same列的数据赋值x_kmean
    x_kmean = pig_three_kmean_feature.values
    # n_clusters=2, 即只聚两类数据
    kmean = KMeans(n_clusters=2)
    kmean.fit(x_kmean)
    # 将聚类标签赋予新的一列cluster
    pig_three_kmean_feature['cluster'] = kmean.predict(x_kmean)
    # 将周几这个特征合并过来
    pig_three_kmean_feature['feature_date_week'] = \
        kl_another_word_feature_test['feature_date_week']
    # 表10-5所示
    print('pig_three_kmean_feature.tail():\n', pig_three_kmean_feature.tail())

    # 表10-6所示
    print('pd.crosstab(pig_three_kmean_feature.feature_date_week, pig_three_kmean_feature.cluster):\n',
          pd.crosstab(pig_three_kmean_feature.feature_date_week, pig_three_kmean_feature.cluster))


"""
    10.5 梦醒时分
"""


def sample_105_0():
    """
    10.5 AbuML
    :return:
    """
    global g_with_date_week_noise
    g_with_date_week_noise = True
    train_x, train_y_regress, train_y_classification, pig_three_feature, \
    test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()

    from abupy import AbuML
    # 通过x, y矩阵和特征的DataFrame对象组成AbuML
    ml = AbuML(train_x, train_y_classification, pig_three_feature)
    # 使用随机森林作为分类器
    _ = ml.estimator.random_forest_classifier()

    # 交织验证结果的正确率
    print('ml.cross_val_accuracy_score():\n', ml.cross_val_accuracy_score())
    # 特征的选择
    print('ml.feature_selection():\n', ml.feature_selection())


"""
    如下内容不能使用沙盒环境, 建议对照阅读：
        abu量化文档－第十九节 数据源
        第20节 美股交易UMP决策
"""


def sample_1051_0():
    """
    10.5.1 回测中生成特征，切分训练测试集，成交买单快照: 数据准备

    如果没有运行过abu量化文档－第十九节 数据源：中使用腾讯数据源进行数据更新，需要运行
    如果运行过就不要重复运行了：
    """
    from abupy import EMarketTargetType, EMarketSourceType, EDataCacheType
    # 关闭沙盒数据环境
    abupy.env.disable_example_env_ipython()
    abupy.env.g_market_source = EMarketSourceType.E_MARKET_SOURCE_tx
    abupy.env.g_data_cache_type = EDataCacheType.E_DATA_CACHE_CSV
    # 首选这里预下载市场中所有股票的6年数据(做5年回测，需要预先下载6年数据)
    abu.run_kl_update(start='2011-08-08', end='2017-08-08', market=EMarketTargetType.E_MARKET_TARGET_US)


def sample_1051_1(from_cache=False, show=True):
    """
    10.5.1 回测中生成特征，切分训练测试集，成交买单快照: 数据准备
    :return:
    """
    from abupy import AbuMetricsBase
    from abupy import AbuFactorBuyBreak
    from abupy import AbuFactorAtrNStop
    from abupy import AbuFactorPreAtrNStop
    from abupy import AbuFactorCloseAtrNStop

    # 关闭沙盒数据环境
    abupy.env.disable_example_env_ipython()
    from abupy import EMarketDataFetchMode
    # 因为sample_94_1下载了预先数据，使用缓存，设置E_DATA_FETCH_FORCE_LOCAL，实际上run_kl_update最后会把设置set到FORCE_LOCAL
    abupy.env.g_data_fetch_mode = EMarketDataFetchMode.E_DATA_FETCH_FORCE_LOCAL

    # 设置选股因子，None为不使用选股因子
    stock_pickers = None
    # 买入因子依然延用向上突破因子
    buy_factors = [{'xd': 60, 'class': AbuFactorBuyBreak},
                   {'xd': 42, 'class': AbuFactorBuyBreak}]

    # 卖出因子继续使用上一章使用的因子
    sell_factors = [
        {'stop_loss_n': 1.0, 'stop_win_n': 3.0,
         'class': AbuFactorAtrNStop},
        {'class': AbuFactorPreAtrNStop, 'pre_atr_n': 1.5},
        {'class': AbuFactorCloseAtrNStop, 'close_atr_n': 1.5}
    ]

    # 回测生成买入时刻特征
    abupy.env.g_enable_ml_feature = True
    # 回测将symbols切割分为训练集数据和测试集数据
    abupy.env.g_enable_train_test_split = True
    # 下面设置回测时切割训练集，测试集使用的切割比例参数，默认为10，即切割为10份，9份做为训练，1份做为测试，
    # 由于美股股票数量多，所以切割分为4份，3份做为训练集，1份做为测试集
    abupy.env.g_split_tt_n_folds = 4

    from abupy import EStoreAbu
    if from_cache:
        abu_result_tuple = \
            abu.load_abu_result_tuple(n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
                                      custom_name='train_us')
    else:
        # 初始化资金500万，资金管理依然使用默认atr
        read_cash = 5000000
        # 每笔交易的买入基数资金设置为万分之15
        abupy.beta.atr.g_atr_pos_base = 0.0015
        # 使用run_loop_back运行策略，因子使用和之前一样，
        # choice_symbols=None为全市场回测，5年历史数据回测
        abu_result_tuple, _ = abu.run_loop_back(read_cash,
                                                buy_factors, sell_factors,
                                                stock_pickers,
                                                choice_symbols=None,
                                                start='2012-08-08', end='2017-08-08')
        # 把运行的结果保存在本地，以便之后分析回测使用，保存回测结果数据代码如下所示
        abu.store_abu_result_tuple(abu_result_tuple, n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
                                   custom_name='train_us')

    if show:
        metrics = AbuMetricsBase(*abu_result_tuple)
        metrics.fit_metrics()
        metrics.plot_returns_cmp(only_show_returns=True)

    "＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊"
    abupy.env.g_enable_train_test_split = False
    # 使用切割好的测试数据
    abupy.env.g_enable_last_split_test = True

    from abupy import EStoreAbu
    if from_cache:
        abu_result_tuple_test = \
            abu.load_abu_result_tuple(n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
                                      custom_name='test_us')
    else:
        read_cash = 5000000
        abupy.beta.atr.g_atr_pos_base = 0.007
        choice_symbols = None
        abu_result_tuple_test, kl_pd_manager_test = abu.run_loop_back(read_cash,
                                                                      buy_factors, sell_factors, stock_pickers,
                                                                      choice_symbols=choice_symbols, start='2012-08-08',
                                                                      end='2017-08-08')
        abu.store_abu_result_tuple(abu_result_tuple_test, n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
                                   custom_name='test_us')

    if show:
        metrics = AbuMetricsBase(*abu_result_tuple_test)
        metrics.fit_metrics()
        metrics.plot_returns_cmp(only_show_returns=True)
        print(abu_result_tuple.orders_pd[abu_result_tuple.orders_pd.result != 0].head())

    return abu_result_tuple, abu_result_tuple_test


# noinspection PyUnresolvedReferences
def sample_1052():
    """
    10.5.2 基于特征的交易预测
    :return:
    """
    # 需要在有缓存的情况下运行
    abu_result_tuple, _ = sample_1051_1(from_cache=True, show=False)

    from abupy.UmpBu.ABuUmpMainMul import UmpMulFiter
    mul = UmpMulFiter(orders_pd=abu_result_tuple.orders_pd, scaler=False)
    print('mul.df.head():\n', mul.df.head())

    # 默认使用svm作为分类器
    print('decision_tree_classifier cv please wait...')
    mul.estimator.decision_tree_classifier()
    mul.cross_val_accuracy_score()

    # 默认使用svm作为分类器
    print('knn_classifier cv please wait...')
    # 默认使用svm作为分类器, 改分类器knn
    mul.estimator.knn_classifier()
    mul.cross_val_accuracy_score()

    from abupy.UmpBu.ABuUmpMainBase import UmpDegFiter
    deg = UmpDegFiter(orders_pd=abu_result_tuple.orders_pd)
    print('deg.df.head():\n', deg.df.head())

    print('xgb_classifier cv please wait...')
    # 分类器使用GradientBoosting
    deg.estimator.xgb_classifier()
    deg.cross_val_accuracy_score()

    print('adaboost_classifier cv please wait...')
    # 分类器使用adaboost
    deg.estimator.adaboost_classifier(base_estimator=None)
    deg.cross_val_accuracy_score()

    print('train_test_split_xy please wait...')
    deg.train_test_split_xy()


if __name__ == "__main__":
    sample_102()
    # sample_103_0()
    # sample_1031_1()
    # sample_1031_2()
    # sample_1031_3()
    # sample_1031_4()
    # sample_1032_1()
    # sample_1032_2()
    # sample_1032_3()
    # sample_1032_4()
    # sample_1032_5()
    # sample_1033_1()
    # sample_1033_2()
    # sample_1041()
    # sample_1042()
    # sample_105_0()
    # sample_1051_0()
    # sample_1051_1(from_cache=True)
    # sample_1051_1(from_cache=False)
    # sample_1052()