From 511e2443d05d25478671f51bdca24f7677d5aa97 Mon Sep 17 00:00:00 2001 From: D-X-Y <280835372@qq.com> Date: Sat, 6 Mar 2021 06:13:22 -0800 Subject: [PATCH] To re-org Q-results --- .latent-data/qlib | 2 +- exps/trading/baselines.py | 14 +-- exps/trading/organize_results.py | 135 ++++++++++++++++++++++++++ exps/trading/workflow_tt.py | 9 +- lib/layers/positional_embedding.py | 4 +- lib/trade_models/quant_transformer.py | 46 ++++----- lib/utils/__init__.py | 2 +- 7 files changed, 172 insertions(+), 40 deletions(-) create mode 100644 exps/trading/organize_results.py diff --git a/.latent-data/qlib b/.latent-data/qlib index 49697b1..91fd53a 160000 --- a/.latent-data/qlib +++ b/.latent-data/qlib @@ -1 +1 @@ -Subproject commit 49697b1f1568608e3077450b72fe3ed5b92ec1e5 +Subproject commit 91fd53ab4d0724df73ccf8855ed83b6e1760bb08 diff --git a/exps/trading/baselines.py b/exps/trading/baselines.py index 5ba90ee..00f3891 100644 --- a/exps/trading/baselines.py +++ b/exps/trading/baselines.py @@ -65,16 +65,16 @@ def update_market(config, market): def run_exp(task_config, dataset, experiment_name, recorder_name, uri): # model initiaiton - print('') - print('[{:}] - [{:}]: {:}'.format(experiment_name, recorder_name, uri)) - print('dataset={:}'.format(dataset)) + print("") + print("[{:}] - [{:}]: {:}".format(experiment_name, recorder_name, uri)) + print("dataset={:}".format(dataset)) model = init_instance_by_config(task_config["model"]) # start exp with R.start(experiment_name=experiment_name, recorder_name=recorder_name, uri=uri): - log_file = R.get_recorder().root_uri / '{:}.log'.format(experiment_name) + log_file = R.get_recorder().root_uri / "{:}.log".format(experiment_name) set_log_basic_config(log_file) # train model @@ -109,12 +109,14 @@ def main(xargs, exp_yaml): qlib.init(**config.get("qlib_init")) dataset_config = config.get("task").get("dataset") dataset = init_instance_by_config(dataset_config) - pprint('args: {:}'.format(xargs)) + pprint("args: {:}".format(xargs)) pprint(dataset_config) pprint(dataset) for irun in range(xargs.times): - run_exp(config.get("task"), dataset, xargs.alg, "recorder-{:02d}-{:02d}".format(irun, xargs.times), xargs.save_dir) + run_exp( + config.get("task"), dataset, xargs.alg, "recorder-{:02d}-{:02d}".format(irun, xargs.times), xargs.save_dir + ) if __name__ == "__main__": diff --git a/exps/trading/organize_results.py b/exps/trading/organize_results.py new file mode 100644 index 0000000..33fd962 --- /dev/null +++ b/exps/trading/organize_results.py @@ -0,0 +1,135 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2021.02 # +##################################################### +# python exps/trading/organize_results.py +##################################################### +import sys, argparse +import numpy as np +from typing import List, Text +from collections import defaultdict, OrderedDict +from pathlib import Path +from pprint import pprint +import ruamel.yaml as yaml + +lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve() +if str(lib_dir) not in sys.path: + sys.path.insert(0, str(lib_dir)) + +import qlib +from qlib.config import REG_CN +from qlib.workflow import R + + +class QResult: + + def __init__(self): + self._result = defaultdict(list) + + def append(self, key, value): + self._result[key].append(value) + + @property + def result(self): + return self._result + + def update(self, metrics, filter_keys=None): + for key, value in metrics.items(): + if filter_keys is not None and key in filter_keys: + key = filter_keys[key] + elif filter_keys is not None: + continue + self.append(key, value) + + @staticmethod + def full_str(xstr, space): + xformat = '{:' + str(space) + 's}' + return xformat.format(str(xstr)) + + def info(self, keys: List[Text], separate: Text = '', space: int = 25, show=True): + avaliable_keys = [] + for key in keys: + if key not in self.result: + print('There are invalid key [{:}].'.format(key)) + else: + avaliable_keys.append(key) + head_str = separate.join([self.full_str(x, space) for x in avaliable_keys]) + values = [] + for key in avaliable_keys: + current_values = self._result[key] + mean = np.mean(current_values) + std = np.std(current_values) + values.append('{:.4f} $\pm$ {:.4f}'.format(mean, std)) + value_str = separate.join([self.full_str(x, space) for x in values]) + if show: + print(head_str) + print(value_str) + else: + return head_str, value_str + + +def compare_results(heads, values, names, space=10): + for idx, x in enumerate(heads): + assert x == heads[0], '[{:}] {:} vs {:}'.format(idx, x, heads[0]) + new_head = QResult.full_str('Name', space) + heads[0] + print(new_head) + for name, value in zip(names, values): + xline = QResult.full_str(name, space) + value + print(xline) + + +def filter_finished(recorders): + returned_recorders = dict() + not_finished = 0 + for key, recorder in recorders.items(): + if recorder.status == "FINISHED": + returned_recorders[key] = recorder + else: + not_finished += 1 + return returned_recorders, not_finished + + +def main(xargs): + R.reset_default_uri(xargs.save_dir) + experiments = R.list_experiments() + + key_map = {"IC": "IC", + "ICIR": "ICIR", + "Rank IC": "Rank_IC", + "Rank ICIR": "Rank_ICIR", + "excess_return_with_cost.annualized_return": "Annualized_Return", + "excess_return_with_cost.information_ratio": "Information_Ratio", + "excess_return_with_cost.max_drawdown": "Max_Drawdown"} + all_keys = list(key_map.values()) + + print("There are {:} experiments.".format(len(experiments))) + head_strs, value_strs, names = [], [], [] + for idx, (key, experiment) in enumerate(experiments.items()): + if experiment.id == '0': + continue + recorders = experiment.list_recorders() + recorders, not_finished = filter_finished(recorders) + print( + "====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders.".format( + idx, len(experiments), experiment.name, len(recorders), len(recorders) + not_finished + ) + ) + result = QResult() + for recorder_id, recorder in recorders.items(): + result.update(recorder.list_metrics(), key_map) + head_str, value_str = result.info(all_keys, show=False) + head_strs.append(head_str) + value_strs.append(value_str) + names.append(experiment.name) + compare_results(head_strs, value_strs, names, space=10) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser("Show Results") + parser.add_argument("--save_dir", type=str, default="./outputs/qlib-baselines", help="The checkpoint directory.") + args = parser.parse_args() + + provider_uri = "~/.qlib/qlib_data/cn_data" + qlib.init(provider_uri=provider_uri, region=REG_CN) + + main(args) diff --git a/exps/trading/workflow_tt.py b/exps/trading/workflow_tt.py index e68ce07..bdfd990 100644 --- a/exps/trading/workflow_tt.py +++ b/exps/trading/workflow_tt.py @@ -4,7 +4,7 @@ # Refer to: # - https://github.com/microsoft/qlib/blob/main/examples/workflow_by_code.ipynb # - https://github.com/microsoft/qlib/blob/main/examples/workflow_by_code.py -# python exps/trading/workflow_tt.py +# python exps/trading/workflow_tt.py --market all ##################################################### import sys, argparse from pathlib import Path @@ -102,10 +102,9 @@ def main(xargs): task = dict(model=model_config, dataset=dataset_config, record=record_config) - # start exp to train model - with R.start(experiment_name="tt_model", uri=xargs.save_dir): - set_log_basic_config(R.get_recorder().root_uri / 'log.log') + with R.start(experiment_name="tt_model", uri=xargs.save_dir + "-" + xargs.market): + set_log_basic_config(R.get_recorder().root_uri / "log.log") model = init_instance_by_config(model_config) dataset = init_instance_by_config(dataset_config) @@ -138,7 +137,7 @@ if __name__ == "__main__": parser.add_argument("--market", type=str, default="csi300", help="The market indicator.") args = parser.parse_args() - provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir + provider_uri = "~/.qlib/qlib_data/cn_data" qlib.init(provider_uri=provider_uri, region=REG_CN) main(args) diff --git a/lib/layers/positional_embedding.py b/lib/layers/positional_embedding.py index c11dcd2..67b6c46 100644 --- a/lib/layers/positional_embedding.py +++ b/lib/layers/positional_embedding.py @@ -24,12 +24,12 @@ class PositionalEncoder(nn.Module): else: pe[pos, i] = math.cos(value) pe = pe.unsqueeze(0) + self.dropout = nn.Dropout(p=dropout) self.register_buffer('pe', pe) def forward(self, x): batch, seq, fdim = x.shape[:3] embeddings = self.pe[:, :seq, :fdim] - import pdb; pdb.set_trace() outs = self.dropout(x + embeddings) - return x + embeddings + return outs diff --git a/lib/trade_models/quant_transformer.py b/lib/trade_models/quant_transformer.py index a8d5912..1e4e1e1 100755 --- a/lib/trade_models/quant_transformer.py +++ b/lib/trade_models/quant_transformer.py @@ -5,6 +5,7 @@ from __future__ import division from __future__ import print_function import os +import math import numpy as np import pandas as pd import copy @@ -43,7 +44,7 @@ class QuantTransformer(Model): d_feat=6, hidden_size=48, depth=5, - dropout=0.0, + pos_dropout=0.1, n_epochs=200, lr=0.001, metric="", @@ -63,7 +64,7 @@ class QuantTransformer(Model): self.d_feat = d_feat self.hidden_size = hidden_size self.depth = depth - self.dropout = dropout + self.pos_dropout = pos_dropout self.n_epochs = n_epochs self.lr = lr self.metric = metric @@ -94,7 +95,7 @@ class QuantTransformer(Model): d_feat, hidden_size, depth, - dropout, + pos_dropout, n_epochs, lr, metric, @@ -114,9 +115,10 @@ class QuantTransformer(Model): self.model = TransformerModel(d_feat=self.d_feat, embed_dim=self.hidden_size, - depth=self.depth) + depth=self.depth, + pos_dropout=pos_dropout) self.logger.info('model: {:}'.format(self.model)) - self.logger.info('model size: {:.3f} MB'.format(count_parameters_in_MB(self.model))) + self.logger.info('model size: {:.3f} MB'.format(count_parameters(self.model))) if optimizer.lower() == "adam": @@ -129,17 +131,10 @@ class QuantTransformer(Model): self.fitted = False self.model.to(self.device) - def mse(self, pred, label): - loss = (pred - label) ** 2 - return torch.mean(loss) - def loss_fn(self, pred, label): mask = ~torch.isnan(label) - if self.loss == "mse": - import pdb; pdb.set_trace() - print('--') - return self.mse(pred[mask], label[mask]) + return F.mse_loss(pred[mask], label[mask]) else: raise ValueError("unknown loss `{:}`".format(self.loss)) @@ -309,7 +304,7 @@ class Attention(nn.Module): self.num_heads = num_heads head_dim = dim // num_heads # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights - self.scale = qk_scale or head_dim ** -0.5 + self.scale = qk_scale or math.sqrt(head_dim) self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) @@ -333,17 +328,18 @@ class Attention(nn.Module): class Block(nn.Module): - def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., - drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, + attn_drop=0., mlp_drop=0., drop_path=0., + act_layer=nn.GELU, norm_layer=nn.LayerNorm): super(Block, self).__init__() self.norm1 = norm_layer(dim) self.attn = Attention( - dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop) + dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=mlp_drop) # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here self.drop_path = xlayers.DropPath(drop_path) if drop_path > 0. else nn.Identity() self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) - self.mlp = xlayers.MLP(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + self.mlp = xlayers.MLP(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=mlp_drop) def forward(self, x): x = x + self.drop_path(self.attn(self.norm1(x))) @@ -356,12 +352,13 @@ class SimpleEmbed(nn.Module): def __init__(self, d_feat, embed_dim): super(SimpleEmbed, self).__init__() self.d_feat = d_feat + self.embed_dim = embed_dim self.proj = nn.Linear(d_feat, embed_dim) def forward(self, x): x = x.reshape(len(x), self.d_feat, -1) # [N, F*T] -> [N, F, T] x = x.permute(0, 2, 1) # [N, F, T] -> [N, T, F] - out = self.proj(x) + out = self.proj(x) * math.sqrt(self.embed_dim) return out @@ -375,7 +372,7 @@ class TransformerModel(nn.Module): mlp_ratio: float = 4., qkv_bias: bool = True, qk_scale: Optional[float] = None, - drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None): + pos_dropout=0., mlp_drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None): """ Args: d_feat (int, tuple): input image size @@ -385,7 +382,8 @@ class TransformerModel(nn.Module): mlp_ratio (int): ratio of mlp hidden dim to embedding dim qkv_bias (bool): enable bias for qkv if True qk_scale (float): override default qk scale of head_dim ** -0.5 if set - drop_rate (float): dropout rate + pos_dropout (float): dropout rate for the positional embedding + mlp_drop_rate (float): the dropout rate for MLP layers in a block attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate norm_layer: (nn.Module): normalization layer @@ -398,14 +396,13 @@ class TransformerModel(nn.Module): self.input_embed = SimpleEmbed(d_feat, embed_dim=embed_dim) self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) - self.pos_embed = xlayers.PositionalEncoder(d_model=embed_dim, max_seq_len=65) - self.pos_drop = nn.Dropout(p=drop_rate) + self.pos_embed = xlayers.PositionalEncoder(d_model=embed_dim, max_seq_len=65, dropout=pos_dropout) dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule self.blocks = nn.ModuleList([ Block( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, - drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer) + attn_drop=attn_drop_rate, mlp_drop=mlp_drop_rate, drop_path=dpr[i], norm_layer=norm_layer) for i in range(depth)]) self.norm = norm_layer(embed_dim) @@ -431,7 +428,6 @@ class TransformerModel(nn.Module): cls_tokens = self.cls_token.expand(batch, -1, -1) # stole cls_tokens impl from Phil Wang, thanks feats_w_ct = torch.cat((cls_tokens, feats), dim=1) feats_w_tp = self.pos_embed(feats_w_ct) - feats_w_tp = self.pos_drop(feats_w_tp) xfeats = feats_w_tp for block in self.blocks: diff --git a/lib/utils/__init__.py b/lib/utils/__init__.py index bceb658..2c36e8a 100644 --- a/lib/utils/__init__.py +++ b/lib/utils/__init__.py @@ -1,6 +1,6 @@ from .evaluation_utils import obtain_accuracy from .gpu_manager import GPUManager -from .flop_benchmark import get_model_infos, count_parameters_in_MB +from .flop_benchmark import get_model_infos, count_parameters, count_parameters_in_MB from .affine_utils import normalize_points, denormalize_points from .affine_utils import identity2affine, solve2theta, affine2image from .hash_utils import get_md5_file