To re-org Q-results
This commit is contained in:
parent
cc28e1589e
commit
511e2443d0
@ -1 +1 @@
|
||||
Subproject commit 49697b1f1568608e3077450b72fe3ed5b92ec1e5
|
||||
Subproject commit 91fd53ab4d0724df73ccf8855ed83b6e1760bb08
|
@ -65,16 +65,16 @@ def update_market(config, market):
|
||||
def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
|
||||
|
||||
# model initiaiton
|
||||
print('')
|
||||
print('[{:}] - [{:}]: {:}'.format(experiment_name, recorder_name, uri))
|
||||
print('dataset={:}'.format(dataset))
|
||||
print("")
|
||||
print("[{:}] - [{:}]: {:}".format(experiment_name, recorder_name, uri))
|
||||
print("dataset={:}".format(dataset))
|
||||
|
||||
model = init_instance_by_config(task_config["model"])
|
||||
|
||||
# start exp
|
||||
with R.start(experiment_name=experiment_name, recorder_name=recorder_name, uri=uri):
|
||||
|
||||
log_file = R.get_recorder().root_uri / '{:}.log'.format(experiment_name)
|
||||
log_file = R.get_recorder().root_uri / "{:}.log".format(experiment_name)
|
||||
set_log_basic_config(log_file)
|
||||
|
||||
# train model
|
||||
@ -109,12 +109,14 @@ def main(xargs, exp_yaml):
|
||||
qlib.init(**config.get("qlib_init"))
|
||||
dataset_config = config.get("task").get("dataset")
|
||||
dataset = init_instance_by_config(dataset_config)
|
||||
pprint('args: {:}'.format(xargs))
|
||||
pprint("args: {:}".format(xargs))
|
||||
pprint(dataset_config)
|
||||
pprint(dataset)
|
||||
|
||||
for irun in range(xargs.times):
|
||||
run_exp(config.get("task"), dataset, xargs.alg, "recorder-{:02d}-{:02d}".format(irun, xargs.times), xargs.save_dir)
|
||||
run_exp(
|
||||
config.get("task"), dataset, xargs.alg, "recorder-{:02d}-{:02d}".format(irun, xargs.times), xargs.save_dir
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
135
exps/trading/organize_results.py
Normal file
135
exps/trading/organize_results.py
Normal file
@ -0,0 +1,135 @@
|
||||
#####################################################
|
||||
# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2021.02 #
|
||||
#####################################################
|
||||
# python exps/trading/organize_results.py
|
||||
#####################################################
|
||||
import sys, argparse
|
||||
import numpy as np
|
||||
from typing import List, Text
|
||||
from collections import defaultdict, OrderedDict
|
||||
from pathlib import Path
|
||||
from pprint import pprint
|
||||
import ruamel.yaml as yaml
|
||||
|
||||
lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve()
|
||||
if str(lib_dir) not in sys.path:
|
||||
sys.path.insert(0, str(lib_dir))
|
||||
|
||||
import qlib
|
||||
from qlib.config import REG_CN
|
||||
from qlib.workflow import R
|
||||
|
||||
|
||||
class QResult:
|
||||
|
||||
def __init__(self):
|
||||
self._result = defaultdict(list)
|
||||
|
||||
def append(self, key, value):
|
||||
self._result[key].append(value)
|
||||
|
||||
@property
|
||||
def result(self):
|
||||
return self._result
|
||||
|
||||
def update(self, metrics, filter_keys=None):
|
||||
for key, value in metrics.items():
|
||||
if filter_keys is not None and key in filter_keys:
|
||||
key = filter_keys[key]
|
||||
elif filter_keys is not None:
|
||||
continue
|
||||
self.append(key, value)
|
||||
|
||||
@staticmethod
|
||||
def full_str(xstr, space):
|
||||
xformat = '{:' + str(space) + 's}'
|
||||
return xformat.format(str(xstr))
|
||||
|
||||
def info(self, keys: List[Text], separate: Text = '', space: int = 25, show=True):
|
||||
avaliable_keys = []
|
||||
for key in keys:
|
||||
if key not in self.result:
|
||||
print('There are invalid key [{:}].'.format(key))
|
||||
else:
|
||||
avaliable_keys.append(key)
|
||||
head_str = separate.join([self.full_str(x, space) for x in avaliable_keys])
|
||||
values = []
|
||||
for key in avaliable_keys:
|
||||
current_values = self._result[key]
|
||||
mean = np.mean(current_values)
|
||||
std = np.std(current_values)
|
||||
values.append('{:.4f} $\pm$ {:.4f}'.format(mean, std))
|
||||
value_str = separate.join([self.full_str(x, space) for x in values])
|
||||
if show:
|
||||
print(head_str)
|
||||
print(value_str)
|
||||
else:
|
||||
return head_str, value_str
|
||||
|
||||
|
||||
def compare_results(heads, values, names, space=10):
|
||||
for idx, x in enumerate(heads):
|
||||
assert x == heads[0], '[{:}] {:} vs {:}'.format(idx, x, heads[0])
|
||||
new_head = QResult.full_str('Name', space) + heads[0]
|
||||
print(new_head)
|
||||
for name, value in zip(names, values):
|
||||
xline = QResult.full_str(name, space) + value
|
||||
print(xline)
|
||||
|
||||
|
||||
def filter_finished(recorders):
|
||||
returned_recorders = dict()
|
||||
not_finished = 0
|
||||
for key, recorder in recorders.items():
|
||||
if recorder.status == "FINISHED":
|
||||
returned_recorders[key] = recorder
|
||||
else:
|
||||
not_finished += 1
|
||||
return returned_recorders, not_finished
|
||||
|
||||
|
||||
def main(xargs):
|
||||
R.reset_default_uri(xargs.save_dir)
|
||||
experiments = R.list_experiments()
|
||||
|
||||
key_map = {"IC": "IC",
|
||||
"ICIR": "ICIR",
|
||||
"Rank IC": "Rank_IC",
|
||||
"Rank ICIR": "Rank_ICIR",
|
||||
"excess_return_with_cost.annualized_return": "Annualized_Return",
|
||||
"excess_return_with_cost.information_ratio": "Information_Ratio",
|
||||
"excess_return_with_cost.max_drawdown": "Max_Drawdown"}
|
||||
all_keys = list(key_map.values())
|
||||
|
||||
print("There are {:} experiments.".format(len(experiments)))
|
||||
head_strs, value_strs, names = [], [], []
|
||||
for idx, (key, experiment) in enumerate(experiments.items()):
|
||||
if experiment.id == '0':
|
||||
continue
|
||||
recorders = experiment.list_recorders()
|
||||
recorders, not_finished = filter_finished(recorders)
|
||||
print(
|
||||
"====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders.".format(
|
||||
idx, len(experiments), experiment.name, len(recorders), len(recorders) + not_finished
|
||||
)
|
||||
)
|
||||
result = QResult()
|
||||
for recorder_id, recorder in recorders.items():
|
||||
result.update(recorder.list_metrics(), key_map)
|
||||
head_str, value_str = result.info(all_keys, show=False)
|
||||
head_strs.append(head_str)
|
||||
value_strs.append(value_str)
|
||||
names.append(experiment.name)
|
||||
compare_results(head_strs, value_strs, names, space=10)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
parser = argparse.ArgumentParser("Show Results")
|
||||
parser.add_argument("--save_dir", type=str, default="./outputs/qlib-baselines", help="The checkpoint directory.")
|
||||
args = parser.parse_args()
|
||||
|
||||
provider_uri = "~/.qlib/qlib_data/cn_data"
|
||||
qlib.init(provider_uri=provider_uri, region=REG_CN)
|
||||
|
||||
main(args)
|
@ -4,7 +4,7 @@
|
||||
# Refer to:
|
||||
# - https://github.com/microsoft/qlib/blob/main/examples/workflow_by_code.ipynb
|
||||
# - https://github.com/microsoft/qlib/blob/main/examples/workflow_by_code.py
|
||||
# python exps/trading/workflow_tt.py
|
||||
# python exps/trading/workflow_tt.py --market all
|
||||
#####################################################
|
||||
import sys, argparse
|
||||
from pathlib import Path
|
||||
@ -102,10 +102,9 @@ def main(xargs):
|
||||
|
||||
task = dict(model=model_config, dataset=dataset_config, record=record_config)
|
||||
|
||||
|
||||
# start exp to train model
|
||||
with R.start(experiment_name="tt_model", uri=xargs.save_dir):
|
||||
set_log_basic_config(R.get_recorder().root_uri / 'log.log')
|
||||
with R.start(experiment_name="tt_model", uri=xargs.save_dir + "-" + xargs.market):
|
||||
set_log_basic_config(R.get_recorder().root_uri / "log.log")
|
||||
|
||||
model = init_instance_by_config(model_config)
|
||||
dataset = init_instance_by_config(dataset_config)
|
||||
@ -138,7 +137,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--market", type=str, default="csi300", help="The market indicator.")
|
||||
args = parser.parse_args()
|
||||
|
||||
provider_uri = "~/.qlib/qlib_data/cn_data" # target_dir
|
||||
provider_uri = "~/.qlib/qlib_data/cn_data"
|
||||
qlib.init(provider_uri=provider_uri, region=REG_CN)
|
||||
|
||||
main(args)
|
||||
|
@ -24,12 +24,12 @@ class PositionalEncoder(nn.Module):
|
||||
else:
|
||||
pe[pos, i] = math.cos(value)
|
||||
pe = pe.unsqueeze(0)
|
||||
self.dropout = nn.Dropout(p=dropout)
|
||||
self.register_buffer('pe', pe)
|
||||
|
||||
|
||||
def forward(self, x):
|
||||
batch, seq, fdim = x.shape[:3]
|
||||
embeddings = self.pe[:, :seq, :fdim]
|
||||
import pdb; pdb.set_trace()
|
||||
outs = self.dropout(x + embeddings)
|
||||
return x + embeddings
|
||||
return outs
|
||||
|
@ -5,6 +5,7 @@ from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import math
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import copy
|
||||
@ -43,7 +44,7 @@ class QuantTransformer(Model):
|
||||
d_feat=6,
|
||||
hidden_size=48,
|
||||
depth=5,
|
||||
dropout=0.0,
|
||||
pos_dropout=0.1,
|
||||
n_epochs=200,
|
||||
lr=0.001,
|
||||
metric="",
|
||||
@ -63,7 +64,7 @@ class QuantTransformer(Model):
|
||||
self.d_feat = d_feat
|
||||
self.hidden_size = hidden_size
|
||||
self.depth = depth
|
||||
self.dropout = dropout
|
||||
self.pos_dropout = pos_dropout
|
||||
self.n_epochs = n_epochs
|
||||
self.lr = lr
|
||||
self.metric = metric
|
||||
@ -94,7 +95,7 @@ class QuantTransformer(Model):
|
||||
d_feat,
|
||||
hidden_size,
|
||||
depth,
|
||||
dropout,
|
||||
pos_dropout,
|
||||
n_epochs,
|
||||
lr,
|
||||
metric,
|
||||
@ -114,9 +115,10 @@ class QuantTransformer(Model):
|
||||
|
||||
self.model = TransformerModel(d_feat=self.d_feat,
|
||||
embed_dim=self.hidden_size,
|
||||
depth=self.depth)
|
||||
depth=self.depth,
|
||||
pos_dropout=pos_dropout)
|
||||
self.logger.info('model: {:}'.format(self.model))
|
||||
self.logger.info('model size: {:.3f} MB'.format(count_parameters_in_MB(self.model)))
|
||||
self.logger.info('model size: {:.3f} MB'.format(count_parameters(self.model)))
|
||||
|
||||
|
||||
if optimizer.lower() == "adam":
|
||||
@ -129,17 +131,10 @@ class QuantTransformer(Model):
|
||||
self.fitted = False
|
||||
self.model.to(self.device)
|
||||
|
||||
def mse(self, pred, label):
|
||||
loss = (pred - label) ** 2
|
||||
return torch.mean(loss)
|
||||
|
||||
def loss_fn(self, pred, label):
|
||||
mask = ~torch.isnan(label)
|
||||
|
||||
if self.loss == "mse":
|
||||
import pdb; pdb.set_trace()
|
||||
print('--')
|
||||
return self.mse(pred[mask], label[mask])
|
||||
return F.mse_loss(pred[mask], label[mask])
|
||||
else:
|
||||
raise ValueError("unknown loss `{:}`".format(self.loss))
|
||||
|
||||
@ -309,7 +304,7 @@ class Attention(nn.Module):
|
||||
self.num_heads = num_heads
|
||||
head_dim = dim // num_heads
|
||||
# NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights
|
||||
self.scale = qk_scale or head_dim ** -0.5
|
||||
self.scale = qk_scale or math.sqrt(head_dim)
|
||||
|
||||
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
|
||||
self.attn_drop = nn.Dropout(attn_drop)
|
||||
@ -333,17 +328,18 @@ class Attention(nn.Module):
|
||||
|
||||
class Block(nn.Module):
|
||||
|
||||
def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
|
||||
drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):
|
||||
def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None,
|
||||
attn_drop=0., mlp_drop=0., drop_path=0.,
|
||||
act_layer=nn.GELU, norm_layer=nn.LayerNorm):
|
||||
super(Block, self).__init__()
|
||||
self.norm1 = norm_layer(dim)
|
||||
self.attn = Attention(
|
||||
dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
|
||||
dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=mlp_drop)
|
||||
# NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
|
||||
self.drop_path = xlayers.DropPath(drop_path) if drop_path > 0. else nn.Identity()
|
||||
self.norm2 = norm_layer(dim)
|
||||
mlp_hidden_dim = int(dim * mlp_ratio)
|
||||
self.mlp = xlayers.MLP(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
|
||||
self.mlp = xlayers.MLP(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=mlp_drop)
|
||||
|
||||
def forward(self, x):
|
||||
x = x + self.drop_path(self.attn(self.norm1(x)))
|
||||
@ -356,12 +352,13 @@ class SimpleEmbed(nn.Module):
|
||||
def __init__(self, d_feat, embed_dim):
|
||||
super(SimpleEmbed, self).__init__()
|
||||
self.d_feat = d_feat
|
||||
self.embed_dim = embed_dim
|
||||
self.proj = nn.Linear(d_feat, embed_dim)
|
||||
|
||||
def forward(self, x):
|
||||
x = x.reshape(len(x), self.d_feat, -1) # [N, F*T] -> [N, F, T]
|
||||
x = x.permute(0, 2, 1) # [N, F, T] -> [N, T, F]
|
||||
out = self.proj(x)
|
||||
out = self.proj(x) * math.sqrt(self.embed_dim)
|
||||
return out
|
||||
|
||||
|
||||
@ -375,7 +372,7 @@ class TransformerModel(nn.Module):
|
||||
mlp_ratio: float = 4.,
|
||||
qkv_bias: bool = True,
|
||||
qk_scale: Optional[float] = None,
|
||||
drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None):
|
||||
pos_dropout=0., mlp_drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None):
|
||||
"""
|
||||
Args:
|
||||
d_feat (int, tuple): input image size
|
||||
@ -385,7 +382,8 @@ class TransformerModel(nn.Module):
|
||||
mlp_ratio (int): ratio of mlp hidden dim to embedding dim
|
||||
qkv_bias (bool): enable bias for qkv if True
|
||||
qk_scale (float): override default qk scale of head_dim ** -0.5 if set
|
||||
drop_rate (float): dropout rate
|
||||
pos_dropout (float): dropout rate for the positional embedding
|
||||
mlp_drop_rate (float): the dropout rate for MLP layers in a block
|
||||
attn_drop_rate (float): attention dropout rate
|
||||
drop_path_rate (float): stochastic depth rate
|
||||
norm_layer: (nn.Module): normalization layer
|
||||
@ -398,14 +396,13 @@ class TransformerModel(nn.Module):
|
||||
self.input_embed = SimpleEmbed(d_feat, embed_dim=embed_dim)
|
||||
|
||||
self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
|
||||
self.pos_embed = xlayers.PositionalEncoder(d_model=embed_dim, max_seq_len=65)
|
||||
self.pos_drop = nn.Dropout(p=drop_rate)
|
||||
self.pos_embed = xlayers.PositionalEncoder(d_model=embed_dim, max_seq_len=65, dropout=pos_dropout)
|
||||
|
||||
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule
|
||||
self.blocks = nn.ModuleList([
|
||||
Block(
|
||||
dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
|
||||
drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer)
|
||||
attn_drop=attn_drop_rate, mlp_drop=mlp_drop_rate, drop_path=dpr[i], norm_layer=norm_layer)
|
||||
for i in range(depth)])
|
||||
self.norm = norm_layer(embed_dim)
|
||||
|
||||
@ -431,7 +428,6 @@ class TransformerModel(nn.Module):
|
||||
cls_tokens = self.cls_token.expand(batch, -1, -1) # stole cls_tokens impl from Phil Wang, thanks
|
||||
feats_w_ct = torch.cat((cls_tokens, feats), dim=1)
|
||||
feats_w_tp = self.pos_embed(feats_w_ct)
|
||||
feats_w_tp = self.pos_drop(feats_w_tp)
|
||||
|
||||
xfeats = feats_w_tp
|
||||
for block in self.blocks:
|
||||
|
@ -1,6 +1,6 @@
|
||||
from .evaluation_utils import obtain_accuracy
|
||||
from .gpu_manager import GPUManager
|
||||
from .flop_benchmark import get_model_infos, count_parameters_in_MB
|
||||
from .flop_benchmark import get_model_infos, count_parameters, count_parameters_in_MB
|
||||
from .affine_utils import normalize_points, denormalize_points
|
||||
from .affine_utils import identity2affine, solve2theta, affine2image
|
||||
from .hash_utils import get_md5_file
|
||||
|
Loading…
Reference in New Issue
Block a user