To re-org Q-results

exps/trading/baselines.py

@@ -65,16 +65,16 @@ def update_market(config, market):
 def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
 
     # model initiaiton
-    print('')
+    print("")
-    print('[{:}] - [{:}]: {:}'.format(experiment_name, recorder_name, uri))
+    print("[{:}] - [{:}]: {:}".format(experiment_name, recorder_name, uri))
-    print('dataset={:}'.format(dataset))
+    print("dataset={:}".format(dataset))
 
     model = init_instance_by_config(task_config["model"])
 
     # start exp
     with R.start(experiment_name=experiment_name, recorder_name=recorder_name, uri=uri):
 
-        log_file = R.get_recorder().root_uri / '{:}.log'.format(experiment_name)
+        log_file = R.get_recorder().root_uri / "{:}.log".format(experiment_name)
         set_log_basic_config(log_file)
 
         # train model
@@ -109,12 +109,14 @@ def main(xargs, exp_yaml):
     qlib.init(**config.get("qlib_init"))
     dataset_config = config.get("task").get("dataset")
     dataset = init_instance_by_config(dataset_config)
-    pprint('args: {:}'.format(xargs))
+    pprint("args: {:}".format(xargs))
     pprint(dataset_config)
     pprint(dataset)
 
     for irun in range(xargs.times):
-        run_exp(config.get("task"), dataset, xargs.alg, "recorder-{:02d}-{:02d}".format(irun, xargs.times), xargs.save_dir)
+        run_exp(
+            config.get("task"), dataset, xargs.alg, "recorder-{:02d}-{:02d}".format(irun, xargs.times), xargs.save_dir
+        )
 
 
 if __name__ == "__main__":

exps/trading/organize_results.py

@@ -0,0 +1,135 @@
+#####################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2021.02 #
+#####################################################
+# python exps/trading/organize_results.py
+#####################################################
+import sys, argparse
+import numpy as np
+from typing import List, Text
+from collections import defaultdict, OrderedDict
+from pathlib import Path
+from pprint import pprint
+import ruamel.yaml as yaml
+
+lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve()
+if str(lib_dir) not in sys.path:
+    sys.path.insert(0, str(lib_dir))
+
+import qlib
+from qlib.config import REG_CN
+from qlib.workflow import R
+
+
+class QResult:
+
+  def __init__(self):
+    self._result = defaultdict(list)
+
+  def append(self, key, value):
+    self._result[key].append(value)
+
+  @property
+  def result(self):
+    return self._result
+
+  def update(self, metrics, filter_keys=None):
+    for key, value in metrics.items():
+      if filter_keys is not None and key in filter_keys:
+        key = filter_keys[key]
+      elif filter_keys is not None:
+        continue
+      self.append(key, value)
+
+  @staticmethod
+  def full_str(xstr, space):
+    xformat = '{:' + str(space) + 's}'
+    return xformat.format(str(xstr))
+
+  def info(self, keys: List[Text], separate: Text = '', space: int = 25, show=True):
+    avaliable_keys = []
+    for key in keys:
+      if key not in self.result:
+        print('There are invalid key [{:}].'.format(key))
+      else:
+        avaliable_keys.append(key)
+    head_str = separate.join([self.full_str(x, space) for x in avaliable_keys])
+    values = []
+    for key in avaliable_keys:
+      current_values = self._result[key]
+      mean = np.mean(current_values)
+      std = np.std(current_values)
+      values.append('{:.4f} $\pm$ {:.4f}'.format(mean, std))
+    value_str = separate.join([self.full_str(x, space) for x in values])
+    if show:
+      print(head_str)
+      print(value_str)
+    else:
+      return head_str, value_str
+
+
+def compare_results(heads, values, names, space=10):
+  for idx, x in enumerate(heads):
+    assert x == heads[0], '[{:}] {:} vs {:}'.format(idx, x, heads[0])
+  new_head = QResult.full_str('Name', space) + heads[0]
+  print(new_head)
+  for name, value in zip(names, values):
+    xline = QResult.full_str(name, space) + value
+    print(xline)
+
+
+def filter_finished(recorders):
+    returned_recorders = dict()
+    not_finished = 0
+    for key, recorder in recorders.items():
+        if recorder.status == "FINISHED":
+            returned_recorders[key] = recorder
+        else:
+            not_finished += 1
+    return returned_recorders, not_finished
+
+
+def main(xargs):
+    R.reset_default_uri(xargs.save_dir)
+    experiments = R.list_experiments()
+
+    key_map = {"IC": "IC",
+               "ICIR": "ICIR",
+               "Rank IC": "Rank_IC",
+               "Rank ICIR": "Rank_ICIR",
+               "excess_return_with_cost.annualized_return": "Annualized_Return",
+               "excess_return_with_cost.information_ratio": "Information_Ratio",
+               "excess_return_with_cost.max_drawdown": "Max_Drawdown"}
+    all_keys = list(key_map.values())
+
+    print("There are {:} experiments.".format(len(experiments)))
+    head_strs, value_strs, names = [], [], []
+    for idx, (key, experiment) in enumerate(experiments.items()):
+        if experiment.id == '0':
+          continue
+        recorders = experiment.list_recorders()
+        recorders, not_finished = filter_finished(recorders)
+        print(
+            "====>>>> {:02d}/{:02d}-th experiment {:9s} has {:02d}/{:02d} finished recorders.".format(
+                idx, len(experiments), experiment.name, len(recorders), len(recorders) + not_finished
+            )
+        )
+        result = QResult()
+        for recorder_id, recorder in recorders.items():
+          result.update(recorder.list_metrics(), key_map)
+        head_str, value_str = result.info(all_keys, show=False)
+        head_strs.append(head_str)
+        value_strs.append(value_str)
+        names.append(experiment.name)
+    compare_results(head_strs, value_strs, names, space=10)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser("Show Results")
+    parser.add_argument("--save_dir", type=str, default="./outputs/qlib-baselines", help="The checkpoint directory.")
+    args = parser.parse_args()
+
+    provider_uri = "~/.qlib/qlib_data/cn_data"
+    qlib.init(provider_uri=provider_uri, region=REG_CN)
+
+    main(args)

exps/trading/workflow_tt.py

@@ -4,7 +4,7 @@
 # Refer to:
 # - https://github.com/microsoft/qlib/blob/main/examples/workflow_by_code.ipynb
 # - https://github.com/microsoft/qlib/blob/main/examples/workflow_by_code.py
-# python exps/trading/workflow_tt.py
+# python exps/trading/workflow_tt.py --market all
 #####################################################
 import sys, argparse
 from pathlib import Path
@@ -102,10 +102,9 @@ def main(xargs):
 
     task = dict(model=model_config, dataset=dataset_config, record=record_config)
 
-
     # start exp to train model
-    with R.start(experiment_name="tt_model", uri=xargs.save_dir):
+    with R.start(experiment_name="tt_model", uri=xargs.save_dir + "-" + xargs.market):
-        set_log_basic_config(R.get_recorder().root_uri / 'log.log')
+        set_log_basic_config(R.get_recorder().root_uri / "log.log")
 
         model = init_instance_by_config(model_config)
         dataset = init_instance_by_config(dataset_config)
@@ -138,7 +137,7 @@ if __name__ == "__main__":
     parser.add_argument("--market", type=str, default="csi300", help="The market indicator.")
     args = parser.parse_args()
 
-    provider_uri = "~/.qlib/qlib_data/cn_data"  # target_dir
+    provider_uri = "~/.qlib/qlib_data/cn_data"
     qlib.init(provider_uri=provider_uri, region=REG_CN)
 
     main(args)

lib/layers/positional_embedding.py

@@ -24,12 +24,12 @@ class PositionalEncoder(nn.Module):
         else:
           pe[pos, i] = math.cos(value)
     pe = pe.unsqueeze(0)
+    self.dropout = nn.Dropout(p=dropout)
     self.register_buffer('pe', pe)
  
   
   def forward(self, x):
     batch, seq, fdim = x.shape[:3]
     embeddings = self.pe[:, :seq, :fdim]
-    import pdb; pdb.set_trace()
     outs = self.dropout(x + embeddings)
-    return x + embeddings
+    return outs

lib/trade_models/quant_transformer.py

@@ -5,6 +5,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import math
 import numpy as np
 import pandas as pd
 import copy
@@ -43,7 +44,7 @@ class QuantTransformer(Model):
     d_feat=6,
     hidden_size=48,
     depth=5,
-    dropout=0.0,
+    pos_dropout=0.1,
     n_epochs=200,
     lr=0.001,
     metric="",
@@ -63,7 +64,7 @@ class QuantTransformer(Model):
     self.d_feat = d_feat
     self.hidden_size = hidden_size
     self.depth = depth
-    self.dropout = dropout
+    self.pos_dropout = pos_dropout
     self.n_epochs = n_epochs
     self.lr = lr
     self.metric = metric
@@ -94,7 +95,7 @@ class QuantTransformer(Model):
         d_feat,
         hidden_size,
         depth,
-        dropout,
+        pos_dropout,
         n_epochs,
         lr,
         metric,
@@ -114,9 +115,10 @@ class QuantTransformer(Model):
 
     self.model = TransformerModel(d_feat=self.d_feat,
                                   embed_dim=self.hidden_size,
-                                  depth=self.depth)
+                                  depth=self.depth,
+                                  pos_dropout=pos_dropout)
     self.logger.info('model: {:}'.format(self.model))
-    self.logger.info('model size: {:.3f} MB'.format(count_parameters_in_MB(self.model)))
+    self.logger.info('model size: {:.3f} MB'.format(count_parameters(self.model)))
   
     
     if optimizer.lower() == "adam":
@@ -129,17 +131,10 @@ class QuantTransformer(Model):
     self.fitted = False
     self.model.to(self.device)
 
-  def mse(self, pred, label):
-    loss = (pred - label) ** 2
-    return torch.mean(loss)
-
   def loss_fn(self, pred, label):
     mask = ~torch.isnan(label)
-
     if self.loss == "mse":
-      import pdb; pdb.set_trace()
+      return F.mse_loss(pred[mask], label[mask])
-      print('--')
-      return self.mse(pred[mask], label[mask])
     else:
       raise ValueError("unknown loss `{:}`".format(self.loss))
 
@@ -309,7 +304,7 @@ class Attention(nn.Module):
     self.num_heads = num_heads
     head_dim = dim // num_heads
     # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights
-    self.scale = qk_scale or head_dim ** -0.5
+    self.scale = qk_scale or math.sqrt(head_dim)
 
     self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
     self.attn_drop = nn.Dropout(attn_drop)
@@ -333,17 +328,18 @@ class Attention(nn.Module):
 
 class Block(nn.Module):
 
-  def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
+  def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None,
-         drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):
+               attn_drop=0., mlp_drop=0., drop_path=0.,
+               act_layer=nn.GELU, norm_layer=nn.LayerNorm):
     super(Block, self).__init__()
     self.norm1 = norm_layer(dim)
     self.attn = Attention(
-      dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+      dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=mlp_drop)
     # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
     self.drop_path = xlayers.DropPath(drop_path) if drop_path > 0. else nn.Identity()
     self.norm2 = norm_layer(dim)
     mlp_hidden_dim = int(dim * mlp_ratio)
-    self.mlp = xlayers.MLP(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+    self.mlp = xlayers.MLP(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=mlp_drop)
 
   def forward(self, x):
     x = x + self.drop_path(self.attn(self.norm1(x)))
@@ -356,12 +352,13 @@ class SimpleEmbed(nn.Module):
   def __init__(self, d_feat, embed_dim):
     super(SimpleEmbed, self).__init__()
     self.d_feat = d_feat
+    self.embed_dim = embed_dim
     self.proj = nn.Linear(d_feat, embed_dim)
 
   def forward(self, x):
     x = x.reshape(len(x), self.d_feat, -1)  # [N, F*T] -> [N, F, T]
     x = x.permute(0, 2, 1)                  # [N, F, T] -> [N, T, F]
-    out = self.proj(x)
+    out = self.proj(x) * math.sqrt(self.embed_dim)
     return out
 
 
@@ -375,7 +372,7 @@ class TransformerModel(nn.Module):
          mlp_ratio: float = 4.,
          qkv_bias: bool = True,
          qk_scale: Optional[float] = None,
-         drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None):
+         pos_dropout=0., mlp_drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None):
     """
     Args:
       d_feat (int, tuple): input image size
@@ -385,7 +382,8 @@ class TransformerModel(nn.Module):
       mlp_ratio (int): ratio of mlp hidden dim to embedding dim
       qkv_bias (bool): enable bias for qkv if True
       qk_scale (float): override default qk scale of head_dim ** -0.5 if set
-      drop_rate (float): dropout rate
+      pos_dropout (float): dropout rate for the positional embedding
+      mlp_drop_rate (float): the dropout rate for MLP layers in a block
       attn_drop_rate (float): attention dropout rate
       drop_path_rate (float): stochastic depth rate
       norm_layer: (nn.Module): normalization layer
@@ -398,14 +396,13 @@ class TransformerModel(nn.Module):
     self.input_embed = SimpleEmbed(d_feat, embed_dim=embed_dim)
 
     self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
-    self.pos_embed = xlayers.PositionalEncoder(d_model=embed_dim, max_seq_len=65)
+    self.pos_embed = xlayers.PositionalEncoder(d_model=embed_dim, max_seq_len=65, dropout=pos_dropout)
-    self.pos_drop = nn.Dropout(p=drop_rate)
 
     dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
     self.blocks = nn.ModuleList([
       Block(
         dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
-        drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer)
+        attn_drop=attn_drop_rate, mlp_drop=mlp_drop_rate, drop_path=dpr[i], norm_layer=norm_layer)
       for i in range(depth)])
     self.norm = norm_layer(embed_dim)
 
@@ -431,7 +428,6 @@ class TransformerModel(nn.Module):
     cls_tokens = self.cls_token.expand(batch, -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
     feats_w_ct = torch.cat((cls_tokens, feats), dim=1)
     feats_w_tp = self.pos_embed(feats_w_ct)
-    feats_w_tp = self.pos_drop(feats_w_tp)
 
     xfeats = feats_w_tp
     for block in self.blocks:

lib/utils/__init__.py

@@ -1,6 +1,6 @@
 from .evaluation_utils import obtain_accuracy
 from .gpu_manager      import GPUManager
-from .flop_benchmark   import get_model_infos, count_parameters_in_MB
+from .flop_benchmark   import get_model_infos, count_parameters, count_parameters_in_MB
 from .affine_utils     import normalize_points, denormalize_points
 from .affine_utils     import identity2affine, solve2theta, affine2image
 from .hash_utils       import get_md5_file