Update models

configs/qlib/workflow_config_transformer_basic_Alpha360.yaml

@@ -0,0 +1,87 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+market: &market all
+benchmark: &benchmark SH000300
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+    infer_processors:
+        - class: RobustZScoreNorm
+          kwargs:
+              fields_group: feature
+              clip_outlier: true
+        - class: Fillna
+          kwargs:
+              fields_group: feature
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSRankNorm
+          kwargs:
+              fields_group: label
+    label: ["Ref($close, -2) / Ref($close, -1) - 1"]
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy.strategy
+        kwargs:
+            topk: 50
+            n_drop: 5
+    backtest:
+        verbose: False
+        limit_threshold: 0.095
+        account: 100000000
+        benchmark: *benchmark
+        deal_price: close
+        open_cost: 0.0005
+        close_cost: 0.0015
+        min_cost: 5
+task:
+    model:
+        class: QuantTransformer
+        module_path: trade_models.quant_transformer
+        kwargs:
+            net_config:
+                name: basic
+                d_feat: 6
+                stem_dim: 48
+                embed_dims: [48, 48, 48, 48, 48]
+                num_heads: [4, 4, 4, 4, 4]
+                mlp_hidden_multipliers: [4, 4, 4, 4, 4]
+                qkv_bias: True
+                pos_drop: 0.1
+                other_drop: 0.1
+            opt_config:
+            loss: mse
+            GPU: 0
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha360
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: {}
+        - class: SignalMseRecord
+          module_path: qlib.contrib.workflow.record_temp
+          kwargs: {}
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

exps/trading/baselines.py

@@ -15,6 +15,7 @@
 # python exps/trading/baselines.py --alg TabNet     #
 #                                                   #
 # python exps/trading/baselines.py --alg Transformer#
+# python exps/trading/baselines.py --alg TSF-A      #
 #####################################################
 import sys
 import argparse
@@ -59,6 +60,7 @@ def retrieve_configs():
     alg2names["NAIVE-V1"] = "workflow_config_naive_v1_Alpha360.yaml"
     alg2names["NAIVE-V2"] = "workflow_config_naive_v2_Alpha360.yaml"
     alg2names["Transformer"] = "workflow_config_transformer_Alpha360.yaml"
+    alg2names["TSF-A"] = "workflow_config_transformer_basic_Alpha360.yaml"
 
     # find the yaml paths
     alg2paths = OrderedDict()

lib/procedures/q_exps.py

@@ -6,7 +6,7 @@ import inspect
 import os
 import pprint
 import logging
-
+from copy import deepcopy
 import qlib
 from qlib.utils import init_instance_by_config
 from qlib.workflow import R
@@ -33,11 +33,14 @@ def set_log_basic_config(filename=None, format=None, level=None):
     if format is None:
         format = C.logging_config["formatters"]["logger_format"]["format"]
 
+    # Remove all handlers associated with the root logger object.
+    for handler in logging.root.handlers[:]:
+        logging.root.removeHandler(handler)
     logging.basicConfig(filename=filename, format=format, level=level)
 
 
 def update_gpu(config, gpu):
-    config = config.copy()
+    config = deepcopy(config)
     if "task" in config and "model" in config["task"]:
         if "GPU" in config["task"]["model"]:
             config["task"]["model"]["GPU"] = gpu
@@ -59,13 +62,20 @@ def update_gpu(config, gpu):
 
 
 def update_market(config, market):
-    config = config.copy()
+    config = deepcopy(config.copy())
     config["market"] = market
     config["data_handler_config"]["instruments"] = market
     return config
 
 
-def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
+def run_exp(
+    task_config,
+    dataset,
+    experiment_name,
+    recorder_name,
+    uri,
+    model_obj_name="model.pkl",
+):
 
     model = init_instance_by_config(task_config["model"])
     model_fit_kwargs = dict(dataset=dataset)
@@ -80,6 +90,7 @@ def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
         # Setup log
         recorder_root_dir = R.get_recorder().get_local_dir()
         log_file = os.path.join(recorder_root_dir, "{:}.log".format(experiment_name))
+
         set_log_basic_config(log_file)
         logger = get_module_logger("q.run_exp")
         logger.info("task_config::\n{:}".format(pprint.pformat(task_config, indent=2)))
@@ -87,20 +98,29 @@ def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
         logger.info("dataset={:}".format(dataset))
 
         # Train model
-        R.log_params(**flatten_dict(task_config))
+        try:
-        if "save_path" in inspect.getfullargspec(model.fit).args:
+            model = R.load_object(model_obj_name)
-            model_fit_kwargs["save_path"] = os.path.join(recorder_root_dir, "model.ckp")
+            logger.info("[Find existing object from {:}]".format(model_obj_name))
-        elif "save_dir" in inspect.getfullargspec(model.fit).args:
+        except OSError:
-            model_fit_kwargs["save_dir"] = os.path.join(recorder_root_dir, "model-ckps")
+            R.log_params(**flatten_dict(task_config))
-        model.fit(**model_fit_kwargs)
+            if "save_path" in inspect.getfullargspec(model.fit).args:
+                model_fit_kwargs["save_path"] = os.path.join(
+                    recorder_root_dir, "model.ckp"
+                )
+            elif "save_dir" in inspect.getfullargspec(model.fit).args:
+                model_fit_kwargs["save_dir"] = os.path.join(
+                    recorder_root_dir, "model-ckps"
+                )
+            model.fit(**model_fit_kwargs)
+            R.save_objects(**{model_obj_name: model})
+        except:
+            raise ValueError("Something wrong.")
         # Get the recorder
         recorder = R.get_recorder()
-        R.save_objects(**{"model.pkl": model})
 
         # Generate records: prediction, backtest, and analysis
-        import pdb; pdb.set_trace()
         for record in task_config["record"]:
-            record = record.copy()
+            record = deepcopy(record)
             if record["class"] == "SignalRecord":
                 srconf = {"model": model, "dataset": dataset, "recorder": recorder}
                 record["kwargs"].update(srconf)

lib/trade_models/transformers.py

@@ -193,19 +193,15 @@ def get_transformer(config):
         raise ValueError("Invalid Configuration: {:}".format(config))
     name = config.get("name", "basic")
     if name == "basic":
-        model = TransformerModel(
+        model = SuperTransformer(
             d_feat=config.get("d_feat"),
-            embed_dim=config.get("embed_dim"),
+            stem_dim=config.get("stem_dim"),
-            depth=config.get("depth"),
+            embed_dims=config.get("embed_dims"),
             num_heads=config.get("num_heads"),
-            mlp_ratio=config.get("mlp_ratio"),
+            mlp_hidden_multipliers=config.get("mlp_hidden_multipliers"),
             qkv_bias=config.get("qkv_bias"),
-            qk_scale=config.get("qkv_scale"),
             pos_drop=config.get("pos_drop"),
-            mlp_drop_rate=config.get("mlp_drop_rate"),
+            other_drop=config.get("other_drop"),
-            attn_drop_rate=config.get("attn_drop_rate"),
-            drop_path_rate=config.get("drop_path_rate"),
-            norm_layer=config.get("norm_layer", None),
         )
     else:
         raise ValueError("Unknown model name: {:}".format(name))

lib/xlayers/super_module.py

@@ -14,6 +14,13 @@ IntSpaceType = Union[int, spaces.Integer, spaces.Categorical]
 BoolSpaceType = Union[bool, spaces.Categorical]
 
 
+class LayerOrder(Enum):
+    """This class defines the enumerations for order of operation in a residual or normalization-based layer."""
+
+    PreNorm = "pre-norm"
+    PostNorm = "post-norm"
+
+
 class SuperRunMode(Enum):
     """This class defines the enumerations for Super Model Running Mode."""
 

lib/xlayers/super_transformer.py

@@ -15,6 +15,7 @@ import torch.nn.functional as F
 import spaces
 from .super_module import IntSpaceType
 from .super_module import BoolSpaceType
+from .super_module import LayerOrder
 from .super_module import SuperModule
 from .super_linear import SuperMLPv2
 from .super_norm import SuperLayerNorm1D
@@ -30,7 +31,8 @@ class SuperTransformerEncoderLayer(SuperModule):
       - PyTorch Implementation: https://pytorch.org/docs/stable/_modules/torch/nn/modules/transformer.html#TransformerEncoderLayer
 
     Details:
-      MHA -> residual -> norm -> MLP -> residual -> norm
+      the original post-norm version: MHA -> residual -> norm -> MLP -> residual -> norm
+      the pre-norm version: norm -> MHA -> residual -> norm -> MLP -> residual
     """
 
     def __init__(
@@ -42,9 +44,10 @@ class SuperTransformerEncoderLayer(SuperModule):
         mlp_hidden_multiplier: IntSpaceType = 4,
         drop: Optional[float] = None,
         act_layer: Callable[[], nn.Module] = nn.GELU,
+        order: LayerOrder = LayerOrder.PreNorm,
     ):
         super(SuperTransformerEncoderLayer, self).__init__()
-        self.mha = SuperAttention(
+        mha = SuperAttention(
             input_dim,
             input_dim,
             num_heads=num_heads,
@@ -52,17 +55,33 @@ class SuperTransformerEncoderLayer(SuperModule):
             attn_drop=drop,
             proj_drop=drop,
         )
-        self.drop1 = nn.Dropout(drop or 0.0)
+        drop1 = nn.Dropout(drop or 0.0)
-        self.norm1 = SuperLayerNorm1D(input_dim)
+        norm1 = SuperLayerNorm1D(input_dim)
-        self.mlp = SuperMLPv2(
+        mlp = SuperMLPv2(
             input_dim,
             hidden_multiplier=mlp_hidden_multiplier,
             out_features=output_dim,
             act_layer=act_layer,
             drop=drop,
         )
-        self.drop2 = nn.Dropout(drop or 0.0)
+        drop2 = nn.Dropout(drop or 0.0)
-        self.norm2 = SuperLayerNorm1D(output_dim)
+        norm2 = SuperLayerNorm1D(output_dim)
+        if order is LayerOrder.PreNorm:
+            self.norm1 = norm1
+            self.mha = mha
+            self.drop1 = drop1
+            self.norm2 = norm2
+            self.mlp = mlp
+            self.drop2 = drop2
+        elif order is LayerOrder.PostNoem:
+            self.mha = mha
+            self.drop1 = drop1
+            self.norm1 = norm1
+            self.mlp = mlp
+            self.drop2 = drop2
+            self.norm2 = norm2
+        else:
+            raise ValueError("Unknown order: {:}".format(order))
 
     @property
     def abstract_search_space(self):
@@ -89,12 +108,18 @@ class SuperTransformerEncoderLayer(SuperModule):
         return self.forward_raw(input)
 
     def forward_raw(self, input: torch.Tensor) -> torch.Tensor:
-        # multi-head attention
+        if order is LayerOrder.PreNorm:
-        x = self.mha(input)
+            x = self.norm1(input)
-        x = x + self.drop1(x)
+            x = x + self.drop1(self.mha(x))
-        x = self.norm1(x)
+            x = self.norm2(x)
-        # feed-forward layer
+            x = x + self.drop2(self.mlp(x))
-        x = self.mlp(x)
+        elif order is LayerOrder.PostNoem:
-        x = x + self.drop2(x)
+            # multi-head attention
-        x = self.norm2(x)
+            x = x + self.drop1(self.mha(input))
+            x = self.norm1(x)
+            # feed-forward layer
+            x = x + self.drop2(self.mlp(x))
+            x = self.norm2(x)
+        else:
+            raise ValueError("Unknown order: {:}".format(order))
         return x