diff --git a/.latent-data/qlib b/.latent-data/qlib index 2b74b4d..0a0c6a3 160000 --- a/.latent-data/qlib +++ b/.latent-data/qlib @@ -1 +1 @@ -Subproject commit 2b74b4dfa4a6996ab6135873c0329022a1b9626b +Subproject commit 0a0c6a3185ac6bcec38b756f039b9ccc64b41827 diff --git a/configs/qlib/workflow_config_transformer_basic_Alpha360.yaml b/configs/qlib/workflow_config_transformer_basic_Alpha360.yaml new file mode 100644 index 0000000..fe44109 --- /dev/null +++ b/configs/qlib/workflow_config_transformer_basic_Alpha360.yaml @@ -0,0 +1,87 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn +market: &market all +benchmark: &benchmark SH000300 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market + infer_processors: + - class: RobustZScoreNorm + kwargs: + fields_group: feature + clip_outlier: true + - class: Fillna + kwargs: + fields_group: feature + learn_processors: + - class: DropnaLabel + - class: CSRankNorm + kwargs: + fields_group: label + label: ["Ref($close, -2) / Ref($close, -1) - 1"] +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy.strategy + kwargs: + topk: 50 + n_drop: 5 + backtest: + verbose: False + limit_threshold: 0.095 + account: 100000000 + benchmark: *benchmark + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: QuantTransformer + module_path: trade_models.quant_transformer + kwargs: + net_config: + name: basic + d_feat: 6 + stem_dim: 48 + embed_dims: [48, 48, 48, 48, 48] + num_heads: [4, 4, 4, 4, 4] + mlp_hidden_multipliers: [4, 4, 4, 4, 4] + qkv_bias: True + pos_drop: 0.1 + other_drop: 0.1 + opt_config: + loss: mse + GPU: 0 + dataset: + class: DatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha360 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: {} + - class: SignalMseRecord + module_path: qlib.contrib.workflow.record_temp + kwargs: {} + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config diff --git a/exps/trading/baselines.py b/exps/trading/baselines.py index 854b96e..0d6ac44 100644 --- a/exps/trading/baselines.py +++ b/exps/trading/baselines.py @@ -15,6 +15,7 @@ # python exps/trading/baselines.py --alg TabNet # # # # python exps/trading/baselines.py --alg Transformer# +# python exps/trading/baselines.py --alg TSF-A # ##################################################### import sys import argparse @@ -59,6 +60,7 @@ def retrieve_configs(): alg2names["NAIVE-V1"] = "workflow_config_naive_v1_Alpha360.yaml" alg2names["NAIVE-V2"] = "workflow_config_naive_v2_Alpha360.yaml" alg2names["Transformer"] = "workflow_config_transformer_Alpha360.yaml" + alg2names["TSF-A"] = "workflow_config_transformer_basic_Alpha360.yaml" # find the yaml paths alg2paths = OrderedDict() diff --git a/lib/procedures/q_exps.py b/lib/procedures/q_exps.py index befcae9..f31e41a 100644 --- a/lib/procedures/q_exps.py +++ b/lib/procedures/q_exps.py @@ -6,7 +6,7 @@ import inspect import os import pprint import logging - +from copy import deepcopy import qlib from qlib.utils import init_instance_by_config from qlib.workflow import R @@ -33,11 +33,14 @@ def set_log_basic_config(filename=None, format=None, level=None): if format is None: format = C.logging_config["formatters"]["logger_format"]["format"] + # Remove all handlers associated with the root logger object. + for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) logging.basicConfig(filename=filename, format=format, level=level) def update_gpu(config, gpu): - config = config.copy() + config = deepcopy(config) if "task" in config and "model" in config["task"]: if "GPU" in config["task"]["model"]: config["task"]["model"]["GPU"] = gpu @@ -59,13 +62,20 @@ def update_gpu(config, gpu): def update_market(config, market): - config = config.copy() + config = deepcopy(config.copy()) config["market"] = market config["data_handler_config"]["instruments"] = market return config -def run_exp(task_config, dataset, experiment_name, recorder_name, uri): +def run_exp( + task_config, + dataset, + experiment_name, + recorder_name, + uri, + model_obj_name="model.pkl", +): model = init_instance_by_config(task_config["model"]) model_fit_kwargs = dict(dataset=dataset) @@ -80,6 +90,7 @@ def run_exp(task_config, dataset, experiment_name, recorder_name, uri): # Setup log recorder_root_dir = R.get_recorder().get_local_dir() log_file = os.path.join(recorder_root_dir, "{:}.log".format(experiment_name)) + set_log_basic_config(log_file) logger = get_module_logger("q.run_exp") logger.info("task_config::\n{:}".format(pprint.pformat(task_config, indent=2))) @@ -87,20 +98,29 @@ def run_exp(task_config, dataset, experiment_name, recorder_name, uri): logger.info("dataset={:}".format(dataset)) # Train model - R.log_params(**flatten_dict(task_config)) - if "save_path" in inspect.getfullargspec(model.fit).args: - model_fit_kwargs["save_path"] = os.path.join(recorder_root_dir, "model.ckp") - elif "save_dir" in inspect.getfullargspec(model.fit).args: - model_fit_kwargs["save_dir"] = os.path.join(recorder_root_dir, "model-ckps") - model.fit(**model_fit_kwargs) + try: + model = R.load_object(model_obj_name) + logger.info("[Find existing object from {:}]".format(model_obj_name)) + except OSError: + R.log_params(**flatten_dict(task_config)) + if "save_path" in inspect.getfullargspec(model.fit).args: + model_fit_kwargs["save_path"] = os.path.join( + recorder_root_dir, "model.ckp" + ) + elif "save_dir" in inspect.getfullargspec(model.fit).args: + model_fit_kwargs["save_dir"] = os.path.join( + recorder_root_dir, "model-ckps" + ) + model.fit(**model_fit_kwargs) + R.save_objects(**{model_obj_name: model}) + except: + raise ValueError("Something wrong.") # Get the recorder recorder = R.get_recorder() - R.save_objects(**{"model.pkl": model}) # Generate records: prediction, backtest, and analysis - import pdb; pdb.set_trace() for record in task_config["record"]: - record = record.copy() + record = deepcopy(record) if record["class"] == "SignalRecord": srconf = {"model": model, "dataset": dataset, "recorder": recorder} record["kwargs"].update(srconf) diff --git a/lib/trade_models/transformers.py b/lib/trade_models/transformers.py index 591a462..2aa3595 100644 --- a/lib/trade_models/transformers.py +++ b/lib/trade_models/transformers.py @@ -193,19 +193,15 @@ def get_transformer(config): raise ValueError("Invalid Configuration: {:}".format(config)) name = config.get("name", "basic") if name == "basic": - model = TransformerModel( + model = SuperTransformer( d_feat=config.get("d_feat"), - embed_dim=config.get("embed_dim"), - depth=config.get("depth"), + stem_dim=config.get("stem_dim"), + embed_dims=config.get("embed_dims"), num_heads=config.get("num_heads"), - mlp_ratio=config.get("mlp_ratio"), + mlp_hidden_multipliers=config.get("mlp_hidden_multipliers"), qkv_bias=config.get("qkv_bias"), - qk_scale=config.get("qkv_scale"), pos_drop=config.get("pos_drop"), - mlp_drop_rate=config.get("mlp_drop_rate"), - attn_drop_rate=config.get("attn_drop_rate"), - drop_path_rate=config.get("drop_path_rate"), - norm_layer=config.get("norm_layer", None), + other_drop=config.get("other_drop"), ) else: raise ValueError("Unknown model name: {:}".format(name)) diff --git a/lib/xlayers/super_module.py b/lib/xlayers/super_module.py index 276a849..564a136 100644 --- a/lib/xlayers/super_module.py +++ b/lib/xlayers/super_module.py @@ -14,6 +14,13 @@ IntSpaceType = Union[int, spaces.Integer, spaces.Categorical] BoolSpaceType = Union[bool, spaces.Categorical] +class LayerOrder(Enum): + """This class defines the enumerations for order of operation in a residual or normalization-based layer.""" + + PreNorm = "pre-norm" + PostNorm = "post-norm" + + class SuperRunMode(Enum): """This class defines the enumerations for Super Model Running Mode.""" diff --git a/lib/xlayers/super_transformer.py b/lib/xlayers/super_transformer.py index aa11511..72684da 100644 --- a/lib/xlayers/super_transformer.py +++ b/lib/xlayers/super_transformer.py @@ -15,6 +15,7 @@ import torch.nn.functional as F import spaces from .super_module import IntSpaceType from .super_module import BoolSpaceType +from .super_module import LayerOrder from .super_module import SuperModule from .super_linear import SuperMLPv2 from .super_norm import SuperLayerNorm1D @@ -30,7 +31,8 @@ class SuperTransformerEncoderLayer(SuperModule): - PyTorch Implementation: https://pytorch.org/docs/stable/_modules/torch/nn/modules/transformer.html#TransformerEncoderLayer Details: - MHA -> residual -> norm -> MLP -> residual -> norm + the original post-norm version: MHA -> residual -> norm -> MLP -> residual -> norm + the pre-norm version: norm -> MHA -> residual -> norm -> MLP -> residual """ def __init__( @@ -42,9 +44,10 @@ class SuperTransformerEncoderLayer(SuperModule): mlp_hidden_multiplier: IntSpaceType = 4, drop: Optional[float] = None, act_layer: Callable[[], nn.Module] = nn.GELU, + order: LayerOrder = LayerOrder.PreNorm, ): super(SuperTransformerEncoderLayer, self).__init__() - self.mha = SuperAttention( + mha = SuperAttention( input_dim, input_dim, num_heads=num_heads, @@ -52,17 +55,33 @@ class SuperTransformerEncoderLayer(SuperModule): attn_drop=drop, proj_drop=drop, ) - self.drop1 = nn.Dropout(drop or 0.0) - self.norm1 = SuperLayerNorm1D(input_dim) - self.mlp = SuperMLPv2( + drop1 = nn.Dropout(drop or 0.0) + norm1 = SuperLayerNorm1D(input_dim) + mlp = SuperMLPv2( input_dim, hidden_multiplier=mlp_hidden_multiplier, out_features=output_dim, act_layer=act_layer, drop=drop, ) - self.drop2 = nn.Dropout(drop or 0.0) - self.norm2 = SuperLayerNorm1D(output_dim) + drop2 = nn.Dropout(drop or 0.0) + norm2 = SuperLayerNorm1D(output_dim) + if order is LayerOrder.PreNorm: + self.norm1 = norm1 + self.mha = mha + self.drop1 = drop1 + self.norm2 = norm2 + self.mlp = mlp + self.drop2 = drop2 + elif order is LayerOrder.PostNoem: + self.mha = mha + self.drop1 = drop1 + self.norm1 = norm1 + self.mlp = mlp + self.drop2 = drop2 + self.norm2 = norm2 + else: + raise ValueError("Unknown order: {:}".format(order)) @property def abstract_search_space(self): @@ -89,12 +108,18 @@ class SuperTransformerEncoderLayer(SuperModule): return self.forward_raw(input) def forward_raw(self, input: torch.Tensor) -> torch.Tensor: - # multi-head attention - x = self.mha(input) - x = x + self.drop1(x) - x = self.norm1(x) - # feed-forward layer - x = self.mlp(x) - x = x + self.drop2(x) - x = self.norm2(x) + if order is LayerOrder.PreNorm: + x = self.norm1(input) + x = x + self.drop1(self.mha(x)) + x = self.norm2(x) + x = x + self.drop2(self.mlp(x)) + elif order is LayerOrder.PostNoem: + # multi-head attention + x = x + self.drop1(self.mha(input)) + x = self.norm1(x) + # feed-forward layer + x = x + self.drop2(self.mlp(x)) + x = self.norm2(x) + else: + raise ValueError("Unknown order: {:}".format(order)) return x