From 01397660de44cb06acdce33dc7d224bd9a91305d Mon Sep 17 00:00:00 2001 From: D-X-Y <280835372@qq.com> Date: Tue, 23 Mar 2021 09:43:45 +0000 Subject: [PATCH] Updates --- .gitignore | 2 +- .latent-data/qlib | 2 +- lib/procedures/q_exps.py | 1 + notebooks/Q/workflow-demo.py | 75 ------------------------------------ notebooks/spaces/test.py | 75 ++++++++++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+), 77 deletions(-) delete mode 100644 notebooks/Q/workflow-demo.py create mode 100644 notebooks/spaces/test.py diff --git a/.gitignore b/.gitignore index b908dfd..a47c212 100644 --- a/.gitignore +++ b/.gitignore @@ -128,7 +128,7 @@ TEMP-L.sh # Visual Studio Code .vscode -mlruns +mlruns* outputs pytest_cache diff --git a/.latent-data/qlib b/.latent-data/qlib index ba56e40..2b74b4d 160000 --- a/.latent-data/qlib +++ b/.latent-data/qlib @@ -1 +1 @@ -Subproject commit ba56e4071efd1c08003eaf7e23978aaf81376dd1 +Subproject commit 2b74b4dfa4a6996ab6135873c0329022a1b9626b diff --git a/lib/procedures/q_exps.py b/lib/procedures/q_exps.py index 5887cfb..befcae9 100644 --- a/lib/procedures/q_exps.py +++ b/lib/procedures/q_exps.py @@ -98,6 +98,7 @@ def run_exp(task_config, dataset, experiment_name, recorder_name, uri): R.save_objects(**{"model.pkl": model}) # Generate records: prediction, backtest, and analysis + import pdb; pdb.set_trace() for record in task_config["record"]: record = record.copy() if record["class"] == "SignalRecord": diff --git a/notebooks/Q/workflow-demo.py b/notebooks/Q/workflow-demo.py deleted file mode 100644 index 8492a7a..0000000 --- a/notebooks/Q/workflow-demo.py +++ /dev/null @@ -1,75 +0,0 @@ -import os -import sys -import ruamel.yaml as yaml -import pprint -import numpy as np -import pandas as pd -from pathlib import Path - -import qlib -from qlib import config as qconfig -from qlib.utils import init_instance_by_config -from qlib.workflow import R -from qlib.data.dataset import DatasetH -from qlib.data.dataset.handler import DataHandlerLP - -qlib.init(provider_uri="~/.qlib/qlib_data/cn_data", region=qconfig.REG_CN) - -dataset_config = { - "class": "DatasetH", - "module_path": "qlib.data.dataset", - "kwargs": { - "handler": { - "class": "Alpha360", - "module_path": "qlib.contrib.data.handler", - "kwargs": { - "start_time": "2008-01-01", - "end_time": "2020-08-01", - "fit_start_time": "2008-01-01", - "fit_end_time": "2014-12-31", - "instruments": "csi300", - "infer_processors": [ - {"class": "RobustZScoreNorm", "kwargs": {"fields_group": "feature", "clip_outlier": True}}, - {"class": "Fillna", "kwargs": {"fields_group": "feature"}}, - ], - "learn_processors": [ - {"class": "DropnaLabel"}, - {"class": "CSRankNorm", "kwargs": {"fields_group": "label"}}, - ], - "label": ["Ref($close, -2) / Ref($close, -1) - 1"] - }, - }, - "segments": { - "train": ("2008-01-01", "2014-12-31"), - "valid": ("2015-01-01", "2016-12-31"), - "test": ("2017-01-01", "2020-08-01"), - }, - }, -} - -if __name__ == "__main__": - - qlib_root_dir = (Path(__file__).parent / '..' / '..' / '.latent-data' / 'qlib').resolve() - demo_yaml_path = qlib_root_dir / 'examples' / 'benchmarks' / 'GRU' / 'workflow_config_gru_Alpha360.yaml' - print('Demo-workflow-yaml: {:}'.format(demo_yaml_path)) - with open(demo_yaml_path, 'r') as fp: - config = yaml.safe_load(fp) - pprint.pprint(config['task']['dataset']) - - dataset = init_instance_by_config(dataset_config) - pprint.pprint(dataset_config) - pprint.pprint(dataset) - - df_train, df_valid, df_test = dataset.prepare( - ["train", "valid", "test"], - col_set=["feature", "label"], - data_key=DataHandlerLP.DK_L, - ) - - x_train, y_train = df_train["feature"], df_train["label"] - - import pdb - - pdb.set_trace() - print("Complete") - diff --git a/notebooks/spaces/test.py b/notebooks/spaces/test.py new file mode 100644 index 0000000..e81342f --- /dev/null +++ b/notebooks/spaces/test.py @@ -0,0 +1,75 @@ +import os +import sys +import qlib +import pprint +import numpy as np +import pandas as pd + +from pathlib import Path +import torch + +__file__ = os.path.dirname(os.path.realpath("__file__")) + +lib_dir = (Path(__file__).parent / ".." / "lib").resolve() +print("library path: {:}".format(lib_dir)) +assert lib_dir.exists(), "{:} does not exist".format(lib_dir) +if str(lib_dir) not in sys.path: + sys.path.insert(0, str(lib_dir)) + +from trade_models import get_transformer + +from qlib import config as qconfig +from qlib.utils import init_instance_by_config +from qlib.model.base import Model +from qlib.data.dataset import DatasetH +from qlib.data.dataset.handler import DataHandlerLP + +qlib.init(provider_uri='~/.qlib/qlib_data/cn_data', region=qconfig.REG_CN) + +dataset_config = { + "class": "DatasetH", + "module_path": "qlib.data.dataset", + "kwargs": { + "handler": { + "class": "Alpha360", + "module_path": "qlib.contrib.data.handler", + "kwargs": { + "start_time": "2008-01-01", + "end_time": "2020-08-01", + "fit_start_time": "2008-01-01", + "fit_end_time": "2014-12-31", + "instruments": "csi100", + }, + }, + "segments": { + "train": ("2008-01-01", "2014-12-31"), + "valid": ("2015-01-01", "2016-12-31"), + "test": ("2017-01-01", "2020-08-01"), + }, + }, + } +pprint.pprint(dataset_config) +dataset = init_instance_by_config(dataset_config) + +df_train, df_valid, df_test = dataset.prepare( + ["train", "valid", "test"], + col_set=["feature", "label"], + data_key=DataHandlerLP.DK_L, + ) +model = get_transformer(None) +print(model) + +features = torch.from_numpy(df_train["feature"].values).float() +labels = torch.from_numpy(df_train["label"].values).squeeze().float() + +batch = list(range(2000)) +predicts = model(features[batch]) +mask = ~torch.isnan(labels[batch]) + +pred = predicts[mask] +label = labels[batch][mask] + +loss = torch.nn.functional.mse_loss(pred, label) + +from sklearn.metrics import mean_squared_error +mse_loss = mean_squared_error(pred.numpy(), label.numpy())