From 97717d826e1ef9c8ccf4fa29b454946772b2137e Mon Sep 17 00:00:00 2001 From: D-X-Y <280835372@qq.com> Date: Fri, 21 May 2021 13:55:38 +0800 Subject: [PATCH] Update NATS-Bench to cope with lib->xautodl --- CHANGE-LOG.md | 2 +- exps/NATS-Bench/show-dataset.py | 1 + exps/NATS-Bench/sss-file-manager.py | 13 ++++---- exps/NATS-Bench/tss-file-manager.py | 13 ++++---- exps/NATS-algos/bohb.py | 14 ++++----- exps/NATS-algos/random_wo_share.py | 44 +++++++++++++++++++++------- exps/NATS-algos/regularized_ea.py | 16 ++++------ exps/NATS-algos/reinforce.py | 17 ++++------- exps/NATS-algos/search-cell.py | 16 ++++------ exps/NATS-algos/search-size.py | 16 ++++------ xautodl/procedures/funcs_nasbench.py | 3 +- 11 files changed, 75 insertions(+), 80 deletions(-) diff --git a/CHANGE-LOG.md b/CHANGE-LOG.md index c031b72..717ffb6 100644 --- a/CHANGE-LOG.md +++ b/CHANGE-LOG.md @@ -9,4 +9,4 @@ - [2020.10.15] [446262a](https://github.com/D-X-Y/AutoDL-Projects/tree/446262a) Update NATS-BENCH to version 1.0 - [2020.12.20] [dae387a](https://github.com/D-X-Y/AutoDL-Projects/tree/dae387a) Update NATS-BENCH to version 1.1 - [2021.05.18] [98fadf8](https://github.com/D-X-Y/AutoDL-Projects/tree/98fadf8) Before moving to `xautodl` -- [2021.05.19] [b50ad2a](https://github.com/D-X-Y/AutoDL-Projects/tree/b50ad2a) `xautodl` is close to ready +- [2021.05.21] [b4e8eae](https://github.com/D-X-Y/AutoDL-Projects/tree/b4e8eae) `xautodl` is close to ready diff --git a/exps/NATS-Bench/show-dataset.py b/exps/NATS-Bench/show-dataset.py index f4419b3..0c66ae5 100644 --- a/exps/NATS-Bench/show-dataset.py +++ b/exps/NATS-Bench/show-dataset.py @@ -8,6 +8,7 @@ import os, sys, time, torch, random, argparse from typing import List, Text, Dict, Any from PIL import ImageFile + ImageFile.LOAD_TRUNCATED_IMAGES = True from copy import deepcopy diff --git a/exps/NATS-Bench/sss-file-manager.py b/exps/NATS-Bench/sss-file-manager.py index ce17652..3e600a9 100644 --- a/exps/NATS-Bench/sss-file-manager.py +++ b/exps/NATS-Bench/sss-file-manager.py @@ -12,14 +12,11 @@ from collections import defaultdict from copy import deepcopy from pathlib import Path -lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve() -if str(lib_dir) not in sys.path: - sys.path.insert(0, str(lib_dir)) -from config_utils import dict2config, load_config -from procedures import bench_evaluate_for_seed -from procedures import get_machine_info -from datasets import get_datasets -from log_utils import Logger, AverageMeter, time_string, convert_secs2time +from xautodl.config_utils import dict2config, load_config +from xautodl.procedures import bench_evaluate_for_seed +from xautodl.procedures import get_machine_info +from xautodl.datasets import get_datasets +from xautodl.log_utils import Logger, AverageMeter, time_string, convert_secs2time def obtain_valid_ckp(save_dir: Text, total: int): diff --git a/exps/NATS-Bench/tss-file-manager.py b/exps/NATS-Bench/tss-file-manager.py index 6038c7b..5ac6f92 100644 --- a/exps/NATS-Bench/tss-file-manager.py +++ b/exps/NATS-Bench/tss-file-manager.py @@ -12,14 +12,11 @@ from collections import defaultdict from copy import deepcopy from pathlib import Path -lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve() -if str(lib_dir) not in sys.path: - sys.path.insert(0, str(lib_dir)) -from config_utils import dict2config, load_config -from procedures import bench_evaluate_for_seed -from procedures import get_machine_info -from datasets import get_datasets -from log_utils import Logger, AverageMeter, time_string, convert_secs2time +from xautodl.config_utils import dict2config, load_config +from xautodl.procedures import bench_evaluate_for_seed +from xautodl.procedures import get_machine_info +from xautodl.datasets import get_datasets +from xautodl.log_utils import Logger, AverageMeter, time_string, convert_secs2time def obtain_valid_ckp(save_dir: Text, total: int, possible_seeds: List[int]): diff --git a/exps/NATS-algos/bohb.py b/exps/NATS-algos/bohb.py index e06f574..23dd3ee 100644 --- a/exps/NATS-algos/bohb.py +++ b/exps/NATS-algos/bohb.py @@ -10,18 +10,14 @@ ################################################################### import os, sys, time, random, argparse, collections from copy import deepcopy -from pathlib import Path import torch -lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve() -if str(lib_dir) not in sys.path: - sys.path.insert(0, str(lib_dir)) -from config_utils import load_config -from datasets import get_datasets, SearchDataset -from procedures import prepare_seed, prepare_logger -from log_utils import AverageMeter, time_string, convert_secs2time +from xautodl.config_utils import load_config +from xautodl.datasets import get_datasets, SearchDataset +from xautodl.procedures import prepare_seed, prepare_logger +from xautodl.log_utils import AverageMeter, time_string, convert_secs2time +from xautodl.models import CellStructure, get_search_spaces from nats_bench import create -from models import CellStructure, get_search_spaces # BOHB: Robust and Efficient Hyperparameter Optimization at Scale, ICML 2018 import ConfigSpace diff --git a/exps/NATS-algos/random_wo_share.py b/exps/NATS-algos/random_wo_share.py index 773ec0c..ac43059 100644 --- a/exps/NATS-algos/random_wo_share.py +++ b/exps/NATS-algos/random_wo_share.py @@ -12,25 +12,47 @@ import numpy as np, collections from copy import deepcopy import torch import torch.nn as nn -from pathlib import Path -lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve() -if str(lib_dir) not in sys.path: - sys.path.insert(0, str(lib_dir)) -from config_utils import load_config, dict2config, configure2str -from datasets import get_datasets, SearchDataset -from procedures import ( +from xautodl.config_utils import load_config, dict2config, configure2str +from xautodl.datasets import get_datasets, SearchDataset +from xautodl.procedures import ( prepare_seed, prepare_logger, save_checkpoint, copy_checkpoint, get_optim_scheduler, ) -from utils import get_model_infos, obtain_accuracy -from log_utils import AverageMeter, time_string, convert_secs2time -from models import get_search_spaces +from xautodl.utils import get_model_infos, obtain_accuracy +from xautodl.log_utils import AverageMeter, time_string, convert_secs2time +from xautodl.models import CellStructure, get_search_spaces from nats_bench import create -from regularized_ea import random_topology_func, random_size_func + + +def random_topology_func(op_names, max_nodes=4): + # Return a random architecture + def random_architecture(): + genotypes = [] + for i in range(1, max_nodes): + xlist = [] + for j in range(i): + node_str = "{:}<-{:}".format(i, j) + op_name = random.choice(op_names) + xlist.append((op_name, j)) + genotypes.append(tuple(xlist)) + return CellStructure(genotypes) + + return random_architecture + + +def random_size_func(info): + # Return a random architecture + def random_architecture(): + channels = [] + for i in range(info["numbers"]): + channels.append(str(random.choice(info["candidates"]))) + return ":".join(channels) + + return random_architecture def main(xargs, api): diff --git a/exps/NATS-algos/regularized_ea.py b/exps/NATS-algos/regularized_ea.py index 6499c8a..f16dd5f 100644 --- a/exps/NATS-algos/regularized_ea.py +++ b/exps/NATS-algos/regularized_ea.py @@ -16,23 +16,19 @@ import numpy as np, collections from copy import deepcopy import torch import torch.nn as nn -from pathlib import Path -lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve() -if str(lib_dir) not in sys.path: - sys.path.insert(0, str(lib_dir)) -from config_utils import load_config, dict2config, configure2str -from datasets import get_datasets, SearchDataset -from procedures import ( +from xautodl.config_utils import load_config, dict2config, configure2str +from xautodl.datasets import get_datasets, SearchDataset +from xautodl.procedures import ( prepare_seed, prepare_logger, save_checkpoint, copy_checkpoint, get_optim_scheduler, ) -from utils import get_model_infos, obtain_accuracy -from log_utils import AverageMeter, time_string, convert_secs2time -from models import CellStructure, get_search_spaces +from xautodl.utils import get_model_infos, obtain_accuracy +from xautodl.log_utils import AverageMeter, time_string, convert_secs2time +from xautodl.models import CellStructure, get_search_spaces from nats_bench import create diff --git a/exps/NATS-algos/reinforce.py b/exps/NATS-algos/reinforce.py index 7299c21..38b94d3 100644 --- a/exps/NATS-algos/reinforce.py +++ b/exps/NATS-algos/reinforce.py @@ -13,26 +13,22 @@ import os, sys, time, glob, random, argparse import numpy as np, collections from copy import deepcopy -from pathlib import Path import torch import torch.nn as nn from torch.distributions import Categorical -lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve() -if str(lib_dir) not in sys.path: - sys.path.insert(0, str(lib_dir)) -from config_utils import load_config, dict2config, configure2str -from datasets import get_datasets, SearchDataset -from procedures import ( +from xautodl.config_utils import load_config, dict2config, configure2str +from xautodl.datasets import get_datasets, SearchDataset +from xautodl.procedures import ( prepare_seed, prepare_logger, save_checkpoint, copy_checkpoint, get_optim_scheduler, ) -from utils import get_model_infos, obtain_accuracy -from log_utils import AverageMeter, time_string, convert_secs2time -from models import CellStructure, get_search_spaces +from xautodl.utils import get_model_infos, obtain_accuracy +from xautodl.log_utils import AverageMeter, time_string, convert_secs2time +from xautodl.models import CellStructure, get_search_spaces from nats_bench import create @@ -206,7 +202,6 @@ def main(xargs, api): if __name__ == "__main__": parser = argparse.ArgumentParser("The REINFORCE Algorithm") - parser.add_argument("--data_path", type=str, help="Path to dataset") parser.add_argument( "--dataset", type=str, diff --git a/exps/NATS-algos/search-cell.py b/exps/NATS-algos/search-cell.py index dc21021..b1632fb 100644 --- a/exps/NATS-algos/search-cell.py +++ b/exps/NATS-algos/search-cell.py @@ -30,23 +30,19 @@ import numpy as np from copy import deepcopy import torch import torch.nn as nn -from pathlib import Path -lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve() -if str(lib_dir) not in sys.path: - sys.path.insert(0, str(lib_dir)) -from config_utils import load_config, dict2config, configure2str -from datasets import get_datasets, get_nas_search_loaders -from procedures import ( +from xautodl.config_utils import load_config, dict2config, configure2str +from xautodl.datasets import get_datasets, get_nas_search_loaders +from xautodl.procedures import ( prepare_seed, prepare_logger, save_checkpoint, copy_checkpoint, get_optim_scheduler, ) -from utils import count_parameters_in_MB, obtain_accuracy -from log_utils import AverageMeter, time_string, convert_secs2time -from models import get_cell_based_tiny_net, get_search_spaces +from xautodl.utils import count_parameters_in_MB, obtain_accuracy +from xautodl.log_utils import AverageMeter, time_string, convert_secs2time +from xautodl.models import get_cell_based_tiny_net, get_search_spaces from nats_bench import create diff --git a/exps/NATS-algos/search-size.py b/exps/NATS-algos/search-size.py index 7bfa33f..c9c6a6d 100644 --- a/exps/NATS-algos/search-size.py +++ b/exps/NATS-algos/search-size.py @@ -31,23 +31,19 @@ import numpy as np from copy import deepcopy import torch import torch.nn as nn -from pathlib import Path -lib_dir = (Path(__file__).parent / ".." / ".." / "lib").resolve() -if str(lib_dir) not in sys.path: - sys.path.insert(0, str(lib_dir)) -from config_utils import load_config, dict2config, configure2str -from datasets import get_datasets, get_nas_search_loaders -from procedures import ( +from xautodl.config_utils import load_config, dict2config, configure2str +from xautodl.datasets import get_datasets, get_nas_search_loaders +from xautodl.procedures import ( prepare_seed, prepare_logger, save_checkpoint, copy_checkpoint, get_optim_scheduler, ) -from utils import count_parameters_in_MB, obtain_accuracy -from log_utils import AverageMeter, time_string, convert_secs2time -from models import get_cell_based_tiny_net, get_search_spaces +from xautodl.utils import count_parameters_in_MB, obtain_accuracy +from xautodl.log_utils import AverageMeter, time_string, convert_secs2time +from xautodl.models import get_cell_based_tiny_net, get_search_spaces from nats_bench import create diff --git a/xautodl/procedures/funcs_nasbench.py b/xautodl/procedures/funcs_nasbench.py index 00566a9..dfe69d3 100644 --- a/xautodl/procedures/funcs_nasbench.py +++ b/xautodl/procedures/funcs_nasbench.py @@ -3,7 +3,6 @@ ##################################################### import os, time, copy, torch, pathlib -# modules in AutoDL from xautodl import datasets from xautodl.config_utils import load_config from xautodl.procedures import prepare_seed, get_optim_scheduler @@ -83,7 +82,7 @@ def procedure(xloader, network, criterion, scheduler, optimizer, mode: str): def evaluate_for_seed( arch_config, opt_config, train_loader, valid_loaders, seed: int, logger ): - + """A modular function to train and evaluate a single network, using the given random seed and optimization config with the provided loaders.""" prepare_seed(seed) # random seed net = get_cell_based_tiny_net(arch_config) # net = TinyNetwork(arch_config['channel'], arch_config['num_cells'], arch, config.class_num)