From 69ca0860aa379bafe488283a1bca59439528c9e7 Mon Sep 17 00:00:00 2001 From: D-X-Y <280835372@qq.com> Date: Fri, 20 Dec 2019 20:41:49 +1100 Subject: [PATCH] update for NAS-Bench-102 --- .gitignore | 3 + NAS-Bench-102.md | 156 +++++++++++ README.md | 38 ++- exps/NAS-Bench-102/functions.py | 31 +- exps/NAS-Bench-102/main.py | 7 +- exps/NAS-Bench-102/statistics.py | 295 ++++++++++++++++++++ exps/algos/BOHB.py | 6 +- exps/algos/RANDOM.py | 4 +- exps/algos/R_EA.py | 4 +- exps/algos/reinforce.py | 2 +- exps/vis/test.py | 27 ++ lib/{aa_nas_api => nas_102_api}/__init__.py | 2 +- lib/{aa_nas_api => nas_102_api}/api.py | 127 +++++++-- 13 files changed, 656 insertions(+), 46 deletions(-) create mode 100644 NAS-Bench-102.md create mode 100644 exps/NAS-Bench-102/statistics.py create mode 100644 exps/vis/test.py rename lib/{aa_nas_api => nas_102_api}/__init__.py (85%) rename lib/{aa_nas_api => nas_102_api}/api.py (72%) diff --git a/.gitignore b/.gitignore index 2e838bb..7173302 100644 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,6 @@ logs a.pth cal-merge*.sh GPU-*.sh +cal.sh +aaa +cx.sh diff --git a/NAS-Bench-102.md b/NAS-Bench-102.md new file mode 100644 index 0000000..b086fe8 --- /dev/null +++ b/NAS-Bench-102.md @@ -0,0 +1,156 @@ +# NAS-BENCH-102: Extending the Scope of Reproducible Neural Architecture Search + +We propose an algorithm-agnostic NAS benchmark (NAS-Bench-102) with a fixed search space, which provides a unified benchmark for almost any up-to-date NAS algorithms. +The design of our search space is inspired from that used in the most popular cell-based searching algorithms, where a cell is represented as a directed acyclic graph. +Each edge here is associated with an operation selected from a predefined operation set. +For it to be applicable for all NAS algorithms, the search space defined in NAS-Bench-102 includes 4 nodes and 5 associated operation options, which generates 15,625 neural cell candidates in total. + +In this Markdown file, we provide: +- Detailed instruction to reproduce NAS-Bench-102. +- 10 NAS algorithms evaluated in our paper. + +Note: please use `PyTorch >= 1.2.0` and `Python >= 3.6.0`. + +## How to Use NAS-Bench-102 + +1. Creating an API instance from a file: +``` +from nas_102_api import NASBench102API +api = NASBench102API('$path_to_meta_nas_bench_file') +api = NASBench102API('NAS-Bench-102-v1_0.pth') +``` + +2. Show the number of architectures `len(api)` and each architecture `api[i]`: +``` +num = len(api) +for i, arch_str in enumerate(api): + print ('{:5d}/{:5d} : {:}'.format(i, len(api), arch_str)) +``` + +3. Show the results of all trials for a single architecture: +``` +# show all information for a specific architecture +api.show(1) +api.show(2) + +# show the mean loss and accuracy of an architecture +info = api.query_meta_info_by_index(1) +loss, accuracy = info.get_metrics('cifar10', 'train') +flops, params, latency = info.get_comput_costs('cifar100') + +# get the detailed information +results = api.query_by_index(1, 'cifar100') +print ('There are {:} trials for this architecture [{:}] on cifar100'.format(len(results), api[1])) +print ('Latency : {:}'.format(results[0].get_latency())) +print ('Train Info : {:}'.format(results[0].get_train())) +print ('Valid Info : {:}'.format(results[0].get_eval('x-valid'))) +print ('Test Info : {:}'.format(results[0].get_eval('x-test'))) +# for the metric after a specific epoch +print ('Train Info [10-th epoch] : {:}'.format(results[0].get_train(10))) +``` + +4. Query the index of an architecture by string +``` +index = api.query_index_by_arch('|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|skip_connect~2|') +api.show(index) +``` + +5. For other usages, please see `lib/aa_nas_api/api.py` + +### Detailed Instruction + +In `nas_102_api`, we define three classes: `NASBench102API`, `ArchResults`, `ResultsCount`. + +`ResultsCount` maintains all information of a specific trial. One can instantiate ResultsCount and get the info via the following codes (`000157-FULL.pth` saves all information of all trials of 157-th architecture): +``` +from nas_102_api import ResultsCount +xdata = torch.load('000157-FULL.pth') +odata = xdata['full']['all_results'][('cifar10-valid', 777)] +result = ResultsCount.create_from_state_dict( odata ) +print(result) # print it +print(result.get_train()) # print the final training loss/accuracy/[optional:time-cost-of-a-training-epoch] +print(result.get_train(11)) # print the training info of the 11-th epoch +print(result.get_eval('x-valid')) # print the final evaluation info on the validation set +print(result.get_eval('x-valid', 11)) # print the info on the validation set of the 11-th epoch +print(result.get_latency()) # print the evaluation latency [in batch] +result.get_net_param() # the trained parameters of this trial +arch_config = result.get_config(CellStructure.str2structure) # create the network with params +net_config = dict2config(arch_config, None) +network = get_cell_based_tiny_net(net_config) +network.load_state_dict(result.get_net_param()) +``` + +`ArchResults` maintains all information of all trials of an architecture. Please see the following usages: +``` +from nas_102_api import ArchResults +xdata = torch.load('000157-FULL.pth') +archRes = ArchResults.create_from_state_dict(xdata['less']) # load trials trained with 12 epochs +archRes = ArchResults.create_from_state_dict(xdata['full']) # load trials trained with 200 epochs + +print(archRes.arch_idx_str()) # print the index of this architecture +print(archRes.get_dataset_names()) # print the supported training data +print(archRes.get_comput_costs('cifar10-valid')) # print all computational info when training on cifar10-valid +print(archRes.get_metrics('cifar10-valid', 'x-valid', None, False)) # print the average loss/accuracy/time on all trials +print(archRes.get_metrics('cifar10-valid', 'x-valid', None, True)) # print loss/accuracy/time of a randomly selected trial +``` + +`NASBench102API` is the topest level api. Please see the following usages: +``` +from nas_102_api import NASBench102API as API +api = API('NAS-Bench-102-v1_0.pth') +``` + +## Instruction to Re-Generate NAS-Bench-102 + +1. generate the meta file for NAS-Bench-102 using the following script, where `NAS-BENCH-102` indicates the name and `4` indicates the maximum number of nodes in a cell. +``` +bash scripts-search/NAS-Bench-102/meta-gen.sh NAS-BENCH-102 4 +``` + +2. train earch architecture on a single GPU (see commands in `output/NAS-BENCH-102-4/BENCH-102-N4.opt-full.script`, which is automatically generated by step-1). +``` +CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/NAS-Bench-102/train-models.sh 0 0 389 -1 '777 888 999' +``` +This command will train 390 architectures (id from 0 to 389) using the following four kinds of splits with three random seeds (777, 888, 999). + +| Dataset | Train | Eval | +|:---------------:|:-------------:|:------------:| +| CIFAR-10 | train | valid / test | +| CIFAR-10 | train + valid | test | +| CIFAR-100 | train | valid / test | +| ImageNet-16-120 | train | valid / test | + +3. calculate the latency, merge the results of all architectures, and simplify the results. +(see commands in `output/NAS-BENCH-102-4/meta-node-4.cal-script.txt` which is automatically generated by step-1). +``` +OMP_NUM_THREADS=6 CUDA_VISIBLE_DEVICES=0 python exps/NAS-Bench-102/statistics.py --mode cal --target_dir 000000-000389-C16-N5 +``` + +4. merge all results into a single file for NAS-Bench-102-API. +``` +OMP_NUM_THREADS=4 python exps/NAS-Bench-102/statistics.py --mode merge +``` +This command will generate a single file `output/NAS-BENCH-102-4/simplifies/C16-N5-final-infos.pth` contains all the data for NAS-Bench-102. +This generated file will serve as the input for our NAS-Bench-102 API. + +[option] train a single architecture on a single GPU. +``` +CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/NAS-Bench-102/train-a-net.sh resnet 16 5 +CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/NAS-Bench-102/train-a-net.sh '|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|skip_connect~1|skip_connect~2|' 16 5 +``` + +## To Reproduce 10 Baseline NAS Algorithms in NAS-Bench-102 + +We have tried our best to implement each method. However, still, some algorithms might obtain non-optimal results since their hyper-parameters might not fit our NAS-Bench-102. +If researchers can provide better results with different hyper-parameters, we are happy to update results according to the new experimental results. We also welcome more NAS algorithms to test on our dataset and would include them accordingly. + +- [1] `CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/DARTS-V1.sh cifar10 -1` +- [2] `CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/DARTS-V2.sh cifar10 -1` +- [3] `CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/GDAS.sh cifar10 -1` +- [4] `CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/SETN.sh cifar10 -1` +- [5] `CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/ENAS.sh cifar10 -1` +- [6] `CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/RANDOM-NAS.sh cifar10 -1` +- [7] `bash ./scripts-search/algos/R-EA.sh -1` +- [8] `bash ./scripts-search/algos/Random.sh -1` +- [9] `bash ./scripts-search/algos/REINFORCE.sh -1` +- [10] `bash ./scripts-search/algos/BOHB.sh -1` diff --git a/README.md b/README.md index 9351a4d..d072bb2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ More NAS resources can be found in [Awesome-NAS](https://github.com/D-X-Y/Awesom - Network Pruning via Transformable Architecture Search, NeurIPS 2019 - One-Shot Neural Architecture Search via Self-Evaluated Template Network, ICCV 2019 - Searching for A Robust Neural Architecture in Four GPU Hours, CVPR 2019 -- 10 NAS algorithms for the neural topology in `exps/algos` +- NAS-Bench-102: Extending the Scope of Reproducible Neural Architecture Search, ICLR 2020 +- 10 NAS algorithms for the neural topology in `exps/algos` (see [NAS-Bench-102.md](https://github.com/D-X-Y/NAS-Projects/blob/master/NAS-Bench-102.md) for more details) - Several typical classification models, e.g., ResNet and DenseNet (see [BASELINE.md](https://github.com/D-X-Y/NAS-Projects/blob/master/BASELINE.md)) @@ -27,6 +28,10 @@ flop, param = get_model_infos(net, (1,3,32,32)) 2. Different NAS-searched architectures are defined [here](https://github.com/D-X-Y/NAS-Projects/blob/master/lib/nas_infer_model/DXYs/genotypes.py). +## NAS-Bench-102: Extending the Scope of Reproducible Neural Architecture Search + +We build a new benchmark for neural architecture search, please see more details in [NAS-Bench-102.md](https://github.com/D-X-Y/NAS-Projects/blob/master/NAS-Bench-102.md). + ## [Network Pruning via Transformable Architecture Search](https://arxiv.org/abs/1905.09717) In this paper, we proposed a differentiable searching strategy for transformable architectures, i.e., searching for the depth and width of a deep neural network. You could see the highlight of our Transformable Architecture Search (TAS) at our [project page](https://xuanyidong.com/assets/projects/NeurIPS-2019-TAS.html). @@ -42,31 +47,33 @@ You could see the highlight of our Transformable Architecture Search (TAS) at ou Use `bash ./scripts/prepare.sh` to prepare data splits for `CIFAR-10`, `CIFARR-100`, and `ILSVRC2012`. If you do not have `ILSVRC2012` data, pleasee comment L12 in `./scripts/prepare.sh`. -Search the depth configuration of ResNet: +args: `cifar10` indicates the dataset name, `ResNet56` indicates the basemodel name, `CIFARX` indicates the searching hyper-parameters, `0.47/0.57` indicates the expected FLOP ratio, `-1` indicates the random seed. + +#### Search for the depth configuration of ResNet: ``` CUDA_VISIBLE_DEVICES=0,1 bash ./scripts-search/search-depth-gumbel.sh cifar10 ResNet110 CIFARX 0.57 -1 ``` -Search the width configuration of ResNet: +#### Search for the width configuration of ResNet: ``` CUDA_VISIBLE_DEVICES=0,1 bash ./scripts-search/search-width-gumbel.sh cifar10 ResNet110 CIFARX 0.57 -1 ``` -Search for both depth and width configuration of ResNet: +#### Search for both depth and width configuration of ResNet: ``` CUDA_VISIBLE_DEVICES=0,1 bash ./scripts-search/search-shape-cifar.sh cifar10 ResNet56 CIFARX 0.47 -1 ``` -args: `cifar10` indicates the dataset name, `ResNet56` indicates the basemodel name, `CIFARX` indicates the searching hyper-parameters, `0.47/0.57` indicates the expected FLOP ratio, `-1` indicates the random seed. - -### Model Configuration -The searched shapes for ResNet-20/32/56/110/164 in Table 3 in the original paper are listed in [`configs/NeurIPS-2019`](https://github.com/D-X-Y/NAS-Projects/tree/master/configs/NeurIPS-2019). +#### Training the searched shape config from TAS If you want to directly train a model with searched configuration of TAS, try these: ``` CUDA_VISIBLE_DEVICES=0,1 bash ./scripts/tas-infer-train.sh cifar10 C010-ResNet32 -1 CUDA_VISIBLE_DEVICES=0,1 bash ./scripts/tas-infer-train.sh cifar100 C100-ResNet32 -1 ``` +### Model Configuration +The searched shapes for ResNet-20/32/56/110/164 in Table 3 in the original paper are listed in [`configs/NeurIPS-2019`](https://github.com/D-X-Y/NAS-Projects/tree/master/configs/NeurIPS-2019). + ## [One-Shot Neural Architecture Search via Self-Evaluated Template Network](https://arxiv.org/abs/1910.05733) @@ -103,6 +110,7 @@ The old version is located at [`others/GDAS`](https://github.com/D-X-Y/NAS-Proje ### Usage +#### Reproducing the results of our searched architecture in GDAS Please use the following scripts to train the searched GDAS-searched CNN on CIFAR-10, CIFAR-100, and ImageNet. ``` CUDA_VISIBLE_DEVICES=0 bash ./scripts/nas-infer-train.sh cifar10 GDAS_V1 96 -1 @@ -110,6 +118,7 @@ CUDA_VISIBLE_DEVICES=0 bash ./scripts/nas-infer-train.sh cifar100 GDAS_V1 96 -1 CUDA_VISIBLE_DEVICES=0,1,2,3 bash ./scripts/nas-infer-train.sh imagenet-1k GDAS_V1 256 -1 ``` +#### Searching on a small search space (NAS-Bench-102) The GDAS searching codes on a small search space: ``` CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/GDAS.sh cifar10 -1 @@ -121,11 +130,24 @@ CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/DARTS-V1.sh cifar10 -1 CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/DARTS-V2.sh cifar10 -1 ``` +#### Training the searched architecture +To train the searched architecture found by the above scripts, please use the following codes: +``` +CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/NAS-Bench-102/train-a-net.sh '|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|skip_connect~1|skip_connect~2|' 16 5 +``` +`|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|skip_connect~1|skip_connect~2|` represents the structure of a searched architecture. My codes will automatically print it during the searching procedure. + # Citation If you find that this project helps your research, please consider citing some of the following papers: ``` +@inproceedings{dong2020nasbench102, + title = {NAS-Bench-102: Extending the Scope of Reproducible Neural Architecture Search}, + author = {Dong, Xuanyi and Yang, Yi}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2020} +} @inproceedings{dong2019tas, title = {Network Pruning via Transformable Architecture Search}, author = {Dong, Xuanyi and Yang, Yi}, diff --git a/exps/NAS-Bench-102/functions.py b/exps/NAS-Bench-102/functions.py index c9f448d..dc1a173 100644 --- a/exps/NAS-Bench-102/functions.py +++ b/exps/NAS-Bench-102/functions.py @@ -10,7 +10,36 @@ from models import get_cell_based_tiny_net -__all__ = ['evaluate_for_seed'] +__all__ = ['evaluate_for_seed', 'pure_evaluate'] + + + +def pure_evaluate(xloader, network, criterion=torch.nn.CrossEntropyLoss()): + data_time, batch_time, batch = AverageMeter(), AverageMeter(), None + losses, top1, top5 = AverageMeter(), AverageMeter(), AverageMeter() + latencies = [] + network.eval() + with torch.no_grad(): + end = time.time() + for i, (inputs, targets) in enumerate(xloader): + targets = targets.cuda(non_blocking=True) + inputs = inputs.cuda(non_blocking=True) + data_time.update(time.time() - end) + # forward + features, logits = network(inputs) + loss = criterion(logits, targets) + batch_time.update(time.time() - end) + if batch is None or batch == inputs.size(0): + batch = inputs.size(0) + latencies.append( batch_time.val - data_time.val ) + # record loss and accuracy + prec1, prec5 = obtain_accuracy(logits.data, targets.data, topk=(1, 5)) + losses.update(loss.item(), inputs.size(0)) + top1.update (prec1.item(), inputs.size(0)) + top5.update (prec5.item(), inputs.size(0)) + end = time.time() + if len(latencies) > 2: latencies = latencies[1:] + return losses.avg, top1.avg, top5.avg, latencies diff --git a/exps/NAS-Bench-102/main.py b/exps/NAS-Bench-102/main.py index 0b813e4..9cde574 100644 --- a/exps/NAS-Bench-102/main.py +++ b/exps/NAS-Bench-102/main.py @@ -1,4 +1,6 @@ ################################################## +# NAS-Bench-102 ################################## +################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # ################################################## import os, sys, time, torch, random, argparse @@ -265,13 +267,14 @@ def generate_meta_info(save_dir, max_node, divide=40): with open(str(script_name), 'w') as cfile: for start in range(0, total_arch, gaps): xend = min(start+gaps, total_arch) - cfile.write('{:} python exps/AA-NAS-statistics.py --mode cal --target_dir {:06d}-{:06d}-C16-N5\n'.format(macro, start, xend-1)) + cfile.write('{:} python exps/NAS-Bench-102/statistics.py --mode cal --target_dir {:06d}-{:06d}-C16-N5\n'.format(macro, start, xend-1)) print ('save the post-processing script into {:}'.format(script_name)) if __name__ == '__main__': #mode_choices = ['meta', 'new', 'cover'] + ['specific-{:}'.format(_) for _ in CellArchitectures.keys()] - parser = argparse.ArgumentParser(description='Algorithm-Agnostic NAS Benchmark', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + #parser = argparse.ArgumentParser(description='Algorithm-Agnostic NAS Benchmark', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description='NAS-Bench-102', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--mode' , type=str, required=True, help='The script mode.') parser.add_argument('--save_dir', type=str, help='Folder to save checkpoints and log.') parser.add_argument('--max_node', type=int, help='The maximum node in a cell.') diff --git a/exps/NAS-Bench-102/statistics.py b/exps/NAS-Bench-102/statistics.py new file mode 100644 index 0000000..4eddc56 --- /dev/null +++ b/exps/NAS-Bench-102/statistics.py @@ -0,0 +1,295 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +import os, sys, time, argparse, collections +from copy import deepcopy +import torch +import torch.nn as nn +from pathlib import Path +from collections import defaultdict +lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve() +if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) +from log_utils import AverageMeter, time_string, convert_secs2time +from config_utils import load_config, dict2config +from datasets import get_datasets +# NAS-Bench-102 related module or function +from models import CellStructure, get_cell_based_tiny_net +from nas_102_api import ArchResults, ResultsCount +from functions import pure_evaluate + + + +def create_result_count(used_seed, dataset, arch_config, results, dataloader_dict): + xresult = ResultsCount(dataset, results['net_state_dict'], results['train_acc1es'], results['train_losses'], \ + results['param'], results['flop'], arch_config, used_seed, results['total_epoch'], None) + + net_config = dict2config({'name': 'infer.tiny', 'C': arch_config['channel'], 'N': arch_config['num_cells'], 'genotype': CellStructure.str2structure(arch_config['arch_str']), 'num_classes':arch_config['class_num']}, None) + network = get_cell_based_tiny_net(net_config) + network.load_state_dict(xresult.get_net_param()) + if 'train_times' in results: # new version + xresult.update_train_info(results['train_acc1es'], results['train_acc5es'], results['train_losses'], results['train_times']) + xresult.update_eval(results['valid_acc1es'], results['valid_losses'], results['valid_times']) + else: + if dataset == 'cifar10-valid': + xresult.update_OLD_eval('x-valid' , results['valid_acc1es'], results['valid_losses']) + loss, top1, top5, latencies = pure_evaluate(dataloader_dict['{:}@{:}'.format('cifar10', 'test')], network.cuda()) + xresult.update_OLD_eval('ori-test', {results['total_epoch']-1: top1}, {results['total_epoch']-1: loss}) + xresult.update_latency(latencies) + elif dataset == 'cifar10': + xresult.update_OLD_eval('ori-test', results['valid_acc1es'], results['valid_losses']) + loss, top1, top5, latencies = pure_evaluate(dataloader_dict['{:}@{:}'.format(dataset, 'test')], network.cuda()) + xresult.update_latency(latencies) + elif dataset == 'cifar100' or dataset == 'ImageNet16-120': + xresult.update_OLD_eval('ori-test', results['valid_acc1es'], results['valid_losses']) + loss, top1, top5, latencies = pure_evaluate(dataloader_dict['{:}@{:}'.format(dataset, 'valid')], network.cuda()) + xresult.update_OLD_eval('x-valid', {results['total_epoch']-1: top1}, {results['total_epoch']-1: loss}) + loss, top1, top5, latencies = pure_evaluate(dataloader_dict['{:}@{:}'.format(dataset, 'test')], network.cuda()) + xresult.update_OLD_eval('x-test' , {results['total_epoch']-1: top1}, {results['total_epoch']-1: loss}) + xresult.update_latency(latencies) + else: + raise ValueError('invalid dataset name : {:}'.format(dataset)) + return xresult + + + +def account_one_arch(arch_index, arch_str, checkpoints, datasets, dataloader_dict): + information = ArchResults(arch_index, arch_str) + + for checkpoint_path in checkpoints: + checkpoint = torch.load(checkpoint_path, map_location='cpu') + used_seed = checkpoint_path.name.split('-')[-1].split('.')[0] + for dataset in datasets: + assert dataset in checkpoint, 'Can not find {:} in arch-{:} from {:}'.format(dataset, arch_index, checkpoint_path) + results = checkpoint[dataset] + assert results['finish-train'], 'This {:} arch seed={:} does not finish train on {:} ::: {:}'.format(arch_index, used_seed, dataset, checkpoint_path) + arch_config = {'channel': results['channel'], 'num_cells': results['num_cells'], 'arch_str': arch_str, 'class_num': results['config']['class_num']} + + xresult = create_result_count(used_seed, dataset, arch_config, results, dataloader_dict) + information.update(dataset, int(used_seed), xresult) + return information + + + +def GET_DataLoaders(workers): + + torch.set_num_threads(workers) + + root_dir = (Path(__file__).parent / '..' / '..').resolve() + torch_dir = Path(os.environ['TORCH_HOME']) + # cifar + cifar_config_path = root_dir / 'configs' / 'nas-benchmark' / 'CIFAR.config' + cifar_config = load_config(cifar_config_path, None, None) + print ('{:} Create data-loader for all datasets'.format(time_string())) + print ('-'*200) + TRAIN_CIFAR10, VALID_CIFAR10, xshape, class_num = get_datasets('cifar10', str(torch_dir/'cifar.python'), -1) + print ('original CIFAR-10 : {:} training images and {:} test images : {:} input shape : {:} number of classes'.format(len(TRAIN_CIFAR10), len(VALID_CIFAR10), xshape, class_num)) + cifar10_splits = load_config(root_dir / 'configs' / 'nas-benchmark' / 'cifar-split.txt', None, None) + assert cifar10_splits.train[:10] == [0, 5, 7, 11, 13, 15, 16, 17, 20, 24] and cifar10_splits.valid[:10] == [1, 2, 3, 4, 6, 8, 9, 10, 12, 14] + temp_dataset = deepcopy(TRAIN_CIFAR10) + temp_dataset.transform = VALID_CIFAR10.transform + # data loader + trainval_cifar10_loader = torch.utils.data.DataLoader(TRAIN_CIFAR10, batch_size=cifar_config.batch_size, shuffle=True , num_workers=workers, pin_memory=True) + train_cifar10_loader = torch.utils.data.DataLoader(TRAIN_CIFAR10, batch_size=cifar_config.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(cifar10_splits.train), num_workers=workers, pin_memory=True) + valid_cifar10_loader = torch.utils.data.DataLoader(temp_dataset , batch_size=cifar_config.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(cifar10_splits.valid), num_workers=workers, pin_memory=True) + test__cifar10_loader = torch.utils.data.DataLoader(VALID_CIFAR10, batch_size=cifar_config.batch_size, shuffle=False, num_workers=workers, pin_memory=True) + print ('CIFAR-10 : trval-loader has {:3d} batch with {:} per batch'.format(len(trainval_cifar10_loader), cifar_config.batch_size)) + print ('CIFAR-10 : train-loader has {:3d} batch with {:} per batch'.format(len(train_cifar10_loader), cifar_config.batch_size)) + print ('CIFAR-10 : valid-loader has {:3d} batch with {:} per batch'.format(len(valid_cifar10_loader), cifar_config.batch_size)) + print ('CIFAR-10 : test--loader has {:3d} batch with {:} per batch'.format(len(test__cifar10_loader), cifar_config.batch_size)) + print ('-'*200) + # CIFAR-100 + TRAIN_CIFAR100, VALID_CIFAR100, xshape, class_num = get_datasets('cifar100', str(torch_dir/'cifar.python'), -1) + print ('original CIFAR-100: {:} training images and {:} test images : {:} input shape : {:} number of classes'.format(len(TRAIN_CIFAR100), len(VALID_CIFAR100), xshape, class_num)) + cifar100_splits = load_config(root_dir / 'configs' / 'nas-benchmark' / 'cifar100-test-split.txt', None, None) + assert cifar100_splits.xvalid[:10] == [1, 3, 4, 5, 8, 10, 13, 14, 15, 16] and cifar100_splits.xtest[:10] == [0, 2, 6, 7, 9, 11, 12, 17, 20, 24] + train_cifar100_loader = torch.utils.data.DataLoader(TRAIN_CIFAR100, batch_size=cifar_config.batch_size, shuffle=True, num_workers=workers, pin_memory=True) + valid_cifar100_loader = torch.utils.data.DataLoader(VALID_CIFAR100, batch_size=cifar_config.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(cifar100_splits.xvalid), num_workers=workers, pin_memory=True) + test__cifar100_loader = torch.utils.data.DataLoader(VALID_CIFAR100, batch_size=cifar_config.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(cifar100_splits.xtest) , num_workers=workers, pin_memory=True) + print ('CIFAR-100 : train-loader has {:3d} batch'.format(len(train_cifar100_loader))) + print ('CIFAR-100 : valid-loader has {:3d} batch'.format(len(valid_cifar100_loader))) + print ('CIFAR-100 : test--loader has {:3d} batch'.format(len(test__cifar100_loader))) + print ('-'*200) + + imagenet16_config_path = 'configs/nas-benchmark/ImageNet-16.config' + imagenet16_config = load_config(imagenet16_config_path, None, None) + TRAIN_ImageNet16_120, VALID_ImageNet16_120, xshape, class_num = get_datasets('ImageNet16-120', str(torch_dir/'cifar.python'/'ImageNet16'), -1) + print ('original TRAIN_ImageNet16_120: {:} training images and {:} test images : {:} input shape : {:} number of classes'.format(len(TRAIN_ImageNet16_120), len(VALID_ImageNet16_120), xshape, class_num)) + imagenet_splits = load_config(root_dir / 'configs' / 'nas-benchmark' / 'imagenet-16-120-test-split.txt', None, None) + assert imagenet_splits.xvalid[:10] == [1, 2, 3, 6, 7, 8, 9, 12, 16, 18] and imagenet_splits.xtest[:10] == [0, 4, 5, 10, 11, 13, 14, 15, 17, 20] + train_imagenet_loader = torch.utils.data.DataLoader(TRAIN_ImageNet16_120, batch_size=imagenet16_config.batch_size, shuffle=True, num_workers=workers, pin_memory=True) + valid_imagenet_loader = torch.utils.data.DataLoader(VALID_ImageNet16_120, batch_size=imagenet16_config.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(imagenet_splits.xvalid), num_workers=workers, pin_memory=True) + test__imagenet_loader = torch.utils.data.DataLoader(VALID_ImageNet16_120, batch_size=imagenet16_config.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(imagenet_splits.xtest) , num_workers=workers, pin_memory=True) + print ('ImageNet-16-120 : train-loader has {:3d} batch with {:} per batch'.format(len(train_imagenet_loader), imagenet16_config.batch_size)) + print ('ImageNet-16-120 : valid-loader has {:3d} batch with {:} per batch'.format(len(valid_imagenet_loader), imagenet16_config.batch_size)) + print ('ImageNet-16-120 : test--loader has {:3d} batch with {:} per batch'.format(len(test__imagenet_loader), imagenet16_config.batch_size)) + + # 'cifar10', 'cifar100', 'ImageNet16-120' + loaders = {'cifar10@trainval': trainval_cifar10_loader, + 'cifar10@train' : train_cifar10_loader, + 'cifar10@valid' : valid_cifar10_loader, + 'cifar10@test' : test__cifar10_loader, + 'cifar100@train' : train_cifar100_loader, + 'cifar100@valid' : valid_cifar100_loader, + 'cifar100@test' : test__cifar100_loader, + 'ImageNet16-120@train': train_imagenet_loader, + 'ImageNet16-120@valid': valid_imagenet_loader, + 'ImageNet16-120@test' : test__imagenet_loader} + return loaders + + + +def simplify(save_dir, meta_file, basestr, target_dir): + meta_infos = torch.load(meta_file, map_location='cpu') + meta_archs = meta_infos['archs'] # a list of architecture strings + meta_num_archs = meta_infos['total'] + meta_max_node = meta_infos['max_node'] + assert meta_num_archs == len(meta_archs), 'invalid number of archs : {:} vs {:}'.format(meta_num_archs, len(meta_archs)) + + sub_model_dirs = sorted(list(save_dir.glob('*-*-{:}'.format(basestr)))) + print ('{:} find {:} directories used to save checkpoints'.format(time_string(), len(sub_model_dirs))) + + subdir2archs, num_evaluated_arch = collections.OrderedDict(), 0 + num_seeds = defaultdict(lambda: 0) + for index, sub_dir in enumerate(sub_model_dirs): + xcheckpoints = list(sub_dir.glob('arch-*-seed-*.pth')) + arch_indexes = set() + for checkpoint in xcheckpoints: + temp_names = checkpoint.name.split('-') + assert len(temp_names) == 4 and temp_names[0] == 'arch' and temp_names[2] == 'seed', 'invalid checkpoint name : {:}'.format(checkpoint.name) + arch_indexes.add( temp_names[1] ) + subdir2archs[sub_dir] = sorted(list(arch_indexes)) + num_evaluated_arch += len(arch_indexes) + # count number of seeds for each architecture + for arch_index in arch_indexes: + num_seeds[ len(list(sub_dir.glob('arch-{:}-seed-*.pth'.format(arch_index)))) ] += 1 + print('{:} There are {:5d} architectures that have been evaluated ({:} in total).'.format(time_string(), num_evaluated_arch, meta_num_archs)) + for key in sorted( list( num_seeds.keys() ) ): print ('{:} There are {:5d} architectures that are evaluated {:} times.'.format(time_string(), num_seeds[key], key)) + + dataloader_dict = GET_DataLoaders( 6 ) + + to_save_simply = save_dir / 'simplifies' + to_save_allarc = save_dir / 'simplifies' / 'architectures' + if not to_save_simply.exists(): to_save_simply.mkdir(parents=True, exist_ok=True) + if not to_save_allarc.exists(): to_save_allarc.mkdir(parents=True, exist_ok=True) + + assert (save_dir / target_dir) in subdir2archs, 'can not find {:}'.format(target_dir) + arch2infos, datasets = {}, ('cifar10-valid', 'cifar10', 'cifar100', 'ImageNet16-120') + evaluated_indexes = set() + target_directory = save_dir / target_dir + target_less_dir = save_dir / '{:}-LESS'.format(target_dir) + arch_indexes = subdir2archs[ target_directory ] + num_seeds = defaultdict(lambda: 0) + end_time = time.time() + arch_time = AverageMeter() + for idx, arch_index in enumerate(arch_indexes): + checkpoints = list(target_directory.glob('arch-{:}-seed-*.pth'.format(arch_index))) + ckps_less = list(target_less_dir.glob('arch-{:}-seed-*.pth'.format(arch_index))) + # create the arch info for each architecture + try: + arch_info_full = account_one_arch(arch_index, meta_archs[int(arch_index)], checkpoints, datasets, dataloader_dict) + arch_info_less = account_one_arch(arch_index, meta_archs[int(arch_index)], ckps_less, ['cifar10-valid'], dataloader_dict) + num_seeds[ len(checkpoints) ] += 1 + except: + print('Loading {:} failed, : {:}'.format(arch_index, checkpoints)) + continue + assert int(arch_index) not in evaluated_indexes, 'conflict arch-index : {:}'.format(arch_index) + assert 0 <= int(arch_index) < len(meta_archs), 'invalid arch-index {:} (not found in meta_archs)'.format(arch_index) + arch_info = {'full': arch_info_full, 'less': arch_info_less} + evaluated_indexes.add( int(arch_index) ) + arch2infos[int(arch_index)] = arch_info + torch.save({'full': arch_info_full.state_dict(), + 'less': arch_info_less.state_dict()}, to_save_allarc / '{:}-FULL.pth'.format(arch_index)) + arch_info['full'].clear_params() + arch_info['less'].clear_params() + torch.save({'full': arch_info_full.state_dict(), + 'less': arch_info_less.state_dict()}, to_save_allarc / '{:}-SIMPLE.pth'.format(arch_index)) + # measure elapsed time + arch_time.update(time.time() - end_time) + end_time = time.time() + need_time = '{:}'.format( convert_secs2time(arch_time.avg * (len(arch_indexes)-idx-1), True) ) + print('{:} {:} [{:03d}/{:03d}] : {:} still need {:}'.format(time_string(), target_dir, idx, len(arch_indexes), arch_index, need_time)) + # measure time + xstrs = ['{:}:{:03d}'.format(key, num_seeds[key]) for key in sorted( list( num_seeds.keys() ) ) ] + print('{:} {:} done : {:}'.format(time_string(), target_dir, xstrs)) + final_infos = {'meta_archs' : meta_archs, + 'total_archs': meta_num_archs, + 'basestr' : basestr, + 'arch2infos' : arch2infos, + 'evaluated_indexes': evaluated_indexes} + save_file_name = to_save_simply / '{:}.pth'.format(target_dir) + torch.save(final_infos, save_file_name) + print ('Save {:} / {:} architecture results into {:}.'.format(len(evaluated_indexes), meta_num_archs, save_file_name)) + + + +def merge_all(save_dir, meta_file, basestr): + meta_infos = torch.load(meta_file, map_location='cpu') + meta_archs = meta_infos['archs'] + meta_num_archs = meta_infos['total'] + meta_max_node = meta_infos['max_node'] + assert meta_num_archs == len(meta_archs), 'invalid number of archs : {:} vs {:}'.format(meta_num_archs, len(meta_archs)) + + sub_model_dirs = sorted(list(save_dir.glob('*-*-{:}'.format(basestr)))) + print ('{:} find {:} directories used to save checkpoints'.format(time_string(), len(sub_model_dirs))) + for index, sub_dir in enumerate(sub_model_dirs): + arch_info_files = sorted( list(sub_dir.glob('arch-*-seed-*.pth') ) ) + print ('The {:02d}/{:02d}-th directory : {:} : {:} runs.'.format(index, len(sub_model_dirs), sub_dir, len(arch_info_files))) + + arch2infos, evaluated_indexes = dict(), set() + for IDX, sub_dir in enumerate(sub_model_dirs): + ckp_path = sub_dir.parent / 'simplifies' / '{:}.pth'.format(sub_dir.name) + if ckp_path.exists(): + sub_ckps = torch.load(ckp_path, map_location='cpu') + assert sub_ckps['total_archs'] == meta_num_archs and sub_ckps['basestr'] == basestr + xarch2infos = sub_ckps['arch2infos'] + xevalindexs = sub_ckps['evaluated_indexes'] + for eval_index in xevalindexs: + assert eval_index not in evaluated_indexes and eval_index not in arch2infos + #arch2infos[eval_index] = xarch2infos[eval_index].state_dict() + arch2infos[eval_index] = {'full': xarch2infos[eval_index]['full'].state_dict(), + 'less': xarch2infos[eval_index]['less'].state_dict()} + evaluated_indexes.add( eval_index ) + print ('{:} [{:03d}/{:03d}] merge data from {:} with {:} models.'.format(time_string(), IDX, len(sub_model_dirs), ckp_path, len(xevalindexs))) + else: + raise ValueError('Can not find {:}'.format(ckp_path)) + #print ('{:} [{:03d}/{:03d}] can not find {:}, skip.'.format(time_string(), IDX, len(subdir2archs), ckp_path)) + + evaluated_indexes = sorted( list( evaluated_indexes ) ) + print ('Finally, there are {:} architectures that have been trained and evaluated.'.format(len(evaluated_indexes))) + + to_save_simply = save_dir / 'simplifies' + if not to_save_simply.exists(): to_save_simply.mkdir(parents=True, exist_ok=True) + final_infos = {'meta_archs' : meta_archs, + 'total_archs': meta_num_archs, + 'arch2infos' : arch2infos, + 'evaluated_indexes': evaluated_indexes} + save_file_name = to_save_simply / '{:}-final-infos.pth'.format(basestr) + torch.save(final_infos, save_file_name) + print ('Save {:} / {:} architecture results into {:}.'.format(len(evaluated_indexes), meta_num_archs, save_file_name)) + + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='NAS-BENCH-102', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--mode' , type=str, choices=['cal', 'merge'], help='The running mode for this script.') + parser.add_argument('--base_save_dir', type=str, default='./output/NAS-BENCH-102-4', help='The base-name of folder to save checkpoints and log.') + parser.add_argument('--target_dir' , type=str, help='The target directory.') + parser.add_argument('--max_node' , type=int, default=4, help='The maximum node in a cell.') + parser.add_argument('--channel' , type=int, default=16, help='The number of channels.') + parser.add_argument('--num_cells' , type=int, default=5, help='The number of cells in one stage.') + args = parser.parse_args() + + save_dir = Path( args.base_save_dir ) + meta_path = save_dir / 'meta-node-{:}.pth'.format(args.max_node) + assert save_dir.exists(), 'invalid save dir path : {:}'.format(save_dir) + assert meta_path.exists(), 'invalid saved meta path : {:}'.format(meta_path) + print ('start the statistics of our nas-benchmark from {:} using {:}.'.format(save_dir, args.target_dir)) + basestr = 'C{:}-N{:}'.format(args.channel, args.num_cells) + + if args.mode == 'cal': + simplify(save_dir, meta_path, basestr, args.target_dir) + elif args.mode == 'merge': + merge_all(save_dir, meta_path, basestr) + else: + raise ValueError('invalid mode : {:}'.format(args.mode)) diff --git a/exps/algos/BOHB.py b/exps/algos/BOHB.py index 9a400ab..ad38dae 100644 --- a/exps/algos/BOHB.py +++ b/exps/algos/BOHB.py @@ -17,7 +17,7 @@ from datasets import get_datasets, SearchDataset from procedures import prepare_seed, prepare_logger, save_checkpoint, copy_checkpoint, get_optim_scheduler from utils import get_model_infos, obtain_accuracy from log_utils import AverageMeter, time_string, convert_secs2time -from aa_nas_api import AANASBenchAPI +from nas_102_api import NASBench102API as API from models import CellStructure, get_search_spaces from R_EA import train_and_eval # BOHB: Robust and Efficient Hyperparameter Optimization at Scale, ICML 2018 @@ -112,7 +112,7 @@ def main(xargs, nas_bench): num_workers = 1 #nas_bench = AANASBenchAPI(xargs.arch_nas_dataset) - logger.log('{:} Create AA-NAS-BENCH-API DONE'.format(time_string())) + #logger.log('{:} Create NAS-BENCH-API DONE'.format(time_string())) workers = [] for i in range(num_workers): w = MyWorker(nameserver=ns_host, nameserver_port=ns_port, convert_func=config2structure, nas_bench=nas_bench, run_id=hb_run_id, id=i) @@ -179,7 +179,7 @@ if __name__ == '__main__': nas_bench = None else: print ('{:} build NAS-Benchmark-API from {:}'.format(time_string(), args.arch_nas_dataset)) - nas_bench = AANASBenchAPI(args.arch_nas_dataset) + nas_bench = API(args.arch_nas_dataset) if args.rand_seed < 0: save_dir, all_indexes, num = None, [], 500 for i in range(num): diff --git a/exps/algos/RANDOM.py b/exps/algos/RANDOM.py index d367e71..ad306c5 100644 --- a/exps/algos/RANDOM.py +++ b/exps/algos/RANDOM.py @@ -15,7 +15,7 @@ from procedures import prepare_seed, prepare_logger, save_checkpoint, copy_che from utils import get_model_infos, obtain_accuracy from log_utils import AverageMeter, time_string, convert_secs2time from models import get_search_spaces -from aa_nas_api import AANASBenchAPI +from nas_102_api import NASBench102API as API from R_EA import train_and_eval, random_architecture_func @@ -92,7 +92,7 @@ if __name__ == '__main__': nas_bench = None else: print ('{:} build NAS-Benchmark-API from {:}'.format(time_string(), args.arch_nas_dataset)) - nas_bench = AANASBenchAPI(args.arch_nas_dataset) + nas_bench = API(args.arch_nas_dataset) if args.rand_seed < 0: save_dir, all_indexes, num = None, [], 500 for i in range(num): diff --git a/exps/algos/R_EA.py b/exps/algos/R_EA.py index e66957c..03a38ed 100644 --- a/exps/algos/R_EA.py +++ b/exps/algos/R_EA.py @@ -16,7 +16,7 @@ from datasets import get_datasets, SearchDataset from procedures import prepare_seed, prepare_logger, save_checkpoint, copy_checkpoint, get_optim_scheduler from utils import get_model_infos, obtain_accuracy from log_utils import AverageMeter, time_string, convert_secs2time -from aa_nas_api import AANASBenchAPI +from nas_102_api import NASBench102API as API from models import CellStructure, get_search_spaces @@ -230,7 +230,7 @@ if __name__ == '__main__': nas_bench = None else: print ('{:} build NAS-Benchmark-API from {:}'.format(time_string(), args.arch_nas_dataset)) - nas_bench = AANASBenchAPI(args.arch_nas_dataset) + nas_bench = API(args.arch_nas_dataset) if args.rand_seed < 0: save_dir, all_indexes, num = None, [], 500 for i in range(num): diff --git a/exps/algos/reinforce.py b/exps/algos/reinforce.py index 80a4e7b..389a0ce 100644 --- a/exps/algos/reinforce.py +++ b/exps/algos/reinforce.py @@ -17,7 +17,7 @@ from datasets import get_datasets, SearchDataset from procedures import prepare_seed, prepare_logger, save_checkpoint, copy_checkpoint, get_optim_scheduler from utils import get_model_infos, obtain_accuracy from log_utils import AverageMeter, time_string, convert_secs2time -from aa_nas_api import AANASBenchAPI +from nas_102_api import NASBench102API from models import CellStructure, get_search_spaces from R_EA import train_and_eval diff --git a/exps/vis/test.py b/exps/vis/test.py new file mode 100644 index 0000000..1c6f2b2 --- /dev/null +++ b/exps/vis/test.py @@ -0,0 +1,27 @@ +# python ./exps/vis/test.py +import os, sys +from pathlib import Path +import torch +import numpy as np +from collections import OrderedDict +lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve() +if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) + + +def test_nas_api(): + from nas_102_api import ArchResults + xdata = torch.load('/home/dxy/FOR-RELEASE/NAS-Projects/output/NAS-BENCH-102-4/simplifies/architectures/000157-FULL.pth') + for key in ['full', 'less']: + print ('\n------------------------- {:} -------------------------'.format(key)) + archRes = ArchResults.create_from_state_dict(xdata[key]) + print(archRes) + print(archRes.arch_idx_str()) + print(archRes.get_dataset_names()) + print(archRes.get_comput_costs('cifar10-valid')) + # get the metrics + print(archRes.get_metrics('cifar10-valid', 'x-valid', None, False)) + print(archRes.get_metrics('cifar10-valid', 'x-valid', None, True)) + print(archRes.query('cifar10-valid', 777)) + +if __name__ == '__main__': + test_nas_api() diff --git a/lib/aa_nas_api/__init__.py b/lib/nas_102_api/__init__.py similarity index 85% rename from lib/aa_nas_api/__init__.py rename to lib/nas_102_api/__init__.py index 594669b..311909d 100644 --- a/lib/aa_nas_api/__init__.py +++ b/lib/nas_102_api/__init__.py @@ -1,5 +1,5 @@ ################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # ################################################## -from .api import AANASBenchAPI +from .api import NASBench102API from .api import ArchResults, ResultsCount diff --git a/lib/aa_nas_api/api.py b/lib/nas_102_api/api.py similarity index 72% rename from lib/aa_nas_api/api.py rename to lib/nas_102_api/api.py index db2a310..07de2f7 100644 --- a/lib/aa_nas_api/api.py +++ b/lib/nas_102_api/api.py @@ -2,7 +2,7 @@ # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # ################################################## import os, sys, copy, random, torch, numpy as np -from collections import OrderedDict +from collections import OrderedDict, defaultdict def print_information(information, extra_info=None, show=False): @@ -30,16 +30,17 @@ def print_information(information, extra_info=None, show=False): return strings -class AANASBenchAPI(object): +class NASBench102API(object): def __init__(self, file_path_or_dict, verbose=True): if isinstance(file_path_or_dict, str): - if verbose: print('try to create AA-NAS-Bench api from {:}'.format(file_path_or_dict)) + if verbose: print('try to create NAS-Bench-102 api from {:}'.format(file_path_or_dict)) assert os.path.isfile(file_path_or_dict), 'invalid path : {:}'.format(file_path_or_dict) file_path_or_dict = torch.load(file_path_or_dict) else: file_path_or_dict = copy.deepcopy( file_path_or_dict ) assert isinstance(file_path_or_dict, dict), 'It should be a dict instead of {:}'.format(type(file_path_or_dict)) + import pdb; pdb.set_trace() # we will update this api soon keys = ('meta_archs', 'arch2infos', 'evaluated_indexes') for key in keys: assert key in file_path_or_dict, 'Can not find key[{:}] in the dict'.format(key) self.meta_archs = copy.deepcopy( file_path_or_dict['meta_archs'] ) @@ -144,27 +145,46 @@ class ArchResults(object): def get_comput_costs(self, dataset): x_seeds = self.dataset_seed[dataset] results = [self.all_results[ (dataset, seed) ] for seed in x_seeds] - flops = [result.flop for result in results] - params = [result.params for result in results] + + flops = [result.flop for result in results] + params = [result.params for result in results] lantencies = [result.get_latency() for result in results] - return np.mean(flops), np.mean(params), np.mean(lantencies) + lantencies = [x for x in lantencies if x > 0] + mean_latency = np.mean(lantencies) if len(lantencies) > 0 else None + time_infos = defaultdict(list) + for result in results: + time_info = result.get_times() + for key, value in time_info.items(): time_infos[key].append( value ) + + info = {'flops' : np.mean(flops), + 'params' : np.mean(params), + 'latency': mean_latency} + for key, value in time_infos.items(): + if len(value) > 0 and value[0] is not None: + info[key] = np.mean(value) + else: info[key] = None + return info def get_metrics(self, dataset, setname, iepoch=None, is_random=False): x_seeds = self.dataset_seed[dataset] results = [self.all_results[ (dataset, seed) ] for seed in x_seeds] - loss, accuracy = [], [] + infos = defaultdict(list) for result in results: if setname == 'train': info = result.get_train(iepoch) else: info = result.get_eval(setname, iepoch) - loss.append( info['loss'] ) - accuracy.append( info['accuracy'] ) + for key, value in info.items(): infos[key].append( value ) + return_info = dict() if is_random: - index = random.randint(0, len(loss)-1) - return loss[index], accuracy[index] + index = random.randint(0, len(results)-1) + for key, value in infos.items(): return_info[key] = value[index] else: - return float(np.mean(loss)), float(np.mean(accuracy)) + for key, value in infos.items(): + if len(value) > 0 and value[0] is not None: + return_info[key] = np.mean(value) + else: return_info[key] = None + return return_info def show(self, is_print=False): return print_information(self, None, is_print) @@ -245,8 +265,10 @@ class ResultsCount(object): def __init__(self, name, state_dict, train_accs, train_losses, params, flop, arch_config, seed, epochs, latency): self.name = name self.net_state_dict = state_dict - self.train_accs = copy.deepcopy(train_accs) + self.train_acc1es = copy.deepcopy(train_accs) + self.train_acc5es = None self.train_losses = copy.deepcopy(train_losses) + self.train_times = None self.arch_config = copy.deepcopy(arch_config) self.params = params self.flop = flop @@ -256,44 +278,97 @@ class ResultsCount(object): # evaluation results self.reset_eval() + def update_train_info(self, train_acc1es, train_acc5es, train_losses, train_times): + self.train_acc1es = train_acc1es + self.train_acc5es = train_acc5es + self.train_losses = train_losses + self.train_times = train_times + def reset_eval(self): self.eval_names = [] - self.eval_accs = {} + self.eval_acc1es = {} + self.eval_times = {} self.eval_losses = {} def update_latency(self, latency): self.latency = copy.deepcopy( latency ) + def update_eval(self, accs, losses, times): # old version + data_names = set([x.split('@')[0] for x in accs.keys()]) + for data_name in data_names: + assert data_name not in self.eval_names, '{:} has already been added into eval-names'.format(data_name) + self.eval_names.append( data_name ) + for iepoch in range(self.epochs): + xkey = '{:}@{:}'.format(data_name, iepoch) + self.eval_acc1es[ xkey ] = accs[ xkey ] + self.eval_losses[ xkey ] = losses[ xkey ] + self.eval_times [ xkey ] = times[ xkey ] + + def update_OLD_eval(self, name, accs, losses): # old version + assert name not in self.eval_names, '{:} has already added'.format(name) + self.eval_names.append( name ) + for iepoch in range(self.epochs): + if iepoch in accs: + self.eval_acc1es['{:}@{:}'.format(name,iepoch)] = accs[iepoch] + self.eval_losses['{:}@{:}'.format(name,iepoch)] = losses[iepoch] + + def __repr__(self): + num_eval = len(self.eval_names) + set_name = '[' + ', '.join(self.eval_names) + ']' + return ('{name}({xname}, arch={arch}, FLOP={flop:.2f}M, Param={param:.3f}MB, seed={seed}, {num_eval} eval-sets: {set_name})'.format(name=self.__class__.__name__, xname=self.name, arch=self.arch_config['arch_str'], flop=self.flop, param=self.params, seed=self.seed, num_eval=num_eval, set_name=set_name)) + def get_latency(self): if self.latency is None: return -1 else: return sum(self.latency) / len(self.latency) - def update_eval(self, name, accs, losses): - assert name not in self.eval_names, '{:} has already added'.format(name) - self.eval_names.append( name ) - self.eval_accs[name] = copy.deepcopy( accs ) - self.eval_losses[name] = copy.deepcopy( losses ) + def get_times(self): + if self.train_times is not None and isinstance(self.train_times, dict): + train_times = list( self.train_times.values() ) + time_info = {'T-train@epoch': np.mean(train_times), 'T-train@total': np.sum(train_times)} + for name in self.eval_names: + xtimes = [self.eval_times['{:}@{:}'.format(name,i)] for i in range(self.epochs)] + time_info['T-{:}@epoch'.format(name)] = np.mean(xtimes) + time_info['T-{:}@total'.format(name)] = np.sum(xtimes) + else: + time_info = {'T-train@epoch': None, 'T-train@total': None } + for name in self.eval_names: + time_info['T-{:}@epoch'.format(name)] = None + time_info['T-{:}@total'.format(name)] = None + return time_info - def __repr__(self): - num_eval = len(self.eval_names) - return ('{name}({xname}, arch={arch}, FLOP={flop:.2f}M, Param={param:.3f}MB, seed={seed}, {num_eval} eval-sets)'.format(name=self.__class__.__name__, xname=self.name, arch=self.arch_config['arch_str'], flop=self.flop, param=self.params, seed=self.seed, num_eval=num_eval)) - - def valid_evaluation_set(self): + def get_eval_set(self): return self.eval_names def get_train(self, iepoch=None): if iepoch is None: iepoch = self.epochs-1 assert 0 <= iepoch < self.epochs, 'invalid iepoch={:} < {:}'.format(iepoch, self.epochs) - return {'loss': self.train_losses[iepoch], 'accuracy': self.train_accs[iepoch]} + if self.train_times is not None: xtime = self.train_times[iepoch] + else : xtime = None + return {'iepoch' : iepoch, + 'loss' : self.train_losses[iepoch], + 'accuracy': self.train_acc1es[iepoch], + 'time' : xtime} def get_eval(self, name, iepoch=None): if iepoch is None: iepoch = self.epochs-1 assert 0 <= iepoch < self.epochs, 'invalid iepoch={:} < {:}'.format(iepoch, self.epochs) - return {'loss': self.eval_losses[name][iepoch], 'accuracy': self.eval_accs[name][iepoch]} + if isinstance(self.eval_times,dict) and len(self.eval_times) > 0: + xtime = self.eval_times['{:}@{:}'.format(name,iepoch)] + else: xtime = None + return {'iepoch' : iepoch, + 'loss' : self.eval_losses['{:}@{:}'.format(name,iepoch)], + 'accuracy': self.eval_acc1es['{:}@{:}'.format(name,iepoch)], + 'time' : xtime} def get_net_param(self): return self.net_state_dict + def get_config(self, str2structure): + #return copy.deepcopy(self.arch_config) + return {'name': 'infer.tiny', 'C': self.arch_config['channel'], \ + 'N' : self.arch_config['num_cells'], \ + 'genotype': str2structure(self.arch_config['arch_str']), 'num_classes': self.arch_config['class_num']} + def state_dict(self): _state_dict = {key: value for key, value in self.__dict__.items()} return _state_dict