diff --git a/README.md b/README.md index fd3d7fd..b66dfd1 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,8 @@ At the moment, this project provides the following algorithms and scripts to run ... ENAS / REA / REINFORCE / BOHB - NAS-Bench-201.md + Please check the original papers + NAS-Bench-201.md NATS-Bench.md HPO diff --git a/README_CN.md b/README_CN.md index 2b6aedf..f517d0e 100644 --- a/README_CN.md +++ b/README_CN.md @@ -59,10 +59,17 @@ NAS-Bench-201: Extending the Scope of Reproducible Neural Architecture Search NAS-Bench-201.md + + NATS-Bench + NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size + NATS-Bench.md + + ... ENAS / REA / REINFORCE / BOHB - NAS-Bench-201.md + Please check the original papers. + NAS-Bench-201.md NATS-Bench.md HPO diff --git a/docs/NATS-Bench.md b/docs/NATS-Bench.md index a38f2eb..0df2b1a 100644 --- a/docs/NATS-Bench.md +++ b/docs/NATS-Bench.md @@ -7,11 +7,23 @@ We analyze the validity of our benchmark in terms of various criteria and perfor We also show the versatility of NATS-Bench by benchmarking 13 recent state-of-the-art NAS algorithms on it. All logs and diagnostic information trained using the same setup for each candidate are provided. This facilitates a much larger community of researchers to focus on developing better NAS algorithms in a more comparable and computationally effective environment. + **coming soon!** + ## How to Use NATS-Bench +## The Procedure of Creating NATS-Bench + +1, train all architecture candidate in the size search space with 90 epochs and use the random seed of `777`. +``` +bash ./scripts/NATS-Bench/train-shapes.sh 00000-32767 90 777 +``` +The checkpoint of all candidates are located at `output/NATS-Bench-size` by default + + + ## To Reproduce 13 Baseline NAS Algorithms in NAS-Bench-201 ### Reproduce NAS methods on the topology search space @@ -50,6 +62,7 @@ python ./exps/algos-v2/search-cell.py --dataset ImageNet16-120 --data_path $TORC ### Reproduce NAS methods on the size search space + ### Final Discovered Architectures for Each Algorithm The architecture index can be found by use `api.query_index_by_arch(architecture_string)`. diff --git a/exps/NAS-Bench-201/xshapes.py b/exps/NATS-Bench/main-sss.py similarity index 92% rename from exps/NAS-Bench-201/xshapes.py rename to exps/NATS-Bench/main-sss.py index 9574aa5..78c32d1 100644 --- a/exps/NAS-Bench-201/xshapes.py +++ b/exps/NATS-Bench/main-sss.py @@ -1,8 +1,8 @@ -############################################################### -# NAS-Bench-201, ICLR 2020 (https://arxiv.org/abs/2001.00326) # -############################################################### -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # -############################################################### +############################################################################## +# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +############################################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.07 # +############################################################################## import os, sys, time, torch, argparse from typing import List, Text, Dict, Any from PIL import ImageFile @@ -189,9 +189,9 @@ def filter_indexes(xlist, mode, save_dir, seeds): if __name__ == '__main__': - parser = argparse.ArgumentParser(description='NAS-Bench-X', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description='NATS-Bench (size search space)', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--mode', type=str, required=True, choices=['new', 'cover'], help='The script mode.') - parser.add_argument('--save_dir', type=str, default='output/NAS-BENCH-202', help='Folder to save checkpoints and log.') + parser.add_argument('--save_dir', type=str, default='output/NATS-Bench-size', help='Folder to save checkpoints and log.') parser.add_argument('--candidateC', type=int, nargs='+', default=[8, 16, 24, 32, 40, 48, 56, 64], help='.') parser.add_argument('--num_layers', type=int, default=5, help='The number of layers in a network.') parser.add_argument('--check_N', type=int, default=32768, help='For safety.') @@ -206,10 +206,12 @@ if __name__ == '__main__': args = parser.parse_args() nets = traverse_net(args.candidateC, args.num_layers) - if len(nets) != args.check_N: raise ValueError('Pre-num-check failed : {:} vs {:}'.format(len(nets), args.check_N)) + if len(nets) != args.check_N: + raise ValueError('Pre-num-check failed : {:} vs {:}'.format(len(nets), args.check_N)) opt_config = './configs/nas-benchmark/hyper-opts/{:}E.config'.format(args.hyper) - if not os.path.isfile(opt_config): raise ValueError('{:} is not a file.'.format(opt_config)) + if not os.path.isfile(opt_config): + raise ValueError('{:} is not a file.'.format(opt_config)) save_dir = Path(args.save_dir) / 'raw-data-{:}'.format(args.hyper) save_dir.mkdir(parents=True, exist_ok=True) if not isinstance(args.srange, str): @@ -218,7 +220,8 @@ if __name__ == '__main__': to_evaluate_indexes = set() for srange in srangestr.split(','): srange = srange.split('-') - if len(srange) != 2: raise ValueError('invalid srange : {:}'.format(srange)) + if len(srange) != 2: + raise ValueError('invalid srange : {:}'.format(srange)) assert len(srange[0]) == len(srange[1]) == 5, 'invalid srange : {:}'.format(srange) srange = (int(srange[0]), int(srange[1])) if not (0 <= srange[0] <= srange[1] < args.check_N): @@ -226,10 +229,12 @@ if __name__ == '__main__': for i in range(srange[0], srange[1]+1): to_evaluate_indexes.add(i) - assert len(args.seeds) > 0, 'invalid length of seeds args: {:}'.format(args.seeds) + if not len(args.seeds): + raise ValueError('invalid length of seeds args: {:}'.format(args.seeds)) if not (len(args.datasets) == len(args.xpaths) == len(args.splits)): raise ValueError('invalid infos : {:} vs {:} vs {:}'.format(len(args.datasets), len(args.xpaths), len(args.splits))) - assert args.workers > 0, 'invalid number of workers : {:}'.format(args.workers) + if args.workers <= 0: + raise ValueError('invalid number of workers : {:}'.format(args.workers)) target_indexes = filter_indexes(to_evaluate_indexes, args.mode, save_dir, args.seeds) @@ -239,4 +244,3 @@ if __name__ == '__main__': torch.set_num_threads(args.workers) main(save_dir, args.workers, args.datasets, args.xpaths, args.splits, tuple(args.seeds), nets, opt_config, target_indexes, args.mode == 'cover') - diff --git a/exps/NAS-Bench-201/xshape-file.py b/exps/NATS-Bench/sss-file-manager.py similarity index 80% rename from exps/NAS-Bench-201/xshape-file.py rename to exps/NATS-Bench/sss-file-manager.py index 49d48c2..1285ecd 100644 --- a/exps/NAS-Bench-201/xshape-file.py +++ b/exps/NATS-Bench/sss-file-manager.py @@ -1,10 +1,10 @@ -############################################################### -# NAS-Bench-201, ICLR 2020 (https://arxiv.org/abs/2001.00326) # -############################################################### -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.01 # -############################################################### -# Usage: python exps/NAS-Bench-201/xshape-file.py --mode check -############################################################### +############################################################################## +# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +############################################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.01 # +############################################################################## +# Usage: python exps/NATS-Bench/sss-file-manager.py --mode check # +############################################################################## import os, sys, time, torch, argparse from typing import List, Text, Dict, Any from shutil import copyfile @@ -55,9 +55,9 @@ def copy_data(source_dir, target_dir, meta_path): if __name__ == '__main__': - parser = argparse.ArgumentParser(description='NAS-Bench-X', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description='NATS-Bench (size search space) file manager.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--mode', type=str, required=True, choices=['check', 'copy'], help='The script mode.') - parser.add_argument('--save_dir', type=str, default='output/NAS-BENCH-202', help='Folder to save checkpoints and log.') + parser.add_argument('--save_dir', type=str, default='output/NATS-Bench-size', help='Folder to save checkpoints and log.') parser.add_argument('--check_N', type=int, default=32768, help='For safety.') # use for train the model args = parser.parse_args() diff --git a/scripts-search/NASNet-space-search-by-GDAS-FRC.sh b/scripts-search/NASNet-space-search-by-GDAS-FRC.sh index 22d26c9..ed0c085 100644 --- a/scripts-search/NASNet-space-search-by-GDAS-FRC.sh +++ b/scripts-search/NASNet-space-search-by-GDAS-FRC.sh @@ -25,7 +25,7 @@ else data_path="$TORCH_HOME/cifar.python/ImageNet16" fi -save_dir=./output/search-cell-${space}/GDAS-${dataset}-BN${track_running_stats} +save_dir=./output/search-cell-${space}/GDAS-FRC-${dataset}-BN${track_running_stats} OMP_NUM_THREADS=4 python ./exps/algos/GDAS.py \ --save_dir ${save_dir} \ diff --git a/scripts-search/X-X/train-shapes-01.sh b/scripts-search/X-X/train-shapes-01.sh deleted file mode 100644 index a9b89f1..0000000 --- a/scripts-search/X-X/train-shapes-01.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -##################################################### -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.01 # -##################################################### -# bash ./scripts-search/X-X/train-shapes-01.sh 0 4 -echo script name: $0 -echo $# arguments -if [ "$#" -ne 2 ] ;then - echo "Input illegal number of parameters " $# - echo "Need 2 parameters for hyper-parameters-opt-file, and seeds" - exit 1 -fi -if [ "$TORCH_HOME" = "" ]; then - echo "Must set TORCH_HOME envoriment variable for data dir saving" - exit 1 -else - echo "TORCH_HOME : $TORCH_HOME" -fi - -srange=00000-32767 -opt=01 -all_seeds=777 -cpus=4 - -save_dir=./output/NAS-BENCH-202/ - -SLURM_PROCID=$1 SLURM_NTASKS=$2 OMP_NUM_THREADS=${cpus} python exps/NAS-Bench-201/xshapes.py \ - --mode new --srange ${srange} --hyper ${opt} --save_dir ${save_dir} \ - --datasets cifar10 cifar10 cifar100 ImageNet16-120 \ - --splits 1 0 0 0 \ - --xpaths $TORCH_HOME/cifar.python \ - $TORCH_HOME/cifar.python \ - $TORCH_HOME/cifar.python \ - $TORCH_HOME/cifar.python/ImageNet16 \ - --workers ${cpus} \ - --seeds ${all_seeds} diff --git a/scripts-search/X-X/train-shapes-v2.sh b/scripts-search/X-X/train-shapes-v2.sh deleted file mode 100644 index 55d049e..0000000 --- a/scripts-search/X-X/train-shapes-v2.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -##################################################### -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.01 # -##################################################### -# SLURM_PROCID=0 SLURM_NTASKS=6 bash ./scripts-search/X-X/train-shapes-v2.sh 12 777 -# -# SLURM_PROCID=0 SLURM_NTASKS=4 bash ./scripts-search/X-X/train-shapes.sh 30000-32767 90 777 -# SLURM_PROCID=0 SLURM_NTASKS=4 bash ./scripts-search/X-X/train-shapes.sh 00000-09999 90 777 -# -echo script name: $0 -echo $# arguments -if [ "$#" -ne 2 ] ;then - echo "Input illegal number of parameters " $# - echo "Need 2 parameters for hyper-parameters-opt-file, and seeds" - exit 1 -fi -if [ "$TORCH_HOME" = "" ]; then - echo "Must set TORCH_HOME envoriment variable for data dir saving" - exit 1 -else - echo "TORCH_HOME : $TORCH_HOME" -fi - -#srange=01000-03999,04050-05000,06000-09000,11000-14500,15000-18500,20000-23500,25000-27500,29000-30000 -#srange=00000-00999,04000-04049,05001-05999,09001-10999,14501-14999,18501-19999,23501-24999,27501-28999,30001-32767 -#srange=00000-09999 -srange=10000-29999 -opt=$1 -all_seeds=$2 -cpus=4 - -save_dir=./output/NAS-BENCH-202/ - -OMP_NUM_THREADS=${cpus} python exps/NAS-Bench-201/xshapes.py \ - --mode new --srange ${srange} --hyper ${opt} --save_dir ${save_dir} \ - --datasets cifar10 cifar10 cifar100 ImageNet16-120 \ - --splits 1 0 0 0 \ - --xpaths $TORCH_HOME/cifar.python \ - $TORCH_HOME/cifar.python \ - $TORCH_HOME/cifar.python \ - $TORCH_HOME/cifar.python/ImageNet16 \ - --workers ${cpus} \ - --seeds ${all_seeds} diff --git a/scripts-search/X-X/train-shapes.sh b/scripts-search/X-X/train-shapes.sh deleted file mode 100644 index 5dd9cbf..0000000 --- a/scripts-search/X-X/train-shapes.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -##################################################### -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.01 # -##################################################### -# [mars6] CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/X-X/train-shapes.sh 00000-05000 12 777 -# [mars6] bash ./scripts-search/X-X/train-shapes.sh 05001-10000 12 777 -# [mars20] bash ./scripts-search/X-X/train-shapes.sh 10001-14500 12 777 -# [mars20] bash ./scripts-search/X-X/train-shapes.sh 14501-18000 12 777 -# [saturn4] bash ./scripts-search/X-X/train-shapes.sh 18001-19500 12 777 -# [saturn4] bash ./scripts-search/X-X/train-shapes.sh 19501-23500 12 777 -# [saturn4] bash ./scripts-search/X-X/train-shapes.sh 23501-27500 12 777 -# [saturn4] bash ./scripts-search/X-X/train-shapes.sh 27501-30000 12 777 -# [x] bash ./scripts-search/X-X/train-shapes.sh 30001-32767 12 777 -# -# CUDA_VISIBLE_DEVICES=2 bash ./scripts-search/X-X/train-shapes.sh 01000-03999,04050-05000,06000-09000,11000-14500,15000-18500,20000-23500,25000-27500,29000-30000 12 777 -# SLURM_PROCID=1 SLURM_NTASKS=5 bash ./scripts-search/X-X/train-shapes.sh 01000-03999,04050-05000,06000-09000,11000-14500,15000-18500,20000-23500,25000-27500,29000-30000 90 777 -# [GCP] bash ./scripts-search/X-X/train-shapes.sh 00000-09999 90 777 -# [UTS] bash ./scripts-search/X-X/train-shapes.sh 30000-32767 90 777 -echo script name: $0 -echo $# arguments -if [ "$#" -ne 3 ] ;then - echo "Input illegal number of parameters " $# - echo "Need 3 parameters for start-and-end, hyper-parameters-opt-file, and seeds" - exit 1 -fi -if [ "$TORCH_HOME" = "" ]; then - echo "Must set TORCH_HOME envoriment variable for data dir saving" - exit 1 -else - echo "TORCH_HOME : $TORCH_HOME" -fi - -srange=$1 -opt=$2 -all_seeds=$3 -cpus=4 - -save_dir=./output/NAS-BENCH-202/ - -OMP_NUM_THREADS=${cpus} python exps/NAS-Bench-201/xshapes.py \ - --mode new --srange ${srange} --hyper ${opt} --save_dir ${save_dir} \ - --datasets cifar10 cifar10 cifar100 ImageNet16-120 \ - --splits 1 0 0 0 \ - --xpaths $TORCH_HOME/cifar.python \ - $TORCH_HOME/cifar.python \ - $TORCH_HOME/cifar.python \ - $TORCH_HOME/cifar.python/ImageNet16 \ - --workers ${cpus} \ - --seeds ${all_seeds} diff --git a/scripts/NATS-Bench/train-shapes.sh b/scripts/NATS-Bench/train-shapes.sh new file mode 100644 index 0000000..b261fe2 --- /dev/null +++ b/scripts/NATS-Bench/train-shapes.sh @@ -0,0 +1,52 @@ +#!/bin/bash +############################################################################## +# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +############################################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.01 # +############################################################################## +# CUDA_VISIBLE_DEVICES=0 bash scripts/NATS-Bench/train-shapes.sh 00000-05000 12 777 +# bash ./scripts/NATS-Bench/train-shapes.sh 05001-10000 12 777 +# bash ./scripts/NATS-Bench/train-shapes.sh 10001-14500 12 777 +# bash ./scripts/NATS-Bench/train-shapes.sh 14501-18000 12 777 +# bash ./scripts/NATS-Bench/train-shapes.sh 18001-19500 12 777 +# bash ./scripts/NATS-Bench/train-shapes.sh 19501-23500 12 777 +# bash ./scripts/NATS-Bench/train-shapes.sh 23501-27500 12 777 +# bash ./scripts/NATS-Bench/train-shapes.sh 27501-30000 12 777 +# bash ./scripts/NATS-Bench/train-shapes.sh 30001-32767 12 777 +# +# CUDA_VISIBLE_DEVICES=2 bash ./scripts/NATS-Bench/train-shapes.sh 01000-03999,04050-05000,06000-09000,11000-14500,15000-18500,20000-23500,25000-27500,29000-30000 12 777 +# SLURM_PROCID=1 SLURM_NTASKS=5 bash ./scripts/NATS-Bench/train-shapes.sh 01000-03999,04050-05000,06000-09000,11000-14500,15000-18500,20000-23500,25000-27500,29000-30000 90 777 +# [GCP] bash ./scripts/NATS-Bench/train-shapes.sh 00000-09999 90 777 +# [UTS] bash ./scripts/NATS-Bench/train-shapes.sh 30000-32767 90 777 +############################################################################## +echo script name: $0 +echo $# arguments +if [ "$#" -ne 3 ] ;then + echo "Input illegal number of parameters " $# + echo "Need 3 parameters for start-and-end, hyper-parameters-opt-file, and seeds" + exit 1 +fi +if [ "$TORCH_HOME" = "" ]; then + echo "Must set TORCH_HOME envoriment variable for data dir saving" + exit 1 +else + echo "TORCH_HOME : $TORCH_HOME" +fi + +srange=$1 +opt=$2 +all_seeds=$3 +cpus=4 + +save_dir=./output/NATS-Bench-size/ + +OMP_NUM_THREADS=${cpus} python exps/NATS-Bench/main-sss.py \ + --mode new --srange ${srange} --hyper ${opt} --save_dir ${save_dir} \ + --datasets cifar10 cifar10 cifar100 ImageNet16-120 \ + --splits 1 0 0 0 \ + --xpaths $TORCH_HOME/cifar.python \ + $TORCH_HOME/cifar.python \ + $TORCH_HOME/cifar.python \ + $TORCH_HOME/cifar.python/ImageNet16 \ + --workers ${cpus} \ + --seeds ${all_seeds}