diff --git a/exps/NAS-Bench-201/xshape-file.py b/exps/NAS-Bench-201/xshape-file.py new file mode 100644 index 0000000..0eee246 --- /dev/null +++ b/exps/NAS-Bench-201/xshape-file.py @@ -0,0 +1,59 @@ +############################################################### +# NAS-Bench-201, ICLR 2020 (https://arxiv.org/abs/2001.00326) # +############################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.01 # +############################################################### +# Usage: python exps/NAS-Bench-201/xshape-file.py --mode check +############################################################### +import os, sys, time, torch, argparse +from typing import List, Text, Dict, Any +from tqdm import tqdm +from collections import defaultdict +from copy import deepcopy +from pathlib import Path + +lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve() +if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) +from config_utils import dict2config, load_config +from procedures import bench_evaluate_for_seed +from procedures import get_machine_info +from datasets import get_datasets +from log_utils import Logger, AverageMeter, time_string, convert_secs2time + + +def obtain_valid_ckp(save_dir: Text, total: int): + possible_seeds = [777, 888] + seed2ckps = defaultdict(list) + miss2ckps = defaultdict(list) + for i in range(total): + for seed in possible_seeds: + path = os.path.join(save_dir, 'arch-{:06d}-seed-{:04d}.pth'.format(i, seed)) + if os.path.exists(path): + seed2ckps[seed].append(i) + else: + miss2ckps[seed].append(i) + """ + ckps = [x for x in save_dir.glob('arch-{:06d}-seed-*.pth'.format(i))] + for ckp in ckps: + seed = ckp.name.split('-seed-')[-1].split('.pth')[0] + seed2ckps[int(seed)].append(i) + """ + for seed, xlist in seed2ckps.items(): + print('[{:}] [seed={:}] has {:}/{:}'.format(save_dir, seed, len(xlist), total)) + return dict(seed2ckps), dict(miss2ckps) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='NAS-Bench-X', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--mode', type=str, required=True, choices=['check', 'copy'], help='The script mode.') + parser.add_argument('--save_dir', type=str, default='output/NAS-BENCH-202', help='Folder to save checkpoints and log.') + parser.add_argument('--check_N', type=int, default=32768, help='For safety.') + # use for train the model + args = parser.parse_args() + possible_configs = ['01', '12', '90'] + if args.mode == 'check': + for config in possible_configs: + cur_save_dir = '{:}/raw-data-{:}'.format(args.save_dir, config) + seed2ckps, miss2ckps = obtain_valid_ckp(cur_save_dir, args.check_N) + torch.save(dict(seed2ckps=seed2ckps, miss2ckps=miss2ckps), '{:}/meta-{:}.pth'.format(args.save_dir, config)) + diff --git a/exps/NAS-Bench-201/xshapes.py b/exps/NAS-Bench-201/xshapes.py index a1de95b..9574aa5 100644 --- a/exps/NAS-Bench-201/xshapes.py +++ b/exps/NAS-Bench-201/xshapes.py @@ -165,7 +165,7 @@ def filter_indexes(xlist, mode, save_dir, seeds): if not temp_path.exists(): all_indexes.append(index) break - print('{:} [FILTER-INDEXES] : there are {:} architectures in total'.format(time_string(), len(all_indexes))) + print('{:} [FILTER-INDEXES] : there are {:}/{:} architectures in total'.format(time_string(), len(all_indexes), len(xlist))) SLURM_PROCID, SLURM_NTASKS = 'SLURM_PROCID', 'SLURM_NTASKS' if SLURM_PROCID in os.environ and SLURM_NTASKS in os.environ: # run on the slurm diff --git a/lib/models/cell_operations.py b/lib/models/cell_operations.py index 1a1953e..fc7af08 100644 --- a/lib/models/cell_operations.py +++ b/lib/models/cell_operations.py @@ -15,8 +15,8 @@ OPS = { 'nor_conv_1x1' : lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(C_in, C_out, (1,1), (stride,stride), (0,0), (1,1), affine, track_running_stats), 'dua_sepc_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: DualSepConv(C_in, C_out, (3,3), (stride,stride), (1,1), (1,1), affine, track_running_stats), 'dua_sepc_5x5' : lambda C_in, C_out, stride, affine, track_running_stats: DualSepConv(C_in, C_out, (5,5), (stride,stride), (2,2), (1,1), affine, track_running_stats), - 'dil_sepc_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: SepConv(C_in, C_out, (3,3), (stride,stride), (2,2), (2,2), affine, track_running_stats), - 'dil_sepc_5x5' : lambda C_in, C_out, stride, affine, track_running_stats: SepConv(C_in, C_out, (5,5), (stride,stride), (4,4), (2,2), affine, track_running_stats), + 'dil_sepc_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: SepConv(C_in, C_out, (3,3), (stride,stride), (2,2), (2,2), affine, track_running_stats), + 'dil_sepc_5x5' : lambda C_in, C_out, stride, affine, track_running_stats: SepConv(C_in, C_out, (5,5), (stride,stride), (4,4), (2,2), affine, track_running_stats), 'skip_connect' : lambda C_in, C_out, stride, affine, track_running_stats: Identity() if stride == 1 and C_in == C_out else FactorizedReduce(C_in, C_out, stride, affine, track_running_stats), } diff --git a/lib/models/shape_searchs/SearchCifarResNet.py b/lib/models/shape_searchs/SearchCifarResNet.py index e944058..828c052 100644 --- a/lib/models/shape_searchs/SearchCifarResNet.py +++ b/lib/models/shape_searchs/SearchCifarResNet.py @@ -172,7 +172,7 @@ class ResNetBasicblock(nn.Module): else: residual, expected_flop_c = inputs, 0 out = additive_func(residual, out_b) - return out, expected_inC_b, sum([expected_flop_a, expected_flop_b, expected_flop_c]) + return nn.functional.relu(out, inplace=True), expected_inC_b, sum([expected_flop_a, expected_flop_b, expected_flop_c]) def basic_forward(self, inputs): basicblock = self.conv_a(inputs) @@ -244,8 +244,7 @@ class ResNetBottleneck(nn.Module): else: residual, expected_flop_c = inputs, 0 out = additive_func(residual, out_1x4) - return out, expected_inC_1x4, sum([expected_flop_1x1, expected_flop_3x3, expected_flop_1x4, expected_flop_c]) - + return nn.functional.relu(out, inplace=True), expected_inC_1x4, sum([expected_flop_1x1, expected_flop_3x3, expected_flop_1x4, expected_flop_c]) class SearchShapeCifarResNet(nn.Module): diff --git a/lib/models/shape_searchs/SearchCifarResNet_width.py b/lib/models/shape_searchs/SearchCifarResNet_width.py index 2e37e98..5c46e1c 100644 --- a/lib/models/shape_searchs/SearchCifarResNet_width.py +++ b/lib/models/shape_searchs/SearchCifarResNet_width.py @@ -156,7 +156,7 @@ class ResNetBasicblock(nn.Module): else: residual, expected_flop_c = inputs, 0 out = additive_func(residual, out_b) - return out, expected_inC_b, sum([expected_flop_a, expected_flop_b, expected_flop_c]) + return nn.functional.relu(out, inplace=True), expected_inC_b, sum([expected_flop_a, expected_flop_b, expected_flop_c]) def basic_forward(self, inputs): basicblock = self.conv_a(inputs) @@ -228,8 +228,7 @@ class ResNetBottleneck(nn.Module): else: residual, expected_flop_c = inputs, 0 out = additive_func(residual, out_1x4) - return out, expected_inC_1x4, sum([expected_flop_1x1, expected_flop_3x3, expected_flop_1x4, expected_flop_c]) - + return nn.functional.relu(out, inplace=True), expected_inC_1x4, sum([expected_flop_1x1, expected_flop_3x3, expected_flop_1x4, expected_flop_c]) class SearchWidthCifarResNet(nn.Module): diff --git a/lib/models/shape_searchs/SearchImagenetResNet.py b/lib/models/shape_searchs/SearchImagenetResNet.py index 7b7654d..ee01d57 100644 --- a/lib/models/shape_searchs/SearchImagenetResNet.py +++ b/lib/models/shape_searchs/SearchImagenetResNet.py @@ -171,7 +171,7 @@ class ResNetBasicblock(nn.Module): else: residual, expected_flop_c = inputs, 0 out = additive_func(residual, out_b) - return out, expected_inC_b, sum([expected_flop_a, expected_flop_b, expected_flop_c]) + return nn.functional.relu(out, inplace=True), expected_inC_b, sum([expected_flop_a, expected_flop_b, expected_flop_c]) def basic_forward(self, inputs): basicblock = self.conv_a(inputs) @@ -243,8 +243,7 @@ class ResNetBottleneck(nn.Module): else: residual, expected_flop_c = inputs, 0 out = additive_func(residual, out_1x4) - return out, expected_inC_1x4, sum([expected_flop_1x1, expected_flop_3x3, expected_flop_1x4, expected_flop_c]) - + return nn.functional.relu(out, inplace=True), expected_inC_1x4, sum([expected_flop_1x1, expected_flop_3x3, expected_flop_1x4, expected_flop_c]) class SearchShapeImagenetResNet(nn.Module): diff --git a/lib/models/shape_searchs/SearchSimResNet_width.py b/lib/models/shape_searchs/SearchSimResNet_width.py index dbd9cad..18785f8 100644 --- a/lib/models/shape_searchs/SearchSimResNet_width.py +++ b/lib/models/shape_searchs/SearchSimResNet_width.py @@ -153,7 +153,7 @@ class SimBlock(nn.Module): else: residual, expected_flop_c = inputs, 0 out = additive_func(residual, out) - return out, expected_next_inC, sum([expected_flop, expected_flop_c]) + return nn.functional.relu(out, inplace=True), expected_next_inC, sum([expected_flop, expected_flop_c]) def basic_forward(self, inputs): basicblock = self.conv(inputs) diff --git a/scripts-search/X-X/train-shapes-01.sh b/scripts-search/X-X/train-shapes-01.sh new file mode 100644 index 0000000..a9b89f1 --- /dev/null +++ b/scripts-search/X-X/train-shapes-01.sh @@ -0,0 +1,36 @@ +#!/bin/bash +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.01 # +##################################################### +# bash ./scripts-search/X-X/train-shapes-01.sh 0 4 +echo script name: $0 +echo $# arguments +if [ "$#" -ne 2 ] ;then + echo "Input illegal number of parameters " $# + echo "Need 2 parameters for hyper-parameters-opt-file, and seeds" + exit 1 +fi +if [ "$TORCH_HOME" = "" ]; then + echo "Must set TORCH_HOME envoriment variable for data dir saving" + exit 1 +else + echo "TORCH_HOME : $TORCH_HOME" +fi + +srange=00000-32767 +opt=01 +all_seeds=777 +cpus=4 + +save_dir=./output/NAS-BENCH-202/ + +SLURM_PROCID=$1 SLURM_NTASKS=$2 OMP_NUM_THREADS=${cpus} python exps/NAS-Bench-201/xshapes.py \ + --mode new --srange ${srange} --hyper ${opt} --save_dir ${save_dir} \ + --datasets cifar10 cifar10 cifar100 ImageNet16-120 \ + --splits 1 0 0 0 \ + --xpaths $TORCH_HOME/cifar.python \ + $TORCH_HOME/cifar.python \ + $TORCH_HOME/cifar.python \ + $TORCH_HOME/cifar.python/ImageNet16 \ + --workers ${cpus} \ + --seeds ${all_seeds} diff --git a/scripts-search/algos/README.md b/scripts-search/algos/README.md new file mode 100644 index 0000000..2edc677 --- /dev/null +++ b/scripts-search/algos/README.md @@ -0,0 +1,3 @@ +# 10 NAS algorithms in NAS-Bench-201 + +Each script in this folder corresponds to one NAS algorithm, you can simple run it by one command.