diff --git a/README.md b/README.md index 6d30977..654b92b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ At the moment, this project provides the following algorithms and scripts to run DARTS DARTS: Differentiable Architecture Search - NAS-Bench-201.md + ICLR-2019-DARTS.md GDAS diff --git a/docs/ICLR-2019-DARTS.md b/docs/ICLR-2019-DARTS.md new file mode 100644 index 0000000..b3f84fb --- /dev/null +++ b/docs/ICLR-2019-DARTS.md @@ -0,0 +1,22 @@ +# DARTS: Differentiable Architecture Search + +DARTS: Differentiable Architecture Search is accepted by ICLR 2019. +In this paper, Hanxiao proposed a differentiable neural architecture search method, named as DARTS. +Recently, DARTS becomes very popular due to its simplicity and performance. + +**Run DARTS on the NAS-Bench-201 search space**: +``` +CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/DARTS-V2.sh cifar10 1 -1 +CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/GDAS.sh cifar10 1 -1 +``` + +# Citation + +``` +@inproceedings{liu2019darts, + title = {{DARTS}: Differentiable architecture search}, + author = {Liu, Hanxiao and Simonyan, Karen and Yang, Yiming}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year = {2019} +} +``` diff --git a/docs/NAS-Bench-201.md b/docs/NAS-Bench-201.md index ffac200..b9e4061 100644 --- a/docs/NAS-Bench-201.md +++ b/docs/NAS-Bench-201.md @@ -181,7 +181,7 @@ If researchers can provide better results with different hyper-parameters, we ar - [4] `CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/SETN.sh cifar10 1 -1` - [5] `CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/ENAS.sh cifar10 1 -1` - [6] `CUDA_VISIBLE_DEVICES=0 bash ./scripts-search/algos/RANDOM-NAS.sh cifar10 1 -1` -- [7] `bash ./scripts-search/algos/R-EA.sh -1` +- [7] `bash ./scripts-search/algos/R-EA.sh cifar10 3 -1` - [8] `bash ./scripts-search/algos/Random.sh -1` - [9] `bash ./scripts-search/algos/REINFORCE.sh 0.5 -1` - [10] `bash ./scripts-search/algos/BOHB.sh -1` diff --git a/exps/NAS-Bench-201/visualize.py b/exps/NAS-Bench-201/visualize.py index 802ebdd..274171b 100644 --- a/exps/NAS-Bench-201/visualize.py +++ b/exps/NAS-Bench-201/visualize.py @@ -517,7 +517,7 @@ def just_show(api): print ('[{:10s}-{:10s} ::: index={:5d}, accuracy={:.2f}'.format(dataset, metric_on_set, arch_index, highest_acc)) -def show_nas_sharing_w(api, dataset, subset, vis_save_dir, file_name, y_lims, x_maxs): +def show_nas_sharing_w(api, dataset, subset, vis_save_dir, sufix, file_name, y_lims, x_maxs): color_set = ['r', 'b', 'g', 'c', 'm', 'y', 'k'] dpi, width, height = 300, 3400, 2600 LabelSize, LegendFontsize = 28, 28 @@ -533,13 +533,14 @@ def show_nas_sharing_w(api, dataset, subset, vis_save_dir, file_name, y_lims, x_ plt.xlabel('The searching epoch', fontsize=LabelSize) plt.ylabel('The accuracy (%)', fontsize=LabelSize) - xpaths = {'RSPS' : 'output/search-cell-nas-bench-201/RANDOM-NAS-cifar10/checkpoint/', - 'DARTS-V1': 'output/search-cell-nas-bench-201/DARTS-V1-cifar10/checkpoint/', - 'DARTS-V2': 'output/search-cell-nas-bench-201/DARTS-V2-cifar10/checkpoint/', - 'GDAS' : 'output/search-cell-nas-bench-201/GDAS-cifar10/checkpoint/', - 'SETN' : 'output/search-cell-nas-bench-201/SETN-cifar10/checkpoint/', - 'ENAS' : 'output/search-cell-nas-bench-201/ENAS-cifar10/checkpoint/', + xpaths = {'RSPS' : 'output/search-cell-nas-bench-201/RANDOM-NAS-cifar10-{:}/checkpoint/'.format(sufix), + 'DARTS-V1': 'output/search-cell-nas-bench-201/DARTS-V1-cifar10-{:}/checkpoint/'.format(sufix), + 'DARTS-V2': 'output/search-cell-nas-bench-201/DARTS-V2-cifar10-{:}/checkpoint/'.format(sufix), + 'GDAS' : 'output/search-cell-nas-bench-201/GDAS-cifar10-{:}/checkpoint/'.format(sufix), + 'SETN' : 'output/search-cell-nas-bench-201/SETN-cifar10-{:}/checkpoint/'.format(sufix), + 'ENAS' : 'output/search-cell-nas-bench-201/ENAS-cifar10-{:}/checkpoint/'.format(sufix), } + """ xseeds = {'RSPS' : [5349, 59613, 5983], 'DARTS-V1': [11416, 72873, 81184, 28640], 'DARTS-V2': [43330, 79405, 79423], @@ -547,6 +548,15 @@ def show_nas_sharing_w(api, dataset, subset, vis_save_dir, file_name, y_lims, x_ 'SETN' : [20518, 61817, 89144], 'ENAS' : [3231, 34238, 96929], } + """ + xseeds = {'RSPS' : [23814, 28015, 95809], + 'DARTS-V1': [48349, 80877, 81920], + 'DARTS-V2': [61712, 7941 , 87041] , + 'GDAS' : [72818, 72996, 78877], + 'SETN' : [26985, 55206, 95404], + 'ENAS' : [21792, 36605, 45029] + } + def get_accs(xdata): epochs, xresults = xdata['epoch'], [] @@ -579,12 +589,13 @@ def show_nas_sharing_w(api, dataset, subset, vis_save_dir, file_name, y_lims, x_ plt.fill_between(epochs, [accyss[:,i].mean()-accyss[:,i].std() for i in epochs], [accyss[:,i].mean()+accyss[:,i].std() for i in epochs], alpha=0.2, color=color_set[idx]) #plt.legend(loc=4, fontsize=LegendFontsize) plt.legend(loc=0, fontsize=LegendFontsize) - save_path = vis_save_dir / '{:}-{:}-{:}-{:}'.format(xox, dataset, subset, file_name) + save_path = vis_save_dir / '{:}.pdf'.format(file_name) print('save figure into {:}\n'.format(save_path)) fig.savefig(str(save_path), dpi=dpi, bbox_inches='tight', format='pdf') -def show_nas_sharing_w_v2(api, data_sub_a, data_sub_b, vis_save_dir, file_name, y_lims, x_maxs): + +def show_nas_sharing_w_v2(api, data_sub_a, data_sub_b, vis_save_dir, sufix, file_name, y_lims, x_maxs): color_set = ['r', 'b', 'g', 'c', 'm', 'y', 'k'] dpi, width, height = 300, 3400, 2600 LabelSize, LegendFontsize = 28, 28 @@ -600,13 +611,14 @@ def show_nas_sharing_w_v2(api, data_sub_a, data_sub_b, vis_save_dir, file_name, plt.xlabel('The searching epoch', fontsize=LabelSize) plt.ylabel('The accuracy (%)', fontsize=LabelSize) - xpaths = {'RSPS' : 'output/search-cell-nas-bench-201/RANDOM-NAS-cifar10/checkpoint/', - 'DARTS-V1': 'output/search-cell-nas-bench-201/DARTS-V1-cifar10/checkpoint/', - 'DARTS-V2': 'output/search-cell-nas-bench-201/DARTS-V2-cifar10/checkpoint/', - 'GDAS' : 'output/search-cell-nas-bench-201/GDAS-cifar10/checkpoint/', - 'SETN' : 'output/search-cell-nas-bench-201/SETN-cifar10/checkpoint/', - 'ENAS' : 'output/search-cell-nas-bench-201/ENAS-cifar10/checkpoint/', + xpaths = {'RSPS' : 'output/search-cell-nas-bench-201/RANDOM-NAS-cifar10-{:}/checkpoint/'.format(sufix), + 'DARTS-V1': 'output/search-cell-nas-bench-201/DARTS-V1-cifar10-{:}/checkpoint/'.format(sufix), + 'DARTS-V2': 'output/search-cell-nas-bench-201/DARTS-V2-cifar10-{:}/checkpoint/'.format(sufix), + 'GDAS' : 'output/search-cell-nas-bench-201/GDAS-cifar10-{:}/checkpoint/'.format(sufix), + 'SETN' : 'output/search-cell-nas-bench-201/SETN-cifar10-{:}/checkpoint/'.format(sufix), + 'ENAS' : 'output/search-cell-nas-bench-201/ENAS-cifar10-{:}/checkpoint/'.format(sufix), } + """ xseeds = {'RSPS' : [5349, 59613, 5983], 'DARTS-V1': [11416, 72873, 81184, 28640], 'DARTS-V2': [43330, 79405, 79423], @@ -614,6 +626,15 @@ def show_nas_sharing_w_v2(api, data_sub_a, data_sub_b, vis_save_dir, file_name, 'SETN' : [20518, 61817, 89144], 'ENAS' : [3231, 34238, 96929], } + """ + xseeds = {'RSPS' : [23814, 28015, 95809], + 'DARTS-V1': [48349, 80877, 81920], + 'DARTS-V2': [61712, 7941 , 87041] , + 'GDAS' : [72818, 72996, 78877], + 'SETN' : [26985, 55206, 95404], + 'ENAS' : [21792, 36605, 45029] + } + def get_accs(xdata, dataset, subset): epochs, xresults = xdata['epoch'], [] @@ -643,8 +664,15 @@ def show_nas_sharing_w_v2(api, data_sub_a, data_sub_b, vis_save_dir, file_name, accyss_B = np.array( [get_accs(xdatas, data_sub_b[0], data_sub_b[1]) for xdatas in all_datas] ) epochs = list(range(accyss_A.shape[1])) for j, accyss in enumerate([accyss_A, accyss_B]): - plt.plot(epochs, [accyss[:,i].mean() for i in epochs], color=color_set[idx*2+j], linestyle='-' if j==0 else '--', label='{:} ({:})'.format(method, 'VALID' if j == 0 else 'TEST'), lw=2, alpha=0.9) - plt.fill_between(epochs, [accyss[:,i].mean()-accyss[:,i].std() for i in epochs], [accyss[:,i].mean()+accyss[:,i].std() for i in epochs], alpha=0.2, color=color_set[idx*2+j]) + if x_maxs == 50: + color, line = color_set[idx*2+j], '-' if j==0 else '--' + elif x_maxs == 250: + color, line = color_set[idx], '-' if j==0 else '--' + else: raise ValueError('invalid x-maxs={:}'.format(x_maxs)) + plt.plot(epochs, [accyss[:,i].mean() for i in epochs], color=color, linestyle=line, label='{:} ({:})'.format(method, 'VALID' if j == 0 else 'TEST'), lw=2, alpha=0.9) + plt.fill_between(epochs, [accyss[:,i].mean()-accyss[:,i].std() for i in epochs], [accyss[:,i].mean()+accyss[:,i].std() for i in epochs], alpha=0.2, color=color) + setname = data_sub_a if j == 0 else data_sub_b + print('{:} -- {:} ---- {:.2f}$\\pm${:.2f}'.format(method, setname, accyss[:,-1].mean(), accyss[:,-1].std())) #plt.legend(loc=4, fontsize=LegendFontsize) plt.legend(loc=0, fontsize=LegendFontsize) save_path = vis_save_dir / '{:}-{:}'.format(xox, file_name) @@ -654,7 +682,7 @@ def show_nas_sharing_w_v2(api, data_sub_a, data_sub_b, vis_save_dir, file_name, def show_reinforce(api, root, dataset, xset, file_name, y_lims): print ('root-path={:}, dataset={:}, xset={:}'.format(root, dataset, xset)) - LRs = ['0.01', '0.02', '0.1', '0.2', '0.5', '1.0', '1.5', '2.0', '2.5', '3.0'] + LRs = ['0.01', '0.02', '0.1', '0.2', '0.5'] checkpoints = ['./output/search-cell-nas-bench-201/REINFORCE-cifar10-{:}/results.pth'.format(x) for x in LRs] acc_lr_dict, indexes = {}, None for lr, checkpoint in zip(LRs, checkpoints): @@ -684,7 +712,8 @@ def show_reinforce(api, root, dataset, xset, file_name, y_lims): for idx, LR in enumerate(LRs): legend = 'LR={:.2f}'.format(float(LR)) - color, linestyle = color_set[idx // 2], '-' if idx % 2 == 0 else '-.' + #color, linestyle = color_set[idx // 2], '-' if idx % 2 == 0 else '-.' + color, linestyle = color_set[idx], '-' plt.plot(indexes, acc_lr_dict[LR], color=color, linestyle=linestyle, label=legend, lw=2, alpha=0.8) print ('{:} : mean = {:}, std = {:} :: {:.2f}$\\pm${:.2f}'.format(legend, np.mean(acc_lr_dict[LR]), np.std(acc_lr_dict[LR]), np.mean(acc_lr_dict[LR]), np.std(acc_lr_dict[LR]))) plt.legend(loc=4, fontsize=LegendFontsize) @@ -693,6 +722,49 @@ def show_reinforce(api, root, dataset, xset, file_name, y_lims): fig.savefig(str(save_path), dpi=dpi, bbox_inches='tight', format='pdf') + +def show_rea(api, root, dataset, xset, file_name, y_lims): + print ('root-path={:}, dataset={:}, xset={:}'.format(root, dataset, xset)) + SSs = [3, 5, 10] + checkpoints = ['./output/search-cell-nas-bench-201/R-EA-cifar10-SS{:}/results.pth'.format(x) for x in SSs] + acc_ss_dict, indexes = {}, None + for ss, checkpoint in zip(SSs, checkpoints): + all_indexes, accuracies = torch.load(checkpoint, map_location='cpu'), [] + for x in all_indexes: + info = api.arch2infos_full[ x ] + metrics = info.get_metrics(dataset, xset, None, False) + accuracies.append( metrics['accuracy'] ) + if indexes is None: indexes = list(range(len(accuracies))) + acc_ss_dict[ss] = np.array( sorted(accuracies) ) + print ('Sample-Size={:2d}, mean={:}, std={:}'.format(ss, acc_ss_dict[ss].mean(), acc_ss_dict[ss].std())) + + color_set = ['r', 'b', 'g', 'c', 'm', 'y', 'k'] + dpi, width, height = 300, 3400, 2600 + LabelSize, LegendFontsize = 28, 22 + figsize = width / float(dpi), height / float(dpi) + fig = plt.figure(figsize=figsize) + x_axis = np.arange(0, 600) + plt.xlim(0, max(indexes)) + plt.ylim(y_lims[0], y_lims[1]) + interval_x, interval_y = 100, y_lims[2] + plt.xticks(np.arange(0, max(indexes), interval_x), fontsize=LegendFontsize) + plt.yticks(np.arange(y_lims[0],y_lims[1], interval_y), fontsize=LegendFontsize) + plt.grid() + plt.xlabel('The index of runs', fontsize=LabelSize) + plt.ylabel('The accuracy (%)', fontsize=LabelSize) + + for idx, ss in enumerate(SSs): + legend = 'sample-size={:2d}'.format(ss) + #color, linestyle = color_set[idx // 2], '-' if idx % 2 == 0 else '-.' + color, linestyle = color_set[idx], '-' + plt.plot(indexes, acc_ss_dict[ss], color=color, linestyle=linestyle, label=legend, lw=2, alpha=0.8) + print ('{:} : mean = {:}, std = {:} :: {:.2f}$\\pm${:.2f}'.format(legend, np.mean(acc_ss_dict[ss]), np.std(acc_ss_dict[ss]), np.mean(acc_ss_dict[ss]), np.std(acc_ss_dict[ss]))) + plt.legend(loc=4, fontsize=LegendFontsize) + save_path = root / '{:}-{:}-{:}.pdf'.format(dataset, xset, file_name) + print('save figure into {:}\n'.format(save_path)) + fig.savefig(str(save_path), dpi=dpi, bbox_inches='tight', format='pdf') + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='NAS-Bench-201', formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -712,9 +784,25 @@ if __name__ == '__main__': #visualize_relative_ranking(vis_save_dir) api = API(args.api_path) - show_reinforce(api, vis_save_dir, 'cifar10-valid' , 'x-valid', 'REINFORCE-CIFAR-10', (75, 95, 5)) - import pdb; pdb.set_trace() + #show_reinforce(api, vis_save_dir, 'cifar10-valid' , 'x-valid', 'REINFORCE-CIFAR-10', (85, 92, 2)) + #show_rea (api, vis_save_dir, 'cifar10-valid' , 'x-valid', 'REA-CIFAR-10', (88, 92, 1)) + #plot_results_nas_v2(api, ('cifar10-valid' , 'x-valid'), ('cifar10' , 'ori-test'), vis_save_dir, 'nas-com-v2-cifar010.pdf', (85,95, 1)) + #plot_results_nas_v2(api, ('cifar100' , 'x-valid'), ('cifar100' , 'x-test' ), vis_save_dir, 'nas-com-v2-cifar100.pdf', (60,75, 3)) + #plot_results_nas_v2(api, ('ImageNet16-120', 'x-valid'), ('ImageNet16-120', 'x-test' ), vis_save_dir, 'nas-com-v2-imagenet.pdf', (35,48, 2)) + + show_nas_sharing_w_v2(api, ('cifar10-valid' , 'x-valid'), ('cifar10' , 'ori-test') , vis_save_dir, 'BN0', 'BN0-DARTS-CIFAR010.pdf', (0, 100,10), 50) + show_nas_sharing_w_v2(api, ('cifar100' , 'x-valid'), ('cifar100' , 'x-test' ) , vis_save_dir, 'BN0', 'BN0-DARTS-CIFAR100.pdf', (0, 100,10), 50) + show_nas_sharing_w_v2(api, ('ImageNet16-120', 'x-valid'), ('ImageNet16-120', 'x-test' ) , vis_save_dir, 'BN0', 'BN0-DARTS-ImageNet.pdf', (0, 100,10), 50) + + show_nas_sharing_w_v2(api, ('cifar10-valid' , 'x-valid'), ('cifar10' , 'ori-test') , vis_save_dir, 'BN0', 'BN0-OTHER-CIFAR010.pdf', (0, 100,10), 250) + show_nas_sharing_w_v2(api, ('cifar100' , 'x-valid'), ('cifar100' , 'x-test' ) , vis_save_dir, 'BN0', 'BN0-OTHER-CIFAR100.pdf', (0, 100,10), 250) + show_nas_sharing_w_v2(api, ('ImageNet16-120', 'x-valid'), ('ImageNet16-120', 'x-test' ) , vis_save_dir, 'BN0', 'BN0-OTHER-ImageNet.pdf', (0, 100,10), 250) + + show_nas_sharing_w(api, 'cifar10-valid' , 'x-valid' , vis_save_dir, 'BN0', 'BN0-XX-CIFAR010-VALID.pdf', (0, 100,10), 250) + show_nas_sharing_w(api, 'cifar10' , 'ori-test', vis_save_dir, 'BN0', 'BN0-XX-CIFAR010-TEST.pdf' , (0, 100,10), 250) + import pdb; pdb.set_trace() + """ for x_maxs in [50, 250]: show_nas_sharing_w(api, 'cifar10-valid' , 'x-valid' , vis_save_dir, 'nas-plot.pdf', (0, 100,10), x_maxs) show_nas_sharing_w(api, 'cifar10' , 'ori-test', vis_save_dir, 'nas-plot.pdf', (0, 100,10), x_maxs) @@ -724,17 +812,11 @@ if __name__ == '__main__': show_nas_sharing_w(api, 'ImageNet16-120', 'x-test' , vis_save_dir, 'nas-plot.pdf', (0, 100,10), x_maxs) show_nas_sharing_w_v2(api, ('cifar10-valid' , 'x-valid'), ('cifar10' , 'ori-test') , vis_save_dir, 'DARTS-CIFAR010.pdf', (0, 100,10), 50) - show_nas_sharing_w_v2(api, ('cifar100' , 'x-valid'), ('cifar100' , 'x-test' ) , vis_save_dir, 'DARTS-CIFAR100.pdf', (0, 100,10), 50) - show_nas_sharing_w_v2(api, ('ImageNet16-120', 'x-valid'), ('ImageNet16-120', 'x-test' ) , vis_save_dir, 'DARTS-ImageNet.pdf', (0, 100,10), 50) - #just_show(api) - """ + just_show(api) plot_results_nas(api, 'cifar10-valid' , 'x-valid' , vis_save_dir, 'nas-com.pdf', (85,95, 1)) plot_results_nas(api, 'cifar10' , 'ori-test', vis_save_dir, 'nas-com.pdf', (85,95, 1)) plot_results_nas(api, 'cifar100' , 'x-valid' , vis_save_dir, 'nas-com.pdf', (55,75, 3)) plot_results_nas(api, 'cifar100' , 'x-test' , vis_save_dir, 'nas-com.pdf', (55,75, 3)) plot_results_nas(api, 'ImageNet16-120', 'x-valid' , vis_save_dir, 'nas-com.pdf', (35,50, 3)) plot_results_nas(api, 'ImageNet16-120', 'x-test' , vis_save_dir, 'nas-com.pdf', (35,50, 3)) - plot_results_nas_v2(api, ('cifar10-valid' , 'x-valid'), ('cifar10' , 'ori-test'), vis_save_dir, 'nas-com-v2-cifar010.pdf', (85,95, 1)) - plot_results_nas_v2(api, ('cifar100' , 'x-valid'), ('cifar100' , 'x-test' ), vis_save_dir, 'nas-com-v2-cifar100.pdf', (60,75, 3)) - plot_results_nas_v2(api, ('ImageNet16-120', 'x-valid'), ('ImageNet16-120', 'x-test' ), vis_save_dir, 'nas-com-v2-imagenet.pdf', (35,48, 2)) """ diff --git a/exps/algos/R_EA.py b/exps/algos/R_EA.py index e421e90..f7834ad 100644 --- a/exps/algos/R_EA.py +++ b/exps/algos/R_EA.py @@ -33,13 +33,38 @@ class Model(object): # This function is to mimic the training and evaluatinig procedure for a single architecture `arch`. # The time_cost is calculated as the total training time for a few (e.g., 12 epochs) plus the evaluation time for one epoch. -def train_and_eval(arch, nas_bench, extra_info): - if nas_bench is not None: +# For use_converged_LR = True, the architecture is trained for 12 epochs, with LR being decaded from 0.1 to 0. +# In this case, the LR schedular is converged. +# For use_converged_LR = False, the architecture is planed to be trained for 200 epochs, but we early stop its procedure. +# +def train_and_eval(arch, nas_bench, extra_info, dataname='cifar10-valid', use_converged_LR=True): + if use_converged_LR and nas_bench is not None: arch_index = nas_bench.query_index_by_arch( arch ) assert arch_index >= 0, 'can not find this arch : {:}'.format(arch) - info = nas_bench.get_more_info(arch_index, 'cifar10-valid', None, True) + info = nas_bench.get_more_info(arch_index, dataname, None, True) valid_acc, time_cost = info['valid-accuracy'], info['train-all-time'] + info['valid-per-time'] #_, valid_acc = info.get_metrics('cifar10-valid', 'x-valid' , 25, True) # use the validation accuracy after 25 training epochs + elif not use_converged_LR and nas_bench is not None: + # Please use `use_converged_LR=False` for cifar10 only. + # It did return values for cifar100 and ImageNet16-120, but it has some potential issues. (Please email me for more details) + arch_index, nepoch = nas_bench.query_index_by_arch( arch ), 25 + assert arch_index >= 0, 'can not find this arch : {:}'.format(arch) + xoinfo = nas_bench.get_more_info(arch_index, 'cifar10-valid', None, True) + xocost = nas_bench.get_cost_info(arch_index, 'cifar10-valid', False) + info = nas_bench.get_more_info(arch_index, dataname, nepoch, False, True) # use the validation accuracy after 25 training epochs, which is used in our ICLR submission (not the camera ready). + cost = nas_bench.get_cost_info(arch_index, dataname, False) + # The following codes are used to estimate the time cost. + # When we build NAS-Bench-201, architectures are trained on different machines and we can not use that time record. + # When we create checkpoints for converged_LR, we run all experiments on 1080Ti, and thus the time for each architecture can be fairly compared. + nums = {'ImageNet16-120-train': 151700, 'ImageNet16-120-valid': 3000, + 'cifar10-valid-train' : 25000, 'cifar10-valid-valid' : 25000, + 'cifar100-train' : 50000, 'cifar100-valid' : 5000} + estimated_train_cost = xoinfo['train-per-time'] / nums['cifar10-valid-train'] * nums['{:}-train'.format(dataname)] / xocost['latency'] * cost['latency'] * nepoch + estimated_valid_cost = xoinfo['valid-per-time'] / nums['cifar10-valid-valid'] * nums['{:}-valid'.format(dataname)] / xocost['latency'] * cost['latency'] + try: + valid_acc, time_cost = info['valid-accuracy'], estimated_train_cost + estimated_valid_cost + except: + valid_acc, time_cost = info['est-valid-accuracy'], estimated_train_cost + estimated_valid_cost else: # train a model from scratch. raise ValueError('NOT IMPLEMENT YET') @@ -79,7 +104,7 @@ def mutate_arch_func(op_names): return mutate_arch_func -def regularized_evolution(cycles, population_size, sample_size, time_budget, random_arch, mutate_arch, nas_bench, extra_info): +def regularized_evolution(cycles, population_size, sample_size, time_budget, random_arch, mutate_arch, nas_bench, extra_info, dataname): """Algorithm for regularized evolution (i.e. aging evolution). Follows "Algorithm 1" in Real et al. "Regularized Evolution for Image @@ -150,6 +175,10 @@ def main(xargs, nas_bench): logger = prepare_logger(args) assert xargs.dataset == 'cifar10', 'currently only support CIFAR-10' + if xargs.dataset == 'cifar10': + dataname = 'cifar10-valid' + else: + dataname = xargs.dataset if xargs.data_path is not None: train_data, valid_data, xshape, class_num = get_datasets(xargs.dataset, xargs.data_path, -1) split_Fpath = 'configs/nas-benchmark/cifar-split.txt' @@ -182,7 +211,7 @@ def main(xargs, nas_bench): x_start_time = time.time() logger.log('{:} use nas_bench : {:}'.format(time_string(), nas_bench)) logger.log('-'*30 + ' start searching with the time budget of {:} s'.format(xargs.time_budget)) - history, total_cost = regularized_evolution(xargs.ea_cycles, xargs.ea_population, xargs.ea_sample_size, xargs.time_budget, random_arch, mutate_arch, nas_bench if args.ea_fast_by_api else None, extra_info) + history, total_cost = regularized_evolution(xargs.ea_cycles, xargs.ea_population, xargs.ea_sample_size, xargs.time_budget, random_arch, mutate_arch, nas_bench if args.ea_fast_by_api else None, extra_info, dataname) logger.log('{:} regularized_evolution finish with history of {:} arch with {:.1f} s (real-cost={:.2f} s).'.format(time_string(), len(history), total_cost, time.time()-x_start_time)) best_arch = max(history, key=lambda i: i.accuracy) best_arch = best_arch.arch diff --git a/lib/nas_201_api/api.py b/lib/nas_201_api/api.py index 556bc74..15566c4 100644 --- a/lib/nas_201_api/api.py +++ b/lib/nas_201_api/api.py @@ -162,6 +162,13 @@ class NASBench201API(object): archresult = arch2infos[index] return archresult.get_net_param(dataset, seed) + # obtain the cost metric for the `index`-th architecture on a dataset + def get_cost_info(self, index, dataset, use_12epochs_result=False): + if use_12epochs_result: basestr, arch2infos = '12epochs' , self.arch2infos_less + else : basestr, arch2infos = '200epochs', self.arch2infos_full + archresult = arch2infos[index] + return archresult.get_comput_costs(dataset) + # obtain the metric for the `index`-th architecture def get_more_info(self, index, dataset, iepoch=None, use_12epochs_result=False, is_random=True): if use_12epochs_result: basestr, arch2infos = '12epochs' , self.arch2infos_less @@ -177,6 +184,7 @@ class NASBench201API(object): total = train_info['iepoch'] + 1 xifo = {'train-loss' : train_info['loss'], 'train-accuracy': train_info['accuracy'], + 'train-per-time': None if train_info['all_time'] is None else train_info['all_time'] / total, 'train-all-time': train_info['all_time'], 'valid-loss' : valid_info['loss'], 'valid-accuracy': valid_info['accuracy'], @@ -188,21 +196,32 @@ class NASBench201API(object): return xifo else: train_info = archresult.get_metrics(dataset, 'train' , iepoch=iepoch, is_random=is_random) - if dataset == 'cifar10': - test__info = archresult.get_metrics(dataset, 'ori-test', iepoch=iepoch, is_random=is_random) - else: - test__info = archresult.get_metrics(dataset, 'x-test', iepoch=iepoch, is_random=is_random) + try: + if dataset == 'cifar10': + test__info = archresult.get_metrics(dataset, 'ori-test', iepoch=iepoch, is_random=is_random) + else: + test__info = archresult.get_metrics(dataset, 'x-test', iepoch=iepoch, is_random=is_random) + except: + valid_info = None try: valid_info = archresult.get_metrics(dataset, 'x-valid', iepoch=iepoch, is_random=is_random) except: valid_info = None + try: + est_valid_info = archresult.get_metrics(dataset, 'ori-test', iepoch=iepoch, is_random=is_random) + except: + est_valid_info = None xifo = {'train-loss' : train_info['loss'], - 'train-accuracy': train_info['accuracy'], - 'test-loss' : test__info['loss'], - 'test-accuracy' : test__info['accuracy']} + 'train-accuracy': train_info['accuracy']} + if valid_info is not None: + xifo['test-loss'] = test__info['loss'], + xifo['test-accuracy'] = test__info['accuracy'] if valid_info is not None: xifo['valid-loss'] = valid_info['loss'] xifo['valid-accuracy'] = valid_info['accuracy'] + if est_valid_info is not None: + xifo['est-valid-loss'] = est_valid_info['loss'] + xifo['est-valid-accuracy'] = est_valid_info['accuracy'] return xifo def show(self, index=-1): diff --git a/scripts-search/algos/GRID-RL.sh b/scripts-search/algos/GRID-RL.sh index 50384f6..8b99d89 100644 --- a/scripts-search/algos/GRID-RL.sh +++ b/scripts-search/algos/GRID-RL.sh @@ -1,7 +1,8 @@ #!/bin/bash echo script name: $0 -lrs="0.01 0.02 0.1 0.2 0.5 1.0 1.5 2.0 2.5 3.0" +#lrs="0.01 0.02 0.1 0.2 0.5 1.0 1.5 2.0 2.5 3.0" +lrs="0.01 0.02 0.1 0.2 0.5" for lr in ${lrs} do diff --git a/scripts-search/algos/R-EA.sh b/scripts-search/algos/R-EA.sh index f208764..6650d78 100644 --- a/scripts-search/algos/R-EA.sh +++ b/scripts-search/algos/R-EA.sh @@ -1,11 +1,11 @@ #!/bin/bash # Regularized Evolution for Image Classifier Architecture Search, AAAI 2019 -# bash ./scripts-search/algos/R-EA.sh -1 +# bash ./scripts-search/algos/R-EA.sh cifar10 3 -1 echo script name: $0 echo $# arguments -if [ "$#" -ne 1 ] ;then +if [ "$#" -ne 3 ] ;then echo "Input illegal number of parameters " $# - echo "Need 1 parameters for seed" + echo "Need 3 parameters for the-dataset-name, the-ea-sample-size and the-seed" exit 1 fi if [ "$TORCH_HOME" = "" ]; then @@ -15,14 +15,16 @@ else echo "TORCH_HOME : $TORCH_HOME" fi -dataset=cifar10 -seed=$1 +#dataset=cifar10 +dataset=$1 +sample_size=$2 +seed=$3 channel=16 num_cells=5 max_nodes=4 space=nas-bench-201 -save_dir=./output/search-cell-${space}/R-EA-${dataset} +save_dir=./output/search-cell-${space}/R-EA-${dataset}-SS${sample_size} OMP_NUM_THREADS=4 python ./exps/algos/R_EA.py \ --save_dir ${save_dir} --max_nodes ${max_nodes} --channel ${channel} --num_cells ${num_cells} \ @@ -30,5 +32,5 @@ OMP_NUM_THREADS=4 python ./exps/algos/R_EA.py \ --search_space_name ${space} \ --arch_nas_dataset ${TORCH_HOME}/NAS-Bench-201-v1_0-e61699.pth \ --time_budget 12000 \ - --ea_cycles 100 --ea_population 10 --ea_sample_size 3 --ea_fast_by_api 1 \ + --ea_cycles 200 --ea_population 10 --ea_sample_size ${sample_size} --ea_fast_by_api 1 \ --workers 4 --print_freq 200 --rand_seed ${seed}