From 06d5c547c922997bb96b0ed00d222a2e7bc39be2 Mon Sep 17 00:00:00 2001 From: D-X-Y <280835372@qq.com> Date: Mon, 25 Jan 2021 21:48:14 +0800 Subject: [PATCH] Updates --- README.md | 12 +++++++----- README_CN.md | 12 +++++++----- docs/NATS-Bench.md | 12 +++++++----- exps/NATS-Bench/Analyze-time.py | 16 ++++++++-------- exps/NATS-Bench/draw-correlations.py | 4 ++-- exps/NATS-Bench/draw-fig2_5.py | 2 +- exps/NATS-Bench/draw-fig6.py | 4 ++-- exps/NATS-Bench/draw-fig7.py | 2 +- exps/NATS-Bench/draw-fig8.py | 4 ++-- exps/NATS-Bench/draw-ranks.py | 2 +- exps/NATS-Bench/draw-table.py | 4 ++-- exps/NATS-Bench/main-sss.py | 2 +- exps/NATS-Bench/main-tss.py | 2 +- exps/NATS-Bench/sss-collect.py | 2 +- exps/NATS-Bench/sss-file-manager.py | 2 +- exps/NATS-Bench/test-nats-api.py | 2 +- exps/NATS-Bench/tss-collect-patcher.py | 2 +- exps/NATS-Bench/tss-collect.py | 2 +- exps/NATS-Bench/tss-file-manager.py | 2 +- 19 files changed, 48 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 5eff53a..ff596fa 100644 --- a/README.md +++ b/README.md @@ -105,11 +105,13 @@ If you find that this project helps your research, please consider citing the re journal={arXiv preprint arXiv:2006.03656}, year={2020} } -@article{dong2020nats, - title={{NATS-Bench}: Benchmarking NAS Algorithms for Architecture Topology and Size}, - author={Dong, Xuanyi and Liu, Lu and Musial, Katarzyna and Gabrys, Bogdan}, - journal={arXiv preprint arXiv:2009.00437}, - year={2020} +@article{dong2021nats, + title = {{NATS-Bench}: Benchmarking NAS Algorithms for Architecture Topology and Size}, + author = {Dong, Xuanyi and Liu, Lu and Musial, Katarzyna and Gabrys, Bogdan}, + doi = {10.1109/TPAMI.2021.3054824}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, + year = {2021}, + note = {\mbox{doi}:\url{10.1109/TPAMI.2021.3054824}} } @inproceedings{dong2020nasbench201, title = {NAS-Bench-201: Extending the Scope of Reproducible Neural Architecture Search}, diff --git a/README_CN.md b/README_CN.md index b919d2c..3b6c10e 100644 --- a/README_CN.md +++ b/README_CN.md @@ -98,11 +98,13 @@ Some methods use knowledge distillation (KD), which require pre-trained models. 如果您发现该项目对您的科研或工程有帮助，请考虑引用下列的某些文献： ``` -@article{dong2020nats, - title={{NATS-Bench}: Benchmarking NAS Algorithms for Architecture Topology and Size}, - author={Dong, Xuanyi and Liu, Lu and Musial, Katarzyna and Gabrys, Bogdan}, - journal={arXiv preprint arXiv:2009.00437}, - year={2020} +@article{dong2021nats, + title = {{NATS-Bench}: Benchmarking NAS Algorithms for Architecture Topology and Size}, + author = {Dong, Xuanyi and Liu, Lu and Musial, Katarzyna and Gabrys, Bogdan}, + doi = {10.1109/TPAMI.2021.3054824}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, + year = {2021}, + note = {\mbox{doi}:\url{10.1109/TPAMI.2021.3054824}} } @inproceedings{dong2020nasbench201, title = {NAS-Bench-201: Extending the Scope of Reproducible Neural Architecture Search}, diff --git a/docs/NATS-Bench.md b/docs/NATS-Bench.md index e397cf9..65cc5b9 100644 --- a/docs/NATS-Bench.md +++ b/docs/NATS-Bench.md @@ -252,10 +252,12 @@ GDAS: If you find that NATS-Bench helps your research, please consider citing it: ``` -@article{dong2020nats, - title={{NATS-Bench}: Benchmarking NAS Algorithms for Architecture Topology and Size}, - author={Dong, Xuanyi and Liu, Lu and Musial, Katarzyna and Gabrys, Bogdan}, - journal={arXiv preprint arXiv:2009.00437}, - year={2020} +@article{dong2021nats, + title = {{NATS-Bench}: Benchmarking NAS Algorithms for Architecture Topology and Size}, + author = {Dong, Xuanyi and Liu, Lu and Musial, Katarzyna and Gabrys, Bogdan}, + doi = {10.1109/TPAMI.2021.3054824}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, + year = {2021}, + note = {\mbox{doi}:\url{10.1109/TPAMI.2021.3054824}} } ``` diff --git a/exps/NATS-Bench/Analyze-time.py b/exps/NATS-Bench/Analyze-time.py index e1517e3..89f351c 100644 --- a/exps/NATS-Bench/Analyze-time.py +++ b/exps/NATS-Bench/Analyze-time.py @@ -1,5 +1,5 @@ ############################################################################## -# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +# NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size # ############################################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.07 # ############################################################################## @@ -19,15 +19,15 @@ from datasets import get_datasets from nats_bench import create -def show_time(api): - print('Show the time for {:} with 12-epoch-training'.format(api)) +def show_time(api, epoch=12): + print('Show the time for {:} with {:}-epoch-training'.format(api, epoch)) all_cifar10_time, all_cifar100_time, all_imagenet_time = 0, 0, 0 for index in tqdm.tqdm(range(len(api))): - info = api.get_more_info(index, 'ImageNet16-120', hp='12') + info = api.get_more_info(index, 'ImageNet16-120', hp=epoch) imagenet_time = info['train-all-time'] - info = api.get_more_info(index, 'cifar10-valid', hp='12') + info = api.get_more_info(index, 'cifar10-valid', hp=epoch) cifar10_time = info['train-all-time'] - info = api.get_more_info(index, 'cifar100', hp='12') + info = api.get_more_info(index, 'cifar100', hp=epoch) cifar100_time = info['train-all-time'] # accumulate the time all_cifar10_time += cifar10_time @@ -41,8 +41,8 @@ def show_time(api): if __name__ == '__main__': api_nats_tss = create(None, 'tss', fast_mode=True, verbose=False) - show_time(api_nats_tss) + show_time(api_nats_tss, 12) api_nats_sss = create(None, 'sss', fast_mode=True, verbose=False) - show_time(api_nats_sss) + show_time(api_nats_sss, 12) diff --git a/exps/NATS-Bench/draw-correlations.py b/exps/NATS-Bench/draw-correlations.py index 7ceb8cb..6afac3b 100644 --- a/exps/NATS-Bench/draw-correlations.py +++ b/exps/NATS-Bench/draw-correlations.py @@ -1,5 +1,5 @@ ############################################################### -# NATS-Bench (https://arxiv.org/pdf/2009.00437.pdf) # +# NATS-Bench (arxiv.org/pdf/2009.00437.pdf), IEEE TPAMI 2021 # ############################################################### # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.06 # ############################################################### @@ -54,7 +54,7 @@ def compute_spearmanr(vectori, vectorj): if __name__ == '__main__': - parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--save_dir', type=str, default='output/vis-nas-bench/nas-algos', help='Folder to save checkpoints and log.') parser.add_argument('--search_space', type=str, choices=['tss', 'sss'], help='Choose the search space.') args = parser.parse_args() diff --git a/exps/NATS-Bench/draw-fig2_5.py b/exps/NATS-Bench/draw-fig2_5.py index f5b44a8..f7cf0b2 100644 --- a/exps/NATS-Bench/draw-fig2_5.py +++ b/exps/NATS-Bench/draw-fig2_5.py @@ -1,5 +1,5 @@ ############################################################### -# NATS-Bench (https://arxiv.org/pdf/2009.00437.pdf) # +# NATS-Bench (arxiv.org/pdf/2009.00437.pdf), IEEE TPAMI 2021 # # The code to draw Figure 2 / 3 / 4 / 5 in our paper. # ############################################################### # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.06 # diff --git a/exps/NATS-Bench/draw-fig6.py b/exps/NATS-Bench/draw-fig6.py index 703c978..fc7a152 100644 --- a/exps/NATS-Bench/draw-fig6.py +++ b/exps/NATS-Bench/draw-fig6.py @@ -1,5 +1,5 @@ ############################################################### -# NATS-Bench (https://arxiv.org/pdf/2009.00437.pdf) # +# NATS-Bench (arxiv.org/pdf/2009.00437.pdf), IEEE TPAMI 2021 # # The code to draw Figure 6 in our paper. # ############################################################### # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.06 # @@ -158,7 +158,7 @@ def visualize_curve(api, vis_save_dir, search_space): if __name__ == '__main__': - parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--save_dir', type=str, default='output/vis-nas-bench/nas-algos', help='Folder to save checkpoints and log.') parser.add_argument('--search_space', type=str, choices=['tss', 'sss'], help='Choose the search space.') args = parser.parse_args() diff --git a/exps/NATS-Bench/draw-fig7.py b/exps/NATS-Bench/draw-fig7.py index 262c9d8..26a66aa 100644 --- a/exps/NATS-Bench/draw-fig7.py +++ b/exps/NATS-Bench/draw-fig7.py @@ -1,5 +1,5 @@ ############################################################### -# NATS-Bench (https://arxiv.org/pdf/2009.00437.pdf) # +# NATS-Bench (arxiv.org/pdf/2009.00437.pdf), IEEE TPAMI 2021 # # The code to draw Figure 7 in our paper. # ############################################################### # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.06 # diff --git a/exps/NATS-Bench/draw-fig8.py b/exps/NATS-Bench/draw-fig8.py index d287c3c..e650ed7 100644 --- a/exps/NATS-Bench/draw-fig8.py +++ b/exps/NATS-Bench/draw-fig8.py @@ -1,5 +1,5 @@ ############################################################### -# NATS-Bench (https://arxiv.org/pdf/2009.00437.pdf) # +# NATS-Bench (arxiv.org/pdf/2009.00437.pdf), IEEE TPAMI 2021 # # The code to draw Figure 6 in our paper. # ############################################################### # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.06 # @@ -175,7 +175,7 @@ def visualize_curve(api_dict, vis_save_dir): if __name__ == '__main__': - parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--save_dir', type=str, default='output/vis-nas-bench/nas-algos-vs-h', help='Folder to save checkpoints and log.') args = parser.parse_args() diff --git a/exps/NATS-Bench/draw-ranks.py b/exps/NATS-Bench/draw-ranks.py index 031ef83..ad69f4b 100644 --- a/exps/NATS-Bench/draw-ranks.py +++ b/exps/NATS-Bench/draw-ranks.py @@ -1,5 +1,5 @@ ############################################################### -# NATS-Bench (https://arxiv.org/pdf/2009.00437.pdf) # +# NATS-Bench (arxiv.org/pdf/2009.00437.pdf), IEEE TPAMI 2021 # # The code to draw Figure 2 / 3 / 4 / 5 in our paper. # ############################################################### # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.06 # diff --git a/exps/NATS-Bench/draw-table.py b/exps/NATS-Bench/draw-table.py index b379dc6..3d53526 100644 --- a/exps/NATS-Bench/draw-table.py +++ b/exps/NATS-Bench/draw-table.py @@ -1,5 +1,5 @@ ############################################################### -# NATS-Bench (https://arxiv.org/pdf/2009.00437.pdf) # +# NATS-Bench (arxiv.org/pdf/2009.00437.pdf), IEEE TPAMI 2021 # # The code to draw some results in Table 4 in our paper. # ############################################################### # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.06 # @@ -156,4 +156,4 @@ if __name__ == '__main__': largest_index = api_sss.query_index_by_arch(largest) print(show_valid_test(api_sss, largest_index)) for dataset in ['cifar10', 'cifar100', 'ImageNet16-120']: - find_best_valid(api_sss, dataset) \ No newline at end of file + find_best_valid(api_sss, dataset) diff --git a/exps/NATS-Bench/main-sss.py b/exps/NATS-Bench/main-sss.py index 8a2badb..062289a 100644 --- a/exps/NATS-Bench/main-sss.py +++ b/exps/NATS-Bench/main-sss.py @@ -1,5 +1,5 @@ ############################################################################## -# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +# NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size # ############################################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.07 # ############################################################################## diff --git a/exps/NATS-Bench/main-tss.py b/exps/NATS-Bench/main-tss.py index b862006..189c0e1 100644 --- a/exps/NATS-Bench/main-tss.py +++ b/exps/NATS-Bench/main-tss.py @@ -1,5 +1,5 @@ ############################################################################## -# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +# NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size # ############################################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.07 # ############################################################################## diff --git a/exps/NATS-Bench/sss-collect.py b/exps/NATS-Bench/sss-collect.py index ce15dc7..036a3e7 100644 --- a/exps/NATS-Bench/sss-collect.py +++ b/exps/NATS-Bench/sss-collect.py @@ -1,5 +1,5 @@ ############################################################################## -# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +# NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size # ############################################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 # ############################################################################## diff --git a/exps/NATS-Bench/sss-file-manager.py b/exps/NATS-Bench/sss-file-manager.py index 64d3b59..93373ff 100644 --- a/exps/NATS-Bench/sss-file-manager.py +++ b/exps/NATS-Bench/sss-file-manager.py @@ -1,5 +1,5 @@ ############################################################################## -# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +# NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size # ############################################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 # ############################################################################## diff --git a/exps/NATS-Bench/test-nats-api.py b/exps/NATS-Bench/test-nats-api.py index cc619a3..a9ca1e4 100644 --- a/exps/NATS-Bench/test-nats-api.py +++ b/exps/NATS-Bench/test-nats-api.py @@ -1,5 +1,5 @@ ############################################################################## -# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +# NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size # ############################################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 # ############################################################################## diff --git a/exps/NATS-Bench/tss-collect-patcher.py b/exps/NATS-Bench/tss-collect-patcher.py index 1337ab5..3a9414e 100644 --- a/exps/NATS-Bench/tss-collect-patcher.py +++ b/exps/NATS-Bench/tss-collect-patcher.py @@ -1,5 +1,5 @@ ############################################################################## -# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +# NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size # ############################################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 # ############################################################################## diff --git a/exps/NATS-Bench/tss-collect.py b/exps/NATS-Bench/tss-collect.py index ca4f0db..62cc0dd 100644 --- a/exps/NATS-Bench/tss-collect.py +++ b/exps/NATS-Bench/tss-collect.py @@ -1,5 +1,5 @@ ############################################################################## -# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +# NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size # ############################################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 # ############################################################################## diff --git a/exps/NATS-Bench/tss-file-manager.py b/exps/NATS-Bench/tss-file-manager.py index e6b3be4..7c0ca6c 100644 --- a/exps/NATS-Bench/tss-file-manager.py +++ b/exps/NATS-Bench/tss-file-manager.py @@ -1,5 +1,5 @@ ############################################################################## -# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size # +# NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size # ############################################################################## # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 # ##############################################################################