From 6fbaee7a6d7ac42d5c1bad8566ac461039ae2438 Mon Sep 17 00:00:00 2001 From: jack-willturner Date: Wed, 3 Jun 2020 17:10:23 +0100 Subject: [PATCH] Final check --- README.md | 17 ++++++++++------- environment.yml | 3 +-- process_results.py | 16 ++++++++-------- reproduce.sh | 20 +++++++++++--------- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 762777b..cee09ae 100644 --- a/README.md +++ b/README.md @@ -10,22 +10,25 @@ To reproduce our results: conda env create -f environment.yml conda activate nas-wot -./reproduce.sh +./reproduce.sh 3 # average accuracy over 3 runs +./reproduce.sh 500 # average accuracy over 500 runs (this will take longer) ``` -For a quick run you can set `--n_runs 3` to get results after 3 runs: +Each command will finish by calling `process_results.py`, which will print a table. `./reproduce.sh 3` should print the following table: | Method | Search time (s) | CIFAR-10 (val) | CIFAR-10 (test) | CIFAR-100 (val) | CIFAR-100 (test) | ImageNet16-120 (val) | ImageNet16-120 (test) | |:-------------|------------------:|:-----------------|:------------------|:------------------|:-------------------|:-----------------------|:------------------------| -| Ours (N=10) | 1.73435 | 88.99 $\pm$ 0.24 | 92.42 $\pm$ 0.33 | 67.86 $\pm$ 0.49 | 67.54 $\pm$ 0.75 | 41.16 $\pm$ 2.31 | 40.98 $\pm$ 2.72 | -| Ours (N=100) | 17.4139 | 89.18 $\pm$ 0.29 | 91.76 $\pm$ 1.28 | 67.17 $\pm$ 2.79 | 67.27 $\pm$ 2.68 | 40.84 $\pm$ 5.36 | 41.33 $\pm$ 5.74 +| Ours (N=10) | 1.73435 | 88.99 +- 0.24 | 92.42 +- 0.33 | 67.86 +- 0.49 | 67.54 +- 0.75 | 41.16 +- 2.31 | 40.98 +- 2.72 | +| Ours (N=100) | 17.4139 | 89.18 +- 0.29 | 91.76 +- 1.28 | 67.17 +- 2.79 | 67.27 +- 2.68 | 40.84 +- 5.36 | 41.33 +- 5.74 -The size of `N` is set with `--n_samples 10`. To produce the results in the paper, set `--n_runs 500`: +`./reproduce 500` will produce the following table (which is the same as what we report in the paper): | Method | Search time (s) | CIFAR-10 (val) | CIFAR-10 (test) | CIFAR-100 (val) | CIFAR-100 (test) | ImageNet16-120 (val) | ImageNet16-120 (test) | |:-------------|------------------:|:-----------------|:------------------|:------------------|:-------------------|:-----------------------|:------------------------| -| Ours (N=10) | 1.73435 | 89.25 $\pm$ 0.08 | 92.21 $\pm$ 0.11 | 68.53 $\pm$ 0.17 | 68.40 $\pm$ 0.14 | 40.42 $\pm$ 1.15 | 40.66 $\pm$ 0.97 | -| Ours (N=100) | 17.4139 | 88.45 $\pm$ 1.46 | 91.61 $\pm$ 1.71 | 66.42 $\pm$ 3.27 | 66.56 $\pm$ 3.28 | 36.56 $\pm$ 6.70 | 36.37 $\pm$ 6.97 +| Ours (N=10) | 1.73435 | 89.25 +- 0.08 | 92.21 +- 0.11 | 68.53 +- 0.17 | 68.40 +- 0.14 | 40.42 +- 1.15 | 40.66 +- 0.97 | +| Ours (N=100) | 17.4139 | 88.45 +- 1.46 | 91.61 +- 1.71 | 66.42 +- 3.27 | 66.56 +- 3.28 | 36.56 +- 6.70 | 36.37 +- 6.97 +To try different sample sizes, simply change the `--n_samples` argument in the call to `search.py`, and update the list of sample sizes on line 51 of `process_results.py`. + The code is licensed under the MIT licence. diff --git a/environment.yml b/environment.yml index ccfa99d..019b5df 100644 --- a/environment.yml +++ b/environment.yml @@ -51,5 +51,4 @@ dependencies: - pip: - argparse==1.4.0 - nas-bench-201==1.3 -prefix: /home/jturner/miniconda3/envs/nas-wot - + - tabulate==0.8.7 diff --git a/process_results.py b/process_results.py index 3d33efa..a0e880c 100644 --- a/process_results.py +++ b/process_results.py @@ -63,25 +63,25 @@ for n_samples in [10, 100]: full_scores = torch.load(filename) if dataset == 'CIFAR-10 (test)': time = median(full_scores['times']) - dataset_top1s['Search time (s)'] = time + time = f"{time:.2f}" accs = [] for n in range(args.n_runs): acc = full_scores[acc_type][n] accs.append(acc) dataset_top1s[dataset] = accs - cifar10_val = f"{mean(dataset_top1s['CIFAR-10 (val)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-10 (val)']):.2f}" - cifar10_test = f"{mean(dataset_top1s['CIFAR-10 (test)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-10 (test)']):.2f}" + cifar10_val = f"{mean(dataset_top1s['CIFAR-10 (val)']):.2f} +- {std(dataset_top1s['CIFAR-10 (val)']):.2f}" + cifar10_test = f"{mean(dataset_top1s['CIFAR-10 (test)']):.2f} +- {std(dataset_top1s['CIFAR-10 (test)']):.2f}" - cifar100_val = f"{mean(dataset_top1s['CIFAR-100 (val)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-100 (val)']):.2f}" - cifar100_test = f"{mean(dataset_top1s['CIFAR-100 (test)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-100 (test)']):.2f}" + cifar100_val = f"{mean(dataset_top1s['CIFAR-100 (val)']):.2f} +- {std(dataset_top1s['CIFAR-100 (val)']):.2f}" + cifar100_test = f"{mean(dataset_top1s['CIFAR-100 (test)']):.2f} +- {std(dataset_top1s['CIFAR-100 (test)']):.2f}" - imagenet_val = f"{mean(dataset_top1s['ImageNet16-120 (val)']):.2f} $\pm$ {std(dataset_top1s['ImageNet16-120 (val)']):.2f}" - imagenet_test = f"{mean(dataset_top1s['ImageNet16-120 (test)']):.2f} $\pm$ {std(dataset_top1s['ImageNet16-120 (test)']):.2f}" + imagenet_val = f"{mean(dataset_top1s['ImageNet16-120 (val)']):.2f} +- {std(dataset_top1s['ImageNet16-120 (val)']):.2f}" + imagenet_test = f"{mean(dataset_top1s['ImageNet16-120 (test)']):.2f} +- {std(dataset_top1s['ImageNet16-120 (test)']):.2f}" df.append([method, time, cifar10_val, cifar10_test, cifar100_val, cifar100_test, imagenet_val, imagenet_test]) df = pd.DataFrame(df, columns=['Method','Search time (s)','CIFAR-10 (val)','CIFAR-10 (test)','CIFAR-100 (val)','CIFAR-100 (test)','ImageNet16-120 (val)','ImageNet16-120 (test)' ]) -df.round(2) + print(tabulate.tabulate(df.values,df.columns, tablefmt="pipe")) diff --git a/reproduce.sh b/reproduce.sh index 5e63cdf..bcbe672 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -1,11 +1,13 @@ -#python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 10 -#python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 10 -#python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs 3 --n_samples 10 -#python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs 3 --n_samples 10 +#!/bin/bash -python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 100 -python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 100 -python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs 3 --n_samples 100 -python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs 3 --n_samples 100 +python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs $1 --n_samples 10 +python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs $1 --n_samples 10 +python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs $1 --n_samples 10 +python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs $1 --n_samples 10 -python process_results.py --n_runs 3 +python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs $1 --n_samples 100 +python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs $1 --n_samples 100 +python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs $1 --n_samples 100 +python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs $1 --n_samples 100 + +python process_results.py --n_runs $1