iicd/naswot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Produce table for multiple Ns

Browse Source
This commit is contained in:
jack-willturner 2020-06-03 16:28:45 +01:00
parent 45901dd7ec
commit 204a72f95e
4 changed files with 40 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
*.pth
__pycache__
*.t7

3
README.md
View File

@ -17,6 +17,7 @@ Will produce the following table:
| Method | Search time (s) | CIFAR-10 (val) | CIFAR-10 (test) | CIFAR-100 (val) | CIFAR-100 (test) | ImageNet16-120 (val) | ImageNet16-120 (test) |
|:-------------|------------------:|:-----------------|:------------------|:------------------|:-------------------|:-----------------------|:------------------------|
| Ours (N=100) | 18.35 | 89.18 +- 0.29 | 91.76 +- 1.28 | 67.17 +- 2.79 | 67.27 +- 2.68 | 40.84 +- 5.36 | 41.33 +- 5.74 |
| Ours (N=10) | 1.73435 | 88.99 $\pm$ 0.24 | 92.42 $\pm$ 0.33 | 67.86 $\pm$ 0.49 | 67.54 $\pm$ 0.75 | 41.16 $\pm$ 2.31 | 40.98 $\pm$ 2.72 |
| Ours (N=100) | 17.4139 | 89.18 $\pm$ 0.29 | 91.76 $\pm$ 1.28 | 67.17 $\pm$ 2.79 | 67.27 $\pm$ 2.68 | 40.84 $\pm$ 5.36 | 41.33 $\pm$ 5.74
The code is licensed under the MIT licence.

55
process_results.py
View File

@ -16,7 +16,6 @@ parser.add_argument('--GPU', default='0', type=str)
parser.add_argument('--seed', default=1, type=int)
parser.add_argument('--trainval', action='store_true')
parser.add_argument('--n_samples', default=100, type=int, help='how many samples to take')
parser.add_argument('--n_runs', default=500, type=int)
args = parser.parse_args()
@ -48,39 +47,41 @@ datasets['ImageNet16-120 (test)'] = ('ImageNet16-120', 'x-test', False)
dataset_top1s = OrderedDict()
dataset_top1s['Method'] = f"Ours (N={args.n_samples})"
dataset_top1s['Search time (s)'] = np.nan
time = 0.
for n_samples in [10, 100]:
method = f"Ours (N={n_samples})"
for dataset, params in datasets.items():
top1s = []
time = 0.
dset = params[0]
acc_type = 'accs' if 'test' in params[1] else 'val_accs'
filename = f"{args.save_loc}/{dset}_{args.n_runs}_{args.n_samples}_{args.seed}.t7"
for dataset, params in datasets.items():
top1s = []
full_scores = torch.load(filename)
if dataset == 'CIFAR-10 (test)':
time = median(full_scores['times'])
dataset_top1s['Search time (s)'] = time
accs = []
for n in range(args.n_runs):
acc = full_scores[acc_type][n]
accs.append(acc)
dataset_top1s[dataset] = accs
dset = params[0]
acc_type = 'accs' if 'test' in params[1] else 'val_accs'
filename = f"{args.save_loc}/{dset}_{args.n_runs}_{n_samples}_{args.seed}.t7"
df = pd.DataFrame(dataset_top1s)
full_scores = torch.load(filename)
if dataset == 'CIFAR-10 (test)':
time = median(full_scores['times'])
dataset_top1s['Search time (s)'] = time
accs = []
for n in range(args.n_runs):
acc = full_scores[acc_type][n]
accs.append(acc)
dataset_top1s[dataset] = accs
df['CIFAR-10 (val)'] = f"{mean(df['CIFAR-10 (val)']):.2f} +- {std(df['CIFAR-10 (val)']):.2f}"
df['CIFAR-10 (test)'] = f"{mean(df['CIFAR-10 (test)']):.2f} +- {std(df['CIFAR-10 (test)']):.2f}"
cifar10_val = f"{mean(dataset_top1s['CIFAR-10 (val)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-10 (val)']):.2f}"
cifar10_test = f"{mean(dataset_top1s['CIFAR-10 (test)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-10 (test)']):.2f}"
df['CIFAR-100 (val)'] = f"{mean(df['CIFAR-100 (val)']):.2f} +- {std(df['CIFAR-100 (val)']):.2f}"
df['CIFAR-100 (test)'] = f"{mean(df['CIFAR-100 (test)']):.2f} +- {std(df['CIFAR-100 (test)']):.2f}"
cifar100_val = f"{mean(dataset_top1s['CIFAR-100 (val)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-100 (val)']):.2f}"
cifar100_test = f"{mean(dataset_top1s['CIFAR-100 (test)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-100 (test)']):.2f}"
df['ImageNet16-120 (val)'] = f"{mean(df['ImageNet16-120 (val)']):.2f} +- {std(df['ImageNet16-120 (val)']):.2f}"
df['ImageNet16-120 (test)'] = f"{mean(df['ImageNet16-120 (test)']):.2f} +- {std(df['ImageNet16-120 (test)']):.2f}"
imagenet_val = f"{mean(dataset_top1s['ImageNet16-120 (val)']):.2f} $\pm$ {std(dataset_top1s['ImageNet16-120 (val)']):.2f}"
imagenet_test = f"{mean(dataset_top1s['ImageNet16-120 (test)']):.2f} $\pm$ {std(dataset_top1s['ImageNet16-120 (test)']):.2f}"
df = df.round(2)
df = df.iloc[:1]
df.append([method, time, cifar10_val, cifar10_test, cifar100_val, cifar100_test, imagenet_val, imagenet_test])
df = pd.DataFrame(df, columns=['Method','Search time (s)','CIFAR-10 (val)','CIFAR-10 (test)','CIFAR-100 (val)','CIFAR-100 (test)','ImageNet16-120 (val)','ImageNet16-120 (test)' ])
df.round(2)
print(tabulate.tabulate(df.values,df.columns, tablefmt="pipe"))

13
reproduce.sh
View File

@ -1,6 +1,11 @@
python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs 3
python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs 3
python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs 3
python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs 3
#python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 10
#python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 10
#python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs 3 --n_samples 10
#python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs 3 --n_samples 10
python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 100
python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 100
python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs 3 --n_samples 100
python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs 3 --n_samples 100
python process_results.py --n_runs 3