diff --git a/.gitignore b/.gitignore index 61f654f..f6131d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pth __pycache__ +*.t7 diff --git a/README.md b/README.md index 5e2df1d..dae08da 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Will produce the following table: | Method | Search time (s) | CIFAR-10 (val) | CIFAR-10 (test) | CIFAR-100 (val) | CIFAR-100 (test) | ImageNet16-120 (val) | ImageNet16-120 (test) | |:-------------|------------------:|:-----------------|:------------------|:------------------|:-------------------|:-----------------------|:------------------------| -| Ours (N=100) | 18.35 | 89.18 +- 0.29 | 91.76 +- 1.28 | 67.17 +- 2.79 | 67.27 +- 2.68 | 40.84 +- 5.36 | 41.33 +- 5.74 | +| Ours (N=10) | 1.73435 | 88.99 $\pm$ 0.24 | 92.42 $\pm$ 0.33 | 67.86 $\pm$ 0.49 | 67.54 $\pm$ 0.75 | 41.16 $\pm$ 2.31 | 40.98 $\pm$ 2.72 | +| Ours (N=100) | 17.4139 | 89.18 $\pm$ 0.29 | 91.76 $\pm$ 1.28 | 67.17 $\pm$ 2.79 | 67.27 $\pm$ 2.68 | 40.84 $\pm$ 5.36 | 41.33 $\pm$ 5.74 The code is licensed under the MIT licence. diff --git a/process_results.py b/process_results.py index a58422c..3d33efa 100644 --- a/process_results.py +++ b/process_results.py @@ -16,7 +16,6 @@ parser.add_argument('--GPU', default='0', type=str) parser.add_argument('--seed', default=1, type=int) parser.add_argument('--trainval', action='store_true') -parser.add_argument('--n_samples', default=100, type=int, help='how many samples to take') parser.add_argument('--n_runs', default=500, type=int) args = parser.parse_args() @@ -48,39 +47,41 @@ datasets['ImageNet16-120 (test)'] = ('ImageNet16-120', 'x-test', False) dataset_top1s = OrderedDict() -dataset_top1s['Method'] = f"Ours (N={args.n_samples})" -dataset_top1s['Search time (s)'] = np.nan -time = 0. +for n_samples in [10, 100]: + method = f"Ours (N={n_samples})" -for dataset, params in datasets.items(): - top1s = [] + time = 0. - dset = params[0] - acc_type = 'accs' if 'test' in params[1] else 'val_accs' - filename = f"{args.save_loc}/{dset}_{args.n_runs}_{args.n_samples}_{args.seed}.t7" + for dataset, params in datasets.items(): + top1s = [] - full_scores = torch.load(filename) - if dataset == 'CIFAR-10 (test)': - time = median(full_scores['times']) - dataset_top1s['Search time (s)'] = time - accs = [] - for n in range(args.n_runs): - acc = full_scores[acc_type][n] - accs.append(acc) - dataset_top1s[dataset] = accs + dset = params[0] + acc_type = 'accs' if 'test' in params[1] else 'val_accs' + filename = f"{args.save_loc}/{dset}_{args.n_runs}_{n_samples}_{args.seed}.t7" -df = pd.DataFrame(dataset_top1s) + full_scores = torch.load(filename) + if dataset == 'CIFAR-10 (test)': + time = median(full_scores['times']) + dataset_top1s['Search time (s)'] = time + accs = [] + for n in range(args.n_runs): + acc = full_scores[acc_type][n] + accs.append(acc) + dataset_top1s[dataset] = accs -df['CIFAR-10 (val)'] = f"{mean(df['CIFAR-10 (val)']):.2f} +- {std(df['CIFAR-10 (val)']):.2f}" -df['CIFAR-10 (test)'] = f"{mean(df['CIFAR-10 (test)']):.2f} +- {std(df['CIFAR-10 (test)']):.2f}" + cifar10_val = f"{mean(dataset_top1s['CIFAR-10 (val)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-10 (val)']):.2f}" + cifar10_test = f"{mean(dataset_top1s['CIFAR-10 (test)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-10 (test)']):.2f}" -df['CIFAR-100 (val)'] = f"{mean(df['CIFAR-100 (val)']):.2f} +- {std(df['CIFAR-100 (val)']):.2f}" -df['CIFAR-100 (test)'] = f"{mean(df['CIFAR-100 (test)']):.2f} +- {std(df['CIFAR-100 (test)']):.2f}" + cifar100_val = f"{mean(dataset_top1s['CIFAR-100 (val)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-100 (val)']):.2f}" + cifar100_test = f"{mean(dataset_top1s['CIFAR-100 (test)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-100 (test)']):.2f}" -df['ImageNet16-120 (val)'] = f"{mean(df['ImageNet16-120 (val)']):.2f} +- {std(df['ImageNet16-120 (val)']):.2f}" -df['ImageNet16-120 (test)'] = f"{mean(df['ImageNet16-120 (test)']):.2f} +- {std(df['ImageNet16-120 (test)']):.2f}" + imagenet_val = f"{mean(dataset_top1s['ImageNet16-120 (val)']):.2f} $\pm$ {std(dataset_top1s['ImageNet16-120 (val)']):.2f}" + imagenet_test = f"{mean(dataset_top1s['ImageNet16-120 (test)']):.2f} $\pm$ {std(dataset_top1s['ImageNet16-120 (test)']):.2f}" -df = df.round(2) -df = df.iloc[:1] + df.append([method, time, cifar10_val, cifar10_test, cifar100_val, cifar100_test, imagenet_val, imagenet_test]) + + +df = pd.DataFrame(df, columns=['Method','Search time (s)','CIFAR-10 (val)','CIFAR-10 (test)','CIFAR-100 (val)','CIFAR-100 (test)','ImageNet16-120 (val)','ImageNet16-120 (test)' ]) +df.round(2) print(tabulate.tabulate(df.values,df.columns, tablefmt="pipe")) diff --git a/reproduce.sh b/reproduce.sh index fdf99dd..dd0fc9b 100755 --- a/reproduce.sh +++ b/reproduce.sh @@ -1,6 +1,11 @@ -python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs 3 -python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs 3 -python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs 3 -python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs 3 +#python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 10 +#python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 10 +#python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs 3 --n_samples 10 +#python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs 3 --n_samples 10 + +python search.py --dataset cifar10 --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 100 +python search.py --dataset cifar10 --trainval --data_loc '../datasets/cifar10' --n_runs 3 --n_samples 100 +python search.py --dataset cifar100 --data_loc '../datasets/cifar100' --n_runs 3 --n_samples 100 +python search.py --dataset ImageNet16-120 --data_loc '../datasets/ImageNet16' --n_runs 3 --n_samples 100 python process_results.py --n_runs 3