91 lines
3.8 KiB
Python
91 lines
3.8 KiB
Python
###############################################################
|
|
# NATS-Bench (https://arxiv.org/pdf/2009.00437.pdf) #
|
|
###############################################################
|
|
# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.06 #
|
|
###############################################################
|
|
# Usage: python exps/NATS-Bench/draw-correlations.py #
|
|
###############################################################
|
|
import os, gc, sys, time, scipy, torch, argparse
|
|
import numpy as np
|
|
from typing import List, Text, Dict, Any
|
|
from shutil import copyfile
|
|
from collections import defaultdict, OrderedDict
|
|
from copy import deepcopy
|
|
from pathlib import Path
|
|
import matplotlib
|
|
import seaborn as sns
|
|
matplotlib.use('agg')
|
|
import matplotlib.pyplot as plt
|
|
import matplotlib.ticker as ticker
|
|
|
|
lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve()
|
|
if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir))
|
|
from config_utils import dict2config, load_config
|
|
from nats_bench import create
|
|
from log_utils import time_string
|
|
|
|
|
|
def get_valid_test_acc(api, arch, dataset):
|
|
is_size_space = api.search_space_name == 'size'
|
|
if dataset == 'cifar10':
|
|
xinfo = api.get_more_info(arch, dataset=dataset, hp=90 if is_size_space else 200, is_random=False)
|
|
test_acc = xinfo['test-accuracy']
|
|
xinfo = api.get_more_info(arch, dataset='cifar10-valid', hp=90 if is_size_space else 200, is_random=False)
|
|
valid_acc = xinfo['valid-accuracy']
|
|
else:
|
|
xinfo = api.get_more_info(arch, dataset=dataset, hp=90 if is_size_space else 200, is_random=False)
|
|
valid_acc = xinfo['valid-accuracy']
|
|
test_acc = xinfo['test-accuracy']
|
|
return valid_acc, test_acc, 'validation = {:.2f}, test = {:.2f}\n'.format(valid_acc, test_acc)
|
|
|
|
|
|
def compute_kendalltau(vectori, vectorj):
|
|
# indexes = list(range(len(vectori)))
|
|
# rank_1 = sorted(indexes, key=lambda i: vectori[i])
|
|
# rank_2 = sorted(indexes, key=lambda i: vectorj[i])
|
|
# import pdb; pdb.set_trace()
|
|
coef, p = scipy.stats.kendalltau(vectori, vectorj)
|
|
return coef
|
|
|
|
|
|
def compute_spearmanr(vectori, vectorj):
|
|
coef, p = scipy.stats.spearmanr(vectori, vectorj)
|
|
return coef
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
parser.add_argument('--save_dir', type=str, default='output/vis-nas-bench/nas-algos', help='Folder to save checkpoints and log.')
|
|
parser.add_argument('--search_space', type=str, choices=['tss', 'sss'], help='Choose the search space.')
|
|
args = parser.parse_args()
|
|
|
|
save_dir = Path(args.save_dir)
|
|
|
|
api = create(None, 'tss', fast_mode=True, verbose=False)
|
|
indexes = list(range(1, 10000, 300))
|
|
scores_1 = []
|
|
scores_2 = []
|
|
for index in indexes:
|
|
valid_acc, test_acc, _ = get_valid_test_acc(api, index, 'cifar10')
|
|
scores_1.append(valid_acc)
|
|
scores_2.append(test_acc)
|
|
correlation = compute_kendalltau(scores_1, scores_2)
|
|
print('The kendall tau correlation of {:} samples : {:}'.format(len(indexes), correlation))
|
|
correlation = compute_spearmanr(scores_1, scores_2)
|
|
print('The spearmanr correlation of {:} samples : {:}'.format(len(indexes), correlation))
|
|
# scores_1 = ['{:.2f}'.format(x) for x in scores_1]
|
|
# scores_2 = ['{:.2f}'.format(x) for x in scores_2]
|
|
# print(', '.join(scores_1))
|
|
# print(', '.join(scores_2))
|
|
|
|
dpi, width, height = 250, 1000, 1000
|
|
figsize = width / float(dpi), height / float(dpi)
|
|
LabelSize, LegendFontsize = 14, 14
|
|
|
|
fig, ax = plt.subplots(1, 1, figsize=figsize)
|
|
ax.scatter(scores_1, scores_2 , marker='^', s=0.5, c='tab:green', alpha=0.8)
|
|
|
|
save_path = '/Users/xuanyidong/Desktop/test-temp-rank.png'
|
|
fig.savefig(save_path, dpi=dpi, bbox_inches='tight', format='png')
|
|
plt.close('all')
|