xautodl/exps/NAS-Bench-201/test-correlation.py

#####################################################
# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 #
########################################################
# python exps/NAS-Bench-201/test-correlation.py --api_path $HOME/.torch/NAS-Bench-201-v1_0-e61699.pth
########################################################
import sys, argparse
import numpy as np
from copy import deepcopy
from tqdm import tqdm
import torch
from pathlib import Path
lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve()
if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir))
from log_utils    import time_string
from models       import CellStructure
from nas_201_api  import NASBench201API as API


def check_unique_arch(meta_file):
  api = API(str(meta_file))
  arch_strs = deepcopy(api.meta_archs)
  xarchs = [CellStructure.str2structure(x) for x in arch_strs]
  def get_unique_matrix(archs, consider_zero):
    UniquStrs = [arch.to_unique_str(consider_zero) for arch in archs]
    print ('{:} create unique-string ({:}/{:}) done'.format(time_string(), len(set(UniquStrs)), len(UniquStrs)))
    Unique2Index = dict()
    for index, xstr in enumerate(UniquStrs):
      if xstr not in Unique2Index: Unique2Index[xstr] = list()
      Unique2Index[xstr].append( index )
    sm_matrix = torch.eye(len(archs)).bool()
    for _, xlist in Unique2Index.items():
      for i in xlist:
        for j in xlist:
          sm_matrix[i,j] = True
    unique_ids, unique_num = [-1 for _ in archs], 0
    for i in range(len(unique_ids)):
      if unique_ids[i] > -1: continue
      neighbours = sm_matrix[i].nonzero().view(-1).tolist()
      for nghb in neighbours:
        assert unique_ids[nghb] == -1, 'impossible'
        unique_ids[nghb] = unique_num
      unique_num += 1
    return sm_matrix, unique_ids, unique_num

  print ('There are {:} valid-archs'.format( sum(arch.check_valid() for arch in xarchs) ))
  sm_matrix, uniqueIDs, unique_num = get_unique_matrix(xarchs, None)
  print ('{:} There are {:} unique architectures (considering nothing).'.format(time_string(), unique_num))
  sm_matrix, uniqueIDs, unique_num = get_unique_matrix(xarchs, False)
  print ('{:} There are {:} unique architectures (not considering zero).'.format(time_string(), unique_num))
  sm_matrix, uniqueIDs, unique_num = get_unique_matrix(xarchs,  True)
  print ('{:} There are {:} unique architectures (considering zero).'.format(time_string(), unique_num))


def check_cor_for_bandit(meta_file, test_epoch, use_less_or_not, is_rand=True, need_print=False):
  if isinstance(meta_file, API):
    api = meta_file
  else:
    api = API(str(meta_file))
  cifar10_currs     = []
  cifar10_valid     = []
  cifar10_test      = []
  cifar100_valid    = []
  cifar100_test     = []
  imagenet_test     = []
  imagenet_valid    = []
  for idx, arch in enumerate(api):
    results = api.get_more_info(idx, 'cifar10-valid' , test_epoch-1, use_less_or_not, is_rand)
    cifar10_currs.append( results['valid-accuracy'] )
    # --->>>>>
    results = api.get_more_info(idx, 'cifar10-valid' , None, False, is_rand)
    cifar10_valid.append( results['valid-accuracy'] )
    results = api.get_more_info(idx, 'cifar10'       , None, False, is_rand)
    cifar10_test.append( results['test-accuracy'] )
    results = api.get_more_info(idx, 'cifar100'      , None, False, is_rand)
    cifar100_test.append( results['test-accuracy'] )
    cifar100_valid.append( results['valid-accuracy'] )
    results = api.get_more_info(idx, 'ImageNet16-120', None, False, is_rand)
    imagenet_test.append( results['test-accuracy'] )
    imagenet_valid.append( results['valid-accuracy'] )
  def get_cor(A, B):
    return float(np.corrcoef(A, B)[0,1])
  cors = []
  for basestr, xlist in zip(['C-010-V', 'C-010-T', 'C-100-V', 'C-100-T', 'I16-V', 'I16-T'], [cifar10_valid, cifar10_test, cifar100_valid, cifar100_test, imagenet_valid, imagenet_test]):
    correlation = get_cor(cifar10_currs, xlist)
    if need_print: print ('With {:3d}/{:}-epochs-training, the correlation between cifar10-valid and {:} is : {:}'.format(test_epoch, '012' if use_less_or_not else '200', basestr, correlation))
    cors.append( correlation )
    #print ('With {:3d}/200-epochs-training, the correlation between cifar10-valid and {:} is : {:}'.format(test_epoch, basestr, get_cor(cifar10_valid_200, xlist)))
    #print('-'*200)
  #print('*'*230)
  return cors


def check_cor_for_bandit_v2(meta_file, test_epoch, use_less_or_not, is_rand):
  corrs = []
  for i in tqdm(range(100)):
    x = check_cor_for_bandit(meta_file, test_epoch, use_less_or_not, is_rand, False)
    corrs.append( x )
  #xstrs = ['CIFAR-010', 'C-100-V', 'C-100-T', 'I16-V', 'I16-T']
  xstrs = ['C-010-V', 'C-010-T', 'C-100-V', 'C-100-T', 'I16-V', 'I16-T']
  correlations = np.array(corrs)
  print('------>>>>>>>> {:03d}/{:} >>>>>>>> ------'.format(test_epoch, '012' if use_less_or_not else '200'))
  for idx, xstr in enumerate(xstrs):
    print ('{:8s} ::: mean={:.4f}, std={:.4f} :: {:.4f}\\pm{:.4f}'.format(xstr, correlations[:,idx].mean(), correlations[:,idx].std(), correlations[:,idx].mean(), correlations[:,idx].std()))
  print('')


if __name__ == '__main__':
  parser = argparse.ArgumentParser("Analysis of NAS-Bench-201")
  parser.add_argument('--save_dir',  type=str, default='./output/search-cell-nas-bench-201/visuals', help='The base-name of folder to save checkpoints and log.')
  parser.add_argument('--api_path',  type=str, default=None,                                         help='The path to the NAS-Bench-201 benchmark file.')
  args = parser.parse_args()

  vis_save_dir = Path(args.save_dir)
  vis_save_dir.mkdir(parents=True, exist_ok=True)
  meta_file = Path(args.api_path)
  assert meta_file.exists(), 'invalid path for api : {:}'.format(meta_file)

  #check_unique_arch(meta_file)
  api = API(str(meta_file))
  #for iepoch in [11, 25, 50, 100, 150, 175, 200]:
  #  check_cor_for_bandit(api,  6, iepoch)
  #  check_cor_for_bandit(api, 12, iepoch)
  check_cor_for_bandit_v2(api,   6,  True, True)
  check_cor_for_bandit_v2(api,  12,  True, True)
  check_cor_for_bandit_v2(api,  12, False, True)
  check_cor_for_bandit_v2(api,  24, False, True)
  check_cor_for_bandit_v2(api, 100, False, True)
  check_cor_for_bandit_v2(api, 150, False, True)
  check_cor_for_bandit_v2(api, 175, False, True)
  check_cor_for_bandit_v2(api, 200, False, True)
  print('----')