Update for Rebuttal

exps/NATS-Bench/draw-table.py

@@ -26,6 +26,27 @@ from nats_bench import create
 from log_utils import time_string
 
 
+def fetch_data(root_dir='./output/search', search_space='tss', dataset=None):
+  ss_dir = '{:}-{:}'.format(root_dir, search_space)
+  alg2name, alg2path = OrderedDict(), OrderedDict()
+  alg2name['REA'] = 'R-EA-SS3'
+  alg2name['REINFORCE'] = 'REINFORCE-0.01'
+  alg2name['RANDOM'] = 'RANDOM'
+  alg2name['BOHB'] = 'BOHB'
+  for alg, name in alg2name.items():
+    alg2path[alg] = os.path.join(ss_dir, dataset, name, 'results.pth')
+    assert os.path.isfile(alg2path[alg]), 'invalid path : {:}'.format(alg2path[alg])
+  alg2data = OrderedDict()
+  for alg, path in alg2path.items():
+    data = torch.load(path)
+    for index, info in data.items():
+      info['time_w_arch'] = [(x, y) for x, y in zip(info['all_total_times'], info['all_archs'])]
+      for j, arch in enumerate(info['all_archs']):
+        assert arch != -1, 'invalid arch from {:} {:} {:} ({:}, {:})'.format(alg, search_space, dataset, index, j)
+    alg2data[alg] = data
+  return alg2data
+
+
 def get_valid_test_acc(api, arch, dataset):
   is_size_space = api.search_space_name == 'size'
   if dataset == 'cifar10':
@@ -52,7 +73,6 @@ def show_valid_test(api, arch):
 def find_best_valid(api, dataset):
   all_valid_accs, all_test_accs = [], []
   for index, arch in enumerate(api):
-    # import pdb; pdb.set_trace()
     valid_acc, test_acc, perf_str = get_valid_test_acc(api, index, dataset)
     all_valid_accs.append((index, valid_acc))
     all_test_accs.append((index, test_acc))
@@ -68,8 +88,62 @@ def find_best_valid(api, dataset):
   print('using test       ::: {:}'.format(perf_str))
 
 
+def interplate_fn(xpair1, xpair2, x):
+  (x1, y1) = xpair1
+  (x2, y2) = xpair2
+  return (x2 - x) / (x2 - x1) * y1 + \
+         (x - x1) / (x2 - x1) * y2
+
+def query_performance(api, info, dataset, ticket):
+  info = deepcopy(info)
+  results, is_size_space = [], api.search_space_name == 'size'
+  time_w_arch = sorted(info['time_w_arch'], key=lambda x: abs(x[0]-ticket))
+  time_a, arch_a = time_w_arch[0]
+  time_b, arch_b = time_w_arch[1]
+
+  v_acc_a, t_acc_a, _ = get_valid_test_acc(api, arch_a, dataset)
+  v_acc_b, t_acc_b, _ = get_valid_test_acc(api, arch_b, dataset)
+  v_acc = interplate_fn((time_a, v_acc_a), (time_b, v_acc_b), ticket)
+  t_acc = interplate_fn((time_a, t_acc_a), (time_b, t_acc_b), ticket)
+  # if True:
+  #   interplate = (time_b-ticket) / (time_b-time_a) * accuracy_a + (ticket-time_a) / (time_b-time_a) * accuracy_b
+  #   results.append(interplate)
+  # return sum(results) / len(results)
+  return v_acc, t_acc
+
+
+def show_multi_trial(search_space):
+  api = create(None, search_space, fast_mode=True, verbose=False)
+  def show(dataset):
+    print('show {:} on {:} done.'.format(dataset, search_space))
+    xdataset, max_time = dataset.split('-T')
+    alg2data = fetch_data(search_space=search_space, dataset=dataset)
+    for idx, (alg, data) in enumerate(alg2data.items()):
+
+      valid_accs, test_accs = [], []
+      for _, x in data.items():
+        v_acc, t_acc = query_performance(api, x, xdataset, float(max_time))
+        valid_accs.append(v_acc)
+        test_accs.append(t_acc)
+      valid_str = '{:.2f}$\pm${:.2f}'.format(np.mean(valid_accs), np.std(valid_accs))
+      test_str = '{:.2f}$\pm${:.2f}'.format(np.mean(test_accs), np.std(test_accs))
+      print('{:} plot alg : {:10s}  | validation = {:} | test = {:}'.format(time_string(), alg, valid_str, test_str))
+  if search_space == 'tss':
+    datasets = ['cifar10-T20000', 'cifar100-T40000', 'ImageNet16-120-T120000']
+  elif search_space == 'sss':
+    datasets = ['cifar10-T20000', 'cifar100-T40000', 'ImageNet16-120-T60000']
+  else:
+    raise ValueError('Unknown search space: {:}'.format(search_space))
+  for dataset in datasets:
+    show(dataset)
+  print('{:} complete show multi-trial results.\n'.format(time_string()))
+
+
 if __name__ == '__main__':
   
+  show_multi_trial('tss')
+  show_multi_trial('sss')
+
   api_tss = create(None, 'tss', fast_mode=False, verbose=False)
   resnet = '|nor_conv_3x3~0|+|none~0|nor_conv_3x3~1|+|skip_connect~0|none~1|skip_connect~2|'
   resnet_index = api_tss.query_index_by_arch(resnet)