Replace nats_bench by soft link

.gitmodules

@@ -1,3 +1,6 @@
 [submodule ".latent-data/qlib"]
 	path = .latent-data/qlib
 	url = git@github.com:D-X-Y/qlib.git
+[submodule ".latent-data/NATS-Bench"]
+	path = .latent-data/NATS-Bench
+	url = git@github.com:D-X-Y/NATS-Bench.git

.latent-data/NATS-Bench

@@ -0,0 +1 @@
+Subproject commit 51187c1e9152ff79b02b11c80bca0b03b402a7e5

README.md

@@ -94,6 +94,12 @@ Some visualization codes may require `opencv`.
 CIFAR and ImageNet should be downloaded and extracted into `$TORCH_HOME`.
 Some methods use knowledge distillation (KD), which require pre-trained models. Please download these models from [Google Drive](https://drive.google.com/open?id=1ANmiYEGX-IQZTfH8w0aSpj-Wypg-0DR-) (or train by yourself) and save into `.latent-data`.
 
+Please use
+```
+git clone --recurse-submodules git@github.com:D-X-Y/AutoDL-Projects.git
+```
+to download this repo with submodules.
+
 ## Citation
 
 If you find that this project helps your research, please consider citing the related paper:

docs/NATS-Bench.md

@@ -8,6 +8,7 @@ We also show the versatility of NATS-Bench by benchmarking 13 recent state-of-th
 This facilitates a much larger community of researchers to focus on developing better NAS algorithms in a more comparable and computationally effective environment.
 
 **You can use `pip install nats_bench` to install the library of NATS-Bench.**
+or install from the [source codes](https://github.com/D-X-Y/NATS-Bench) via `python setup.py install`.
 
 The structure of this Markdown file:
 - [How to use NATS-Bench?](#How-to-Use-NATS-Bench)

exps/NATS-algos/README.md

@@ -9,6 +9,11 @@ The Python files in this folder are used to re-produce the results in ``NATS-Ben
 - [`regularized_ea.py`](https://github.com/D-X-Y/AutoDL-Projects/blob/main/exps/NATS-algos/regularized_ea.py) contains the REA algorithm for both search spaces.
 - [`reinforce.py`](https://github.com/D-X-Y/AutoDL-Projects/blob/main/exps/NATS-algos/reinforce.py) contains the REINFORCE algorithm for both search spaces.
 
+## Requirements
+
+- `nats_bench`>=v1.1 : you can use `pip install nats_bench` to install or from [sources](https://github.com/D-X-Y/NATS-Bench)
+- `hpbandster` : if you want to run BOHB
+
 ## Citation
 
 If you find that this project helps your research, please consider citing the related paper:

lib/layers/super_mlp.py

@@ -0,0 +1,24 @@
+import torch.nn as nn
+from typing import Optional
+
+class MLP(nn.Module):
+  # MLP: FC -> Activation -> Drop -> FC -> Drop
+  def __init__(self, in_features, hidden_features: Optional[int] = None,
+               out_features: Optional[int] = None,
+               act_layer=nn.GELU,
+               drop: Optional[float] = None):
+    super(MLP, self).__init__()
+    out_features = out_features or in_features
+    hidden_features = hidden_features or in_features
+    self.fc1 = nn.Linear(in_features, hidden_features)
+    self.act = act_layer()
+    self.fc2 = nn.Linear(hidden_features, out_features)
+    self.drop = nn.Dropout(drop or 0)
+
+  def forward(self, x):
+    x = self.fc1(x)
+    x = self.act(x)
+    x = self.drop(x)
+    x = self.fc2(x)
+    x = self.drop(x)
+    return x

lib/layers/super_module.py

@@ -0,0 +1,7 @@
+import torch.nn as nn
+
+class SuperModule(nn.Module):
+  def __init__(self):
+    super(SuperModule, self).__init__()
+
+  

lib/nats_bench/__init__.py

@@ -1,70 +0,0 @@
-##############################################################################
-# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 ##########################
-##############################################################################
-# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size #
-##############################################################################
-"""The official Application Programming Interface (API) for NATS-Bench."""
-from nats_bench.api_size import NATSsize
-from nats_bench.api_topology import NATStopology
-from nats_bench.api_utils import ArchResults
-from nats_bench.api_utils import pickle_load
-from nats_bench.api_utils import pickle_save
-from nats_bench.api_utils import ResultsCount
-
-
-NATS_BENCH_API_VERSIONs = ['v1.0',    # [2020.08.31]
-                           'v1.1']    # [2020.12.20] adding unit tests
-NATS_BENCH_SSS_NAMEs = ('sss', 'size')
-NATS_BENCH_TSS_NAMEs = ('tss', 'topology')
-
-
-def version():
-  return NATS_BENCH_API_VERSIONs[-1]
-
-
-def create(file_path_or_dict, search_space, fast_mode=False, verbose=True):
-  """Create the instead for NATS API.
-
-  Args:
-    file_path_or_dict: None or a file path or a directory path.
-    search_space: This is a string indicates the search space in NATS-Bench.
-    fast_mode: If True, we will not load all the data at initialization,
-      instead, the data for each candidate architecture will be loaded when
-      quering it; If False, we will load all the data during initialization.
-    verbose: This is a flag to indicate whether log additional information.
-
-  Raises:
-    ValueError: If not find the matched serach space description.
-
-  Returns:
-    The created NATS-Bench API.
-  """
-  if search_space in NATS_BENCH_TSS_NAMEs:
-    return NATStopology(file_path_or_dict, fast_mode, verbose)
-  elif search_space in NATS_BENCH_SSS_NAMEs:
-    return NATSsize(file_path_or_dict, fast_mode, verbose)
-  else:
-    raise ValueError('invalid search space : {:}'.format(search_space))
-
-
-def search_space_info(main_tag, aux_tag):
-  """Obtain the search space information."""
-  nats_sss = dict(candidates=[8, 16, 24, 32, 40, 48, 56, 64],
-                  num_layers=5)
-  nats_tss = dict(op_names=['none', 'skip_connect',
-                            'nor_conv_1x1', 'nor_conv_3x3',
-                            'avg_pool_3x3'],
-                  num_nodes=4)
-  if main_tag == 'nats-bench':
-    if aux_tag in NATS_BENCH_SSS_NAMEs:
-      return nats_sss
-    elif aux_tag in NATS_BENCH_TSS_NAMEs:
-      return nats_tss
-    else:
-      raise ValueError('Unknown auxiliary tag: {:}'.format(aux_tag))
-  elif main_tag == 'nas-bench-201':
-    if aux_tag is not None:
-      raise ValueError('For NAS-Bench-201, the auxiliary tag should be None.')
-    return nats_tss
-  else:
-    raise ValueError('Unknown main tag: {:}'.format(main_tag))

lib/nats_bench/api_size.py

@@ -1,291 +0,0 @@
-#####################################################
-# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 #
-##############################################################################
-# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size #
-##############################################################################
-# The history of benchmark files are as follows,                             #
-# where the format is (the name is NATS-sss-[version]-[md5].pickle.pbz2)     #
-# [2020.08.31] NATS-sss-v1_0-50262.pickle.pbz2                               #
-##############################################################################
-# pylint: disable=line-too-long
-"""The API for size search space in NATS-Bench."""
-import collections
-import copy
-import os
-import random
-from typing import Dict, Optional, Text, Union, Any
-
-from nats_bench.api_utils import ArchResults
-from nats_bench.api_utils import NASBenchMetaAPI
-from nats_bench.api_utils import get_torch_home
-from nats_bench.api_utils import nats_is_dir
-from nats_bench.api_utils import nats_is_file
-from nats_bench.api_utils import PICKLE_EXT
-from nats_bench.api_utils import pickle_load
-from nats_bench.api_utils import time_string
-
-
-ALL_BASE_NAMES = ['NATS-sss-v1_0-50262']
-
-
-def print_information(information, extra_info=None, show=False):
-  """print out the information of a given ArchResults."""
-  dataset_names = information.get_dataset_names()
-  strings = [
-      information.arch_str,
-      'datasets : {:}, extra-info : {:}'.format(dataset_names, extra_info)
-  ]
-
-  def metric2str(loss, acc):
-    return 'loss = {:.3f} & top1 = {:.2f}%'.format(loss, acc)
-
-  for dataset in dataset_names:
-    metric = information.get_compute_costs(dataset)
-    flop, param, latency = metric['flops'], metric['params'], metric['latency']
-    str1 = '{:14s} FLOP={:6.2f} M, Params={:.3f} MB, latency={:} ms.'.format(
-        dataset, flop, param,
-        '{:.2f}'.format(latency *
-                        1000) if latency is not None and latency > 0 else None)
-    train_info = information.get_metrics(dataset, 'train')
-    if dataset == 'cifar10-valid':
-      valid_info = information.get_metrics(dataset, 'x-valid')
-      test__info = information.get_metrics(dataset, 'ori-test')
-      str2 = '{:14s} train : [{:}], valid : [{:}], test : [{:}]'.format(
-          dataset, metric2str(train_info['loss'], train_info['accuracy']),
-          metric2str(valid_info['loss'], valid_info['accuracy']),
-          metric2str(test__info['loss'], test__info['accuracy']))
-    elif dataset == 'cifar10':
-      test__info = information.get_metrics(dataset, 'ori-test')
-      str2 = '{:14s} train : [{:}], test  : [{:}]'.format(
-          dataset, metric2str(train_info['loss'], train_info['accuracy']),
-          metric2str(test__info['loss'], test__info['accuracy']))
-    else:
-      valid_info = information.get_metrics(dataset, 'x-valid')
-      test__info = information.get_metrics(dataset, 'x-test')
-      str2 = '{:14s} train : [{:}], valid : [{:}], test : [{:}]'.format(
-          dataset, metric2str(train_info['loss'], train_info['accuracy']),
-          metric2str(valid_info['loss'], valid_info['accuracy']),
-          metric2str(test__info['loss'], test__info['accuracy']))
-    strings += [str1, str2]
-  if show: print('\n'.join(strings))
-  return strings
-
-
-class NATSsize(NASBenchMetaAPI):
-  """This is the class for the API of size search space in NATS-Bench."""
-
-  def __init__(self,
-               file_path_or_dict: Optional[Union[Text, Dict[Text, Any]]] = None,
-               fast_mode: bool = False,
-               verbose: bool = True):
-    """The initialization function that takes the dataset file path (or a dict loaded from that path) as input."""
-    self._all_base_names = ALL_BASE_NAMES
-    self.filename = None
-    self._search_space_name = 'size'
-    self._fast_mode = fast_mode
-    self._archive_dir = None
-    self._full_train_epochs = 90
-    self.reset_time()
-    if file_path_or_dict is None:
-      if self._fast_mode:
-        self._archive_dir = os.path.join(
-            get_torch_home(), '{:}-simple'.format(ALL_BASE_NAMES[-1]))
-      else:
-        file_path_or_dict = os.path.join(
-            get_torch_home(), '{:}.{:}'.format(
-                ALL_BASE_NAMES[-1], PICKLE_EXT))
-      print('{:} Try to use the default NATS-Bench (size) path from '
-            'fast_mode={:} and path={:}.'.format(time_string(), self._fast_mode,
-                                                 file_path_or_dict))
-    if isinstance(file_path_or_dict, str):
-      file_path_or_dict = str(file_path_or_dict)
-      if verbose:
-        print('{:} Try to create the NATS-Bench (size) api '
-              'from {:} with fast_mode={:}'.format(
-                  time_string(), file_path_or_dict, fast_mode))
-      if not nats_is_file(file_path_or_dict) and not nats_is_dir(
-          file_path_or_dict):
-        raise ValueError('{:} is neither a file or a dir.'.format(
-            file_path_or_dict))
-      self.filename = os.path.basename(file_path_or_dict)
-      if fast_mode:
-        if nats_is_file(file_path_or_dict):
-          raise ValueError('fast_mode={:} must feed the path for directory '
-                           ': {:}'.format(fast_mode, file_path_or_dict))
-        else:
-          self._archive_dir = file_path_or_dict
-      else:
-        if nats_is_dir(file_path_or_dict):
-          raise ValueError('fast_mode={:} must feed the path for file '
-                           ': {:}'.format(fast_mode, file_path_or_dict))
-        else:
-          file_path_or_dict = pickle_load(file_path_or_dict)
-    elif isinstance(file_path_or_dict, dict):
-      file_path_or_dict = copy.deepcopy(file_path_or_dict)
-    self.verbose = verbose
-    if isinstance(file_path_or_dict, dict):
-      keys = ('meta_archs', 'arch2infos', 'evaluated_indexes')
-      for key in keys:
-        if key not in file_path_or_dict:
-          raise ValueError('Can not find key[{:}] in the dict'.format(key))
-      self.meta_archs = copy.deepcopy(file_path_or_dict['meta_archs'])
-      # NOTE(xuanyidong): This is a dict mapping each architecture to a dict,
-      # where the key is #epochs and the value is ArchResults
-      self.arch2infos_dict = collections.OrderedDict()
-      self._avaliable_hps = set()
-      for xkey in sorted(list(file_path_or_dict['arch2infos'].keys())):
-        all_infos = file_path_or_dict['arch2infos'][xkey]
-        hp2archres = collections.OrderedDict()
-        for hp_key, results in all_infos.items():
-          hp2archres[hp_key] = ArchResults.create_from_state_dict(results)
-          self._avaliable_hps.add(hp_key)  # save the avaliable hyper-parameter
-        self.arch2infos_dict[xkey] = hp2archres
-      self.evaluated_indexes = set(file_path_or_dict['evaluated_indexes'])
-    elif self.archive_dir is not None:
-      benchmark_meta = pickle_load('{:}/meta.{:}'.format(
-          self.archive_dir, PICKLE_EXT))
-      self.meta_archs = copy.deepcopy(benchmark_meta['meta_archs'])
-      self.arch2infos_dict = collections.OrderedDict()
-      self._avaliable_hps = set()
-      self.evaluated_indexes = set()
-    else:
-      raise ValueError('file_path_or_dict [{:}] must be a dict or archive_dir '
-                       'must be set'.format(type(file_path_or_dict)))
-    self.archstr2index = {}
-    for idx, arch in enumerate(self.meta_archs):
-      if arch in self.archstr2index:
-        raise ValueError('This [{:}]-th arch {:} already in the '
-                         'dict ({:}).'.format(
-                             idx, arch, self.archstr2index[arch]))
-      self.archstr2index[arch] = idx
-    if self.verbose:
-      print('{:} Create NATS-Bench (size) done with {:}/{:} architectures '
-            'avaliable.'.format(time_string(),
-                                len(self.evaluated_indexes),
-                                len(self.meta_archs)))
-
-  def query_info_str_by_arch(self, arch, hp: Text = '12'):
-    """Query the information of a specific architecture.
-
-    Args:
-      arch: it can be an architecture index or an architecture string.
-
-      hp: the hyperparamete indicator, could be 01, 12, or 90. The difference
-          between these three configurations are the number of training epochs.
-
-    Returns:
-      ArchResults instance
-    """
-    if self.verbose:
-      print('{:} Call query_info_str_by_arch with arch={:}'
-            'and hp={:}'.format(time_string(), arch, hp))
-    return self._query_info_str_by_arch(arch, hp, print_information)
-
-  def get_more_info(self,
-                    index,
-                    dataset,
-                    iepoch=None,
-                    hp: Text = '12',
-                    is_random: bool = True):
-    """Return the metric for the `index`-th architecture.
-
-    Args:
-      index: the architecture index.
-      dataset:
-          'cifar10-valid'  : using the proposed train set of CIFAR-10 as the training set
-          'cifar10'        : using the proposed train+valid set of CIFAR-10 as the training set
-          'cifar100'       : using the proposed train set of CIFAR-100 as the training set
-          'ImageNet16-120' : using the proposed train set of ImageNet-16-120 as the training set
-      iepoch: the index of training epochs from 0 to 11/199.
-          When iepoch=None, it will return the metric for the last training epoch
-          When iepoch=11, it will return the metric for the 11-th training epoch (starting from 0)
-      hp: indicates different hyper-parameters for training
-          When hp=01, it trains the network with 01 epochs and the LR decayed from 0.1 to 0 within 01 epochs
-          When hp=12, it trains the network with 01 epochs and the LR decayed from 0.1 to 0 within 12 epochs
-          When hp=90, it trains the network with 01 epochs and the LR decayed from 0.1 to 0 within 90 epochs
-      is_random:
-          When is_random=True, the performance of a random architecture will be returned
-          When is_random=False, the performanceo of all trials will be averaged.
-
-    Returns:
-      a dict, where key is the metric name and value is its value.
-    """
-    if self.verbose:
-      print('{:} Call the get_more_info function with index={:}, dataset={:}, '
-            'iepoch={:}, hp={:}, and is_random={:}.'.format(
-                time_string(), index, dataset, iepoch, hp, is_random))
-    index = self.query_index_by_arch(index)  # To avoid the input is a string or an instance of a arch object
-    self._prepare_info(index)
-    if index not in self.arch2infos_dict:
-      raise ValueError('Did not find {:} from arch2infos_dict.'.format(index))
-    archresult = self.arch2infos_dict[index][str(hp)]
-    # if randomly select one trial, select the seed at first
-    if isinstance(is_random, bool) and is_random:
-      seeds = archresult.get_dataset_seeds(dataset)
-      is_random = random.choice(seeds)
-    # collect the training information
-    train_info = archresult.get_metrics(
-        dataset, 'train', iepoch=iepoch, is_random=is_random)
-    total = train_info['iepoch'] + 1
-    xinfo = {
-        'train-loss': train_info['loss'],
-        'train-accuracy': train_info['accuracy'],
-        'train-per-time': train_info['all_time'] / total,
-        'train-all-time': train_info['all_time']
-    }
-    # collect the evaluation information
-    if dataset == 'cifar10-valid':
-      valid_info = archresult.get_metrics(
-          dataset, 'x-valid', iepoch=iepoch, is_random=is_random)
-      try:
-        test_info = archresult.get_metrics(
-            dataset, 'ori-test', iepoch=iepoch, is_random=is_random)
-      except Exception as unused_e:  # pylint: disable=broad-except
-        test_info = None
-      valtest_info = None
-      xinfo['comment'] = 'In this dict, train-loss/accuracy/time is the metric on the train set of CIFAR-10. The test-loss/accuracy/time is the performance of the CIFAR-10 test set after training on the train set by {:} epochs. The per-time and total-time indicate the per epoch and total time costs, respectively.'.format(hp)
-    else:
-      if dataset == 'cifar10':
-        xinfo['comment'] = 'In this dict, train-loss/accuracy/time is the metric on the train+valid sets of CIFAR-10. The test-loss/accuracy/time is the performance of the CIFAR-10 test set after training on the train+valid sets by {:} epochs. The per-time and total-time indicate the per epoch and total time costs, respectively.'.format(hp)
-      try:  # collect results on the proposed test set
-        if dataset == 'cifar10':
-          test_info = archresult.get_metrics(
-              dataset, 'ori-test', iepoch=iepoch, is_random=is_random)
-        else:
-          test_info = archresult.get_metrics(
-              dataset, 'x-test', iepoch=iepoch, is_random=is_random)
-      except Exception as unused_e:  # pylint: disable=broad-except
-        test_info = None
-      try:  # collect results on the proposed validation set
-        valid_info = archresult.get_metrics(
-            dataset, 'x-valid', iepoch=iepoch, is_random=is_random)
-      except Exception as unused_e:  # pylint: disable=broad-except
-        valid_info = None
-      try:
-        if dataset != 'cifar10':
-          valtest_info = archresult.get_metrics(
-              dataset, 'ori-test', iepoch=iepoch, is_random=is_random)
-        else:
-          valtest_info = None
-      except Exception as unused_e:  # pylint: disable=broad-except
-        valtest_info = None
-    if valid_info is not None:
-      xinfo['valid-loss'] = valid_info['loss']
-      xinfo['valid-accuracy'] = valid_info['accuracy']
-      xinfo['valid-per-time'] = valid_info['all_time'] / total
-      xinfo['valid-all-time'] = valid_info['all_time']
-    if test_info is not None:
-      xinfo['test-loss'] = test_info['loss']
-      xinfo['test-accuracy'] = test_info['accuracy']
-      xinfo['test-per-time'] = test_info['all_time'] / total
-      xinfo['test-all-time'] = test_info['all_time']
-    if valtest_info is not None:
-      xinfo['valtest-loss'] = valtest_info['loss']
-      xinfo['valtest-accuracy'] = valtest_info['accuracy']
-      xinfo['valtest-per-time'] = valtest_info['all_time'] / total
-      xinfo['valtest-all-time'] = valtest_info['all_time']
-    return xinfo
-
-  def show(self, index: int = -1) -> None:
-    """Print the information of a specific (or all) architecture(s)."""
-    self._show(index, print_information)

lib/nats_bench/api_test.py

@@ -1,131 +0,0 @@
-##############################################################################
-# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 ##########################
-##############################################################################
-# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size #
-##############################################################################
-# pytest --capture=tee-sys                                                   #
-##############################################################################
-"""This file is used to quickly test the API."""
-import os
-import pytest
-import random
-
-from nats_bench.api_size import NATSsize
-from nats_bench.api_size import ALL_BASE_NAMES as sss_base_names
-from nats_bench.api_topology import NATStopology
-from nats_bench.api_topology import ALL_BASE_NAMES as tss_base_names
-
-
-def get_fake_torch_home_dir():
-  print('This file is {:}'.format(os.path.abspath(__file__)))
-  print('The current directory is {:}'.format(os.path.abspath(os.getcwd())))
-  xname = 'FAKE_TORCH_HOME'
-  if xname in os.environ:
-    return os.environ['FAKE_TORCH_HOME']
-  else:
-    return os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'fake_torch_dir')
-
-
-class TestNATSBench(object):
-
-  def test_nats_bench_tss(self, benchmark_dir=None, fake_random=True):
-    if benchmark_dir is None:
-      benchmark_dir = os.path.join(get_fake_torch_home_dir(), sss_base_names[-1] + '-simple')
-    return _test_nats_bench(benchmark_dir, True, fake_random)
-
-  def test_nats_bench_sss(self, benchmark_dir=None, fake_random=True):
-    if benchmark_dir is None:
-      benchmark_dir = os.path.join(get_fake_torch_home_dir(), tss_base_names[-1] + '-simple')
-    return _test_nats_bench(benchmark_dir, False, fake_random)
-
-  def prepare_fake_tss(self):
-    print('')
-    tss_benchmark_dir = os.path.join(get_fake_torch_home_dir(), tss_base_names[-1] + '-simple')
-    api = NATStopology(tss_benchmark_dir, True, False)
-    return api
-
-  def test_01_th_issue(self):
-    # Link: https://github.com/D-X-Y/NATS-Bench/issues/1
-    api = self.prepare_fake_tss()
-    # The performance of 0-th architecture on CIFAR-10 (trained by 12 epochs)
-    info = api.get_more_info(0, 'cifar10', hp=12)
-    # First of all, the data split in NATS-Bench is different from that in the official CIFAR paper.
-    # In NATS-Bench, we split the original CIFAR-10 training set into two parts, i.e., a training set and a validation set.
-    # In the following, we will use the splits of NATS-Bench to explain.
-    print(info['comment'])
-    print('The loss on the training + validation sets of CIFAR-10: {:}'.format(info['train-loss']))
-    print('The total training time for 12 epochs on the training + validation sets of CIFAR-10: {:}'.format(info['train-all-time']))
-    print('The per-epoch training time on CIFAR-10: {:}'.format(info['train-per-time']))
-    print('The total evaluation time on the test set of CIFAR-10 for 12 times: {:}'.format(info['test-all-time']))
-    print('The evaluation time on the test set of CIFAR-10: {:}'.format(info['test-per-time']))
-    cost_info = api.get_cost_info(0, 'cifar10')
-    xkeys = ['T-train@epoch',     # The per epoch training time on the training + validation sets of CIFAR-10.
-             'T-train@total',
-             'T-ori-test@epoch',  # The time cost for the evaluation on CIFAR-10 test set.
-             'T-ori-test@total']  # T-ori-test@epoch * 12 times.
-    for xkey in xkeys:
-      print('The cost info [{:}] for 0-th architecture on CIFAR-10 is {:}'.format(xkey, cost_info[xkey]))
-    
-  def test_02_th_issue(self):
-    # https://github.com/D-X-Y/NATS-Bench/issues/2
-    api = self.prepare_fake_tss()
-    data = api.query_by_index(284, dataname='cifar10', hp=200)
-    for xkey, xvalue in data.items():
-      print('{:} : {:}'.format(xkey, xvalue))
-    xinfo = data[777].get_train()
-    print(xinfo)
-    print(data[777].train_acc1es)
-
-    info_012_epochs = api.get_more_info(284, 'cifar10', hp= 12)
-    print('Train accuracy for  12 epochs is {:}'.format(info_012_epochs['train-accuracy']))
-    info_200_epochs = api.get_more_info(284, 'cifar10', hp=200)
-    print('Train accuracy for 200 epochs is {:}'.format(info_200_epochs['train-accuracy']))
- 
-
-def _test_nats_bench(benchmark_dir, is_tss, fake_random, verbose=False):
-  """The main test entry for NATS-Bench."""
-  if is_tss:
-    api = NATStopology(benchmark_dir, True, verbose)
-  else:
-    api = NATSsize(benchmark_dir, True, verbose)
-
-  if fake_random:
-    test_indexes = [0, 11, 284]
-  else:
-    test_indexes = [random.randint(0, len(api) - 1) for _ in range(10)]
-
-  key2dataset = {'cifar10': 'CIFAR-10',
-                 'cifar100': 'CIFAR-100',
-                 'ImageNet16-120': 'ImageNet16-120'}
-
-  for index in test_indexes:
-    print('\n\nEvaluate the {:5d}-th architecture.'.format(index))
-
-    for key, dataset in key2dataset.items():
-      # Query the loss / accuracy / time for the `index`-th candidate
-      #   architecture on CIFAR-10
-      # info is a dict, where you can easily figure out the meaning by key
-      info = api.get_more_info(index, key)
-      print('  -->> The performance on {:}: {:}'.format(dataset, info))
-
-      # Query the flops, params, latency. info is a dict.
-      info = api.get_cost_info(index, key)
-      print('  -->> The cost info on {:}: {:}'.format(dataset, info))
-
-      # Simulate the training of the `index`-th candidate:
-      validation_accuracy, latency, time_cost, current_total_time_cost = api.simulate_train_eval(
-          index, dataset=key, hp='12')
-      print('  -->> The validation accuracy={:}, latency={:}, '
-            'the current time cost={:} s, accumulated time cost={:} s'
-            .format(validation_accuracy, latency, time_cost,
-                    current_total_time_cost))
-
-      # Print the configuration of the `index`-th architecture on CIFAR-10
-      config = api.get_net_config(index, key)
-      print('  -->> The configuration on {:} is {:}'.format(dataset, config))
-
-    # Show the information of the `index`-th architecture
-    api.show(index)
-
-  with pytest.raises(ValueError):
-    api.get_more_info(100000, 'cifar10')

lib/nats_bench/api_topology.py

@@ -1,338 +0,0 @@
-#####################################################
-# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.08 #
-##############################################################################
-# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size #
-##############################################################################
-# The history of benchmark files are as follows,                             #
-# where the format is (the name is NATS-tss-[version]-[md5].pickle.pbz2)     #
-# [2020.08.31] NATS-tss-v1_0-3ffb9.pickle.pbz2                               #
-##############################################################################
-# pylint: disable=line-too-long
-"""The API for topology search space in NATS-Bench."""
-import collections
-import copy
-import os
-import random
-from typing import Any, Dict, List, Optional, Text, Union
-
-from nats_bench.api_utils import ArchResults
-from nats_bench.api_utils import NASBenchMetaAPI
-from nats_bench.api_utils import get_torch_home
-from nats_bench.api_utils import nats_is_dir
-from nats_bench.api_utils import nats_is_file
-from nats_bench.api_utils import PICKLE_EXT
-from nats_bench.api_utils import pickle_load
-from nats_bench.api_utils import time_string
-
-import numpy as np
-
-
-ALL_BASE_NAMES = ['NATS-tss-v1_0-3ffb9']
-
-
-def print_information(information, extra_info=None, show=False):
-  """print out the information of a given ArchResults."""
-  dataset_names = information.get_dataset_names()
-  strings = [
-      information.arch_str,
-      'datasets : {:}, extra-info : {:}'.format(dataset_names, extra_info)
-  ]
-
-  def metric2str(loss, acc):
-    return 'loss = {:.3f} & top1 = {:.2f}%'.format(loss, acc)
-
-  for dataset in dataset_names:
-    metric = information.get_compute_costs(dataset)
-    flop, param, latency = metric['flops'], metric['params'], metric['latency']
-    str1 = '{:14s} FLOP={:6.2f} M, Params={:.3f} MB, latency={:} ms.'.format(
-        dataset, flop, param,
-        '{:.2f}'.format(latency *
-                        1000) if latency is not None and latency > 0 else None)
-    train_info = information.get_metrics(dataset, 'train')
-    if dataset == 'cifar10-valid':
-      valid_info = information.get_metrics(dataset, 'x-valid')
-      str2 = '{:14s} train : [{:}], valid : [{:}]'.format(
-          dataset, metric2str(train_info['loss'], train_info['accuracy']),
-          metric2str(valid_info['loss'], valid_info['accuracy']))
-    elif dataset == 'cifar10':
-      test__info = information.get_metrics(dataset, 'ori-test')
-      str2 = '{:14s} train : [{:}], test  : [{:}]'.format(
-          dataset, metric2str(train_info['loss'], train_info['accuracy']),
-          metric2str(test__info['loss'], test__info['accuracy']))
-    else:
-      valid_info = information.get_metrics(dataset, 'x-valid')
-      test__info = information.get_metrics(dataset, 'x-test')
-      str2 = '{:14s} train : [{:}], valid : [{:}], test : [{:}]'.format(
-          dataset, metric2str(train_info['loss'], train_info['accuracy']),
-          metric2str(valid_info['loss'], valid_info['accuracy']),
-          metric2str(test__info['loss'], test__info['accuracy']))
-    strings += [str1, str2]
-  if show: print('\n'.join(strings))
-  return strings
-
-
-class NATStopology(NASBenchMetaAPI):
-  """This is the class for the API of topology search space in NATS-Bench."""
-
-  def __init__(self,
-               file_path_or_dict: Optional[Union[Text, Dict[Text, Any]]] = None,
-               fast_mode: bool = False,
-               verbose: bool = True):
-    """The initialization function that takes the dataset file path (or a dict loaded from that path) as input."""
-    self._all_base_names = ALL_BASE_NAMES
-    self.filename = None
-    self._search_space_name = 'topology'
-    self._fast_mode = fast_mode
-    self._archive_dir = None
-    self._full_train_epochs = 200
-    self.reset_time()
-    if file_path_or_dict is None:
-      if self._fast_mode:
-        self._archive_dir = os.path.join(
-            get_torch_home(), '{:}-simple'.format(ALL_BASE_NAMES[-1]))
-      else:
-        file_path_or_dict = os.path.join(
-            get_torch_home(), '{:}.{:}'.format(
-                ALL_BASE_NAMES[-1], PICKLE_EXT))
-      print('{:} Try to use the default NATS-Bench (topology) path from '
-            'fast_mode={:} and path={:}.'.format(time_string(), self._fast_mode, file_path_or_dict))
-    if isinstance(file_path_or_dict, str):
-      file_path_or_dict = str(file_path_or_dict)
-      if verbose:
-        print('{:} Try to create the NATS-Bench (topology) api '
-              'from {:} with fast_mode={:}'.format(
-                  time_string(), file_path_or_dict, fast_mode))
-      if not nats_is_file(file_path_or_dict) and not nats_is_dir(
-          file_path_or_dict):
-        raise ValueError('{:} is neither a file or a dir.'.format(
-            file_path_or_dict))
-      self.filename = os.path.basename(file_path_or_dict)
-      if fast_mode:
-        if nats_is_file(file_path_or_dict):
-          raise ValueError('fast_mode={:} must feed the path for directory '
-                           ': {:}'.format(fast_mode, file_path_or_dict))
-        else:
-          self._archive_dir = file_path_or_dict
-      else:
-        if nats_is_dir(file_path_or_dict):
-          raise ValueError('fast_mode={:} must feed the path for file '
-                           ': {:}'.format(fast_mode, file_path_or_dict))
-        else:
-          file_path_or_dict = pickle_load(file_path_or_dict)
-    elif isinstance(file_path_or_dict, dict):
-      file_path_or_dict = copy.deepcopy(file_path_or_dict)
-    self.verbose = verbose
-    if isinstance(file_path_or_dict, dict):
-      keys = ('meta_archs', 'arch2infos', 'evaluated_indexes')
-      for key in keys:
-        if key not in file_path_or_dict:
-          raise ValueError('Can not find key[{:}] in the dict'.format(key))
-      self.meta_archs = copy.deepcopy(file_path_or_dict['meta_archs'])
-      # NOTE(xuanyidong): This is a dict mapping each architecture to a dict,
-      # where the key is #epochs and the value is ArchResults
-      self.arch2infos_dict = collections.OrderedDict()
-      self._avaliable_hps = set()
-      for xkey in sorted(list(file_path_or_dict['arch2infos'].keys())):
-        all_infos = file_path_or_dict['arch2infos'][xkey]
-        hp2archres = collections.OrderedDict()
-        for hp_key, results in all_infos.items():
-          hp2archres[hp_key] = ArchResults.create_from_state_dict(results)
-          self._avaliable_hps.add(hp_key)  # save the avaliable hyper-parameter
-        self.arch2infos_dict[xkey] = hp2archres
-      self.evaluated_indexes = set(file_path_or_dict['evaluated_indexes'])
-    elif self.archive_dir is not None:
-      benchmark_meta = pickle_load('{:}/meta.{:}'.format(
-          self.archive_dir, PICKLE_EXT))
-      self.meta_archs = copy.deepcopy(benchmark_meta['meta_archs'])
-      self.arch2infos_dict = collections.OrderedDict()
-      self._avaliable_hps = set()
-      self.evaluated_indexes = set()
-    else:
-      raise ValueError('file_path_or_dict [{:}] must be a dict or archive_dir '
-                       'must be set'.format(type(file_path_or_dict)))
-    self.archstr2index = {}
-    for idx, arch in enumerate(self.meta_archs):
-      if arch in self.archstr2index:
-        raise ValueError('This [{:}]-th arch {:} already in the '
-                         'dict ({:}).'.format(
-                             idx, arch, self.archstr2index[arch]))
-      self.archstr2index[arch] = idx
-    if self.verbose:
-      print('{:} Create NATS-Bench (topology) done with {:}/{:} architectures '
-            'avaliable.'.format(time_string(),
-                                len(self.evaluated_indexes),
-                                len(self.meta_archs)))
-
-  def query_info_str_by_arch(self, arch, hp: Text = '12'):
-    """Query the information of a specific architecture.
-
-    Args:
-      arch: it can be an architecture index or an architecture string.
-
-      hp: the hyperparamete indicator, could be 12 or 200. The difference
-          between these three configurations are the number of training epochs.
-
-    Returns:
-      ArchResults instance
-    """
-    if self.verbose:
-      print('{:} Call query_info_str_by_arch with arch={:}'
-            'and hp={:}'.format(time_string(), arch, hp))
-    return self._query_info_str_by_arch(arch, hp, print_information)
-
-  def get_more_info(self,
-                    index,
-                    dataset,
-                    iepoch=None,
-                    hp: Text = '12',
-                    is_random: bool = True):
-    """Return the metric for the `index`-th architecture."""
-    if self.verbose:
-      print('{:} Call the get_more_info function with index={:}, dataset={:}, '
-            'iepoch={:}, hp={:}, and is_random={:}.'.format(
-                time_string(), index, dataset, iepoch, hp, is_random))
-    index = self.query_index_by_arch(index)  # To avoid the input is a string or an instance of a arch object
-    self._prepare_info(index)
-    if index not in self.arch2infos_dict:
-      raise ValueError('Did not find {:} from arch2infos_dict.'.format(index))
-    archresult = self.arch2infos_dict[index][str(hp)]
-    # if randomly select one trial, select the seed at first
-    if isinstance(is_random, bool) and is_random:
-      seeds = archresult.get_dataset_seeds(dataset)
-      is_random = random.choice(seeds)
-    # collect the training information
-    train_info = archresult.get_metrics(dataset, 'train', iepoch=iepoch, is_random=is_random)
-    total = train_info['iepoch'] + 1
-    xinfo = {
-        'train-loss':
-            train_info['loss'],
-        'train-accuracy':
-            train_info['accuracy'],
-        'train-per-time':
-            train_info['all_time'] /
-            total if train_info['all_time'] is not None else None,
-        'train-all-time':
-            train_info['all_time']
-    }
-    # collect the evaluation information
-    if dataset == 'cifar10-valid':
-      valid_info = archresult.get_metrics(dataset, 'x-valid', iepoch=iepoch, is_random=is_random)
-      try:
-        test_info = archresult.get_metrics(dataset, 'ori-test', iepoch=iepoch, is_random=is_random)
-      except Exception as unused_e:  # pylint: disable=broad-except
-        test_info = None
-      valtest_info = None
-      xinfo['comment'] = 'In this dict, train-loss/accuracy/time is the metric on the train set of CIFAR-10. The test-loss/accuracy/time is the performance of the CIFAR-10 test set after training on the train set by {:} epochs. The per-time and total-time indicate the per epoch and total time costs, respectively.'.format(hp)
-    else:
-      if dataset == 'cifar10':
-        xinfo['comment'] = 'In this dict, train-loss/accuracy/time is the metric on the train+valid sets of CIFAR-10. The test-loss/accuracy/time is the performance of the CIFAR-10 test set after training on the train+valid sets by {:} epochs. The per-time and total-time indicate the per epoch and total time costs, respectively.'.format(hp)
-      try:  # collect results on the proposed test set
-        if dataset == 'cifar10':
-          test_info = archresult.get_metrics(dataset, 'ori-test', iepoch=iepoch, is_random=is_random)
-        else:
-          test_info = archresult.get_metrics(dataset, 'x-test', iepoch=iepoch, is_random=is_random)
-      except Exception as unused_e:  # pylint: disable=broad-except
-        test_info = None
-      try:  # collect results on the proposed validation set
-        valid_info = archresult.get_metrics(dataset, 'x-valid', iepoch=iepoch, is_random=is_random)
-      except Exception as unused_e:  # pylint: disable=broad-except
-        valid_info = None
-      try:
-        if dataset != 'cifar10':
-          valtest_info = archresult.get_metrics(dataset, 'ori-test', iepoch=iepoch, is_random=is_random)
-        else:
-          valtest_info = None
-      except Exception as unused_e:  # pylint: disable=broad-except
-        valtest_info = None
-    if valid_info is not None:
-      xinfo['valid-loss'] = valid_info['loss']
-      xinfo['valid-accuracy'] = valid_info['accuracy']
-      xinfo['valid-per-time'] = valid_info['all_time'] / total if valid_info['all_time'] is not None else None
-      xinfo['valid-all-time'] = valid_info['all_time']
-    if test_info is not None:
-      xinfo['test-loss'] = test_info['loss']
-      xinfo['test-accuracy'] = test_info['accuracy']
-      xinfo['test-per-time'] = test_info['all_time'] / total if test_info['all_time'] is not None else None
-      xinfo['test-all-time'] = test_info['all_time']
-    if valtest_info is not None:
-      xinfo['valtest-loss'] = valtest_info['loss']
-      xinfo['valtest-accuracy'] = valtest_info['accuracy']
-      xinfo['valtest-per-time'] = valtest_info['all_time'] / total if valtest_info['all_time'] is not None else None
-      xinfo['valtest-all-time'] = valtest_info['all_time']
-    return xinfo
-
-  def show(self, index: int = -1) -> None:
-    """This function will print the information of a specific (or all) architecture(s)."""
-    self._show(index, print_information)
-
-  @staticmethod
-  def str2lists(arch_str: Text) -> List[Any]:
-    """Shows how to read the string-based architecture encoding.
-
-    Args:
-      arch_str: the input is a string indicates the architecture topology, such as
-                    |nor_conv_1x1~0|+|none~0|none~1|+|none~0|none~1|skip_connect~2|
-    Returns:
-      a list of tuple, contains multiple (op, input_node_index) pairs.
-
-    [USAGE]
-    It is the same as the `str2structure` func in AutoDL-Projects:
-      `github.com/D-X-Y/AutoDL-Projects/lib/models/cell_searchs/genotypes.py`
-    ```
-      arch = api.str2lists( '|nor_conv_1x1~0|+|none~0|none~1|+|none~0|none~1|skip_connect~2|' )
-      print ('there are {:} nodes in this arch'.format(len(arch)+1)) # arch is a list
-      for i, node in enumerate(arch):
-        print('the {:}-th node is the sum of these {:} nodes with op: {:}'.format(i+1, len(node), node))
-    ```
-    """
-    node_strs = arch_str.split('+')
-    genotypes = []
-    for unused_i, node_str in enumerate(node_strs):
-      inputs = list(filter(lambda x: x != '', node_str.split('|')))  # pylint: disable=g-explicit-bool-comparison
-      for xinput in inputs:
-        assert len(
-            xinput.split('~')) == 2, 'invalid input length : {:}'.format(xinput)
-      inputs = (xi.split('~') for xi in inputs)
-      input_infos = tuple((op, int(idx)) for (op, idx) in inputs)
-      genotypes.append(input_infos)
-    return genotypes
-
-  @staticmethod
-  def str2matrix(arch_str: Text,
-                 search_space: List[Text] = ('none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3')) -> np.ndarray:
-    """Convert the string-based architecture encoding to the encoding strategy in NAS-Bench-101.
-
-    Args:
-      arch_str: the input is a string indicates the architecture topology, such as
-                    |nor_conv_1x1~0|+|none~0|none~1|+|none~0|none~1|skip_connect~2|
-      search_space: a list of operation string, the default list is the topology search space for NATS-BENCH.
-        the default value should be be consistent with this line https://github.com/D-X-Y/AutoDL-Projects/blob/main/lib/models/cell_operations.py#L24
-
-    Returns:
-      the numpy matrix (2-D np.ndarray) representing the DAG of this architecture topology
-
-    [USAGE]
-      matrix = api.str2matrix( '|nor_conv_1x1~0|+|none~0|none~1|+|none~0|none~1|skip_connect~2|' )
-      This matrix is 4-by-4 matrix representing a cell with 4 nodes (only the lower left triangle is useful).
-         [ [0, 0, 0, 0],  # the first line represents the input (0-th) node
-           [2, 0, 0, 0],  # the second line represents the 1-st node, is calculated by 2-th-op( 0-th-node )
-           [0, 0, 0, 0],  # the third line represents the 2-nd node, is calculated by 0-th-op( 0-th-node ) + 0-th-op( 1-th-node )
-           [0, 0, 1, 0] ] # the fourth line represents the 3-rd node, is calculated by 0-th-op( 0-th-node ) + 0-th-op( 1-th-node ) + 1-th-op( 2-th-node )
-      In the topology search space in NATS-BENCH, 0-th-op is 'none', 1-th-op is 'skip_connect',
-         2-th-op is 'nor_conv_1x1', 3-th-op is 'nor_conv_3x3', 4-th-op is 'avg_pool_3x3'.
-    [NOTE]
-      If a node has two input-edges from the same node, this function does not work. One edge will be overlapped.
-    """
-    node_strs = arch_str.split('+')
-    num_nodes = len(node_strs) + 1
-    matrix = np.zeros((num_nodes, num_nodes))
-    for i, node_str in enumerate(node_strs):
-      inputs = list(filter(lambda x: x != '', node_str.split('|')))  # pylint: disable=g-explicit-bool-comparison
-      for xinput in inputs:
-        assert len(xinput.split('~')) == 2, 'invalid input length : {:}'.format(xinput)
-      for xi in inputs:
-        op, idx = xi.split('~')
-        if op not in search_space: raise ValueError('this op ({:}) is not in {:}'.format(op, search_space))
-        op_idx, node_idx = search_space.index(op), int(idx)
-        matrix[i+1, node_idx] = op_idx
-    return matrix

lib/spaces/__init__.py

@@ -0,0 +1 @@
+#