From 5456939d81fa677de08553566c37b2c4e5ede2d6 Mon Sep 17 00:00:00 2001
From: D-X-Y <280835372@qq.com>
Date: Thu, 26 Nov 2020 14:43:28 +0800
Subject: [PATCH] Update time_budget for NATS (algos)

---
 exps/NATS-Bench/Analyze-time.py    | 48 ++++++++++++++++++++++
 exps/NATS-algos/bohb.py            |  3 +-
 exps/NATS-algos/random_wo_share.py |  3 +-
 exps/NATS-algos/regularized_ea.py  |  3 +-
 exps/NATS-algos/reinforce.py       |  3 +-
 exps/NATS-algos/run-all.sh         | 65 +++++++++++++++++++++++-------
 6 files changed, 106 insertions(+), 19 deletions(-)
 create mode 100644 exps/NATS-Bench/Analyze-time.py

diff --git a/exps/NATS-Bench/Analyze-time.py b/exps/NATS-Bench/Analyze-time.py
new file mode 100644
index 0000000..e1517e3
--- /dev/null
+++ b/exps/NATS-Bench/Analyze-time.py
@@ -0,0 +1,48 @@
+##############################################################################
+# NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size #
+##############################################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.07                          #
+##############################################################################
+# python ./exps/NATS-Bench/Analyze-time.py                                   #
+##############################################################################
+import os, sys, time, tqdm, torch, random, argparse
+from typing import List, Text, Dict, Any
+from PIL     import ImageFile
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+from copy    import deepcopy
+from pathlib import Path
+
+lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve()
+if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir))
+from config_utils import dict2config, load_config
+from datasets import get_datasets
+from nats_bench import create
+
+
+def show_time(api):
+  print('Show the time for {:} with 12-epoch-training'.format(api))
+  all_cifar10_time, all_cifar100_time, all_imagenet_time = 0, 0, 0
+  for index in tqdm.tqdm(range(len(api))):
+    info = api.get_more_info(index, 'ImageNet16-120', hp='12')
+    imagenet_time = info['train-all-time']
+    info = api.get_more_info(index, 'cifar10-valid', hp='12')
+    cifar10_time = info['train-all-time']
+    info = api.get_more_info(index, 'cifar100', hp='12')
+    cifar100_time = info['train-all-time']
+    # accumulate the time
+    all_cifar10_time += cifar10_time
+    all_cifar100_time += cifar100_time
+    all_imagenet_time += imagenet_time
+  print('The total training time for CIFAR-10        (held-out train set) is {:} seconds'.format(all_cifar10_time))
+  print('The total training time for CIFAR-100       (held-out train set) is {:} seconds, {:.2f} times longer than that on CIFAR-10'.format(all_cifar100_time, all_cifar100_time / all_cifar10_time))
+  print('The total training time for ImageNet-16-120 (held-out train set) is {:} seconds, {:.2f} times longer than that on CIFAR-10'.format(all_imagenet_time, all_imagenet_time / all_cifar10_time))
+
+
+if __name__ == '__main__':
+
+  api_nats_tss = create(None, 'tss', fast_mode=True, verbose=False)
+  show_time(api_nats_tss)
+
+  api_nats_sss = create(None, 'sss', fast_mode=True, verbose=False)
+  show_time(api_nats_sss)
+
diff --git a/exps/NATS-algos/bohb.py b/exps/NATS-algos/bohb.py
index 50df9f9..0d9479f 100644
--- a/exps/NATS-algos/bohb.py
+++ b/exps/NATS-algos/bohb.py
@@ -169,7 +169,8 @@ if __name__ == '__main__':
   
   api = create(None, args.search_space, fast_mode=True, verbose=False)
 
-  args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space), args.dataset, 'BOHB')
+  args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space),
+                               '{:}-T{:}'.format(args.dataset, args.time_budget), 'BOHB')
   print('save-dir : {:}'.format(args.save_dir))
 
   if args.rand_seed < 0:
diff --git a/exps/NATS-algos/random_wo_share.py b/exps/NATS-algos/random_wo_share.py
index 275eaa6..efe03f3 100644
--- a/exps/NATS-algos/random_wo_share.py
+++ b/exps/NATS-algos/random_wo_share.py
@@ -73,7 +73,8 @@ if __name__ == '__main__':
   
   api = create(None, args.search_space, fast_mode=True, verbose=False)
 
-  args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space), args.dataset, 'RANDOM')
+  args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space),
+                               '{:}-T{:}'.format(args.dataset, args.time_budget), 'RANDOM')
   print('save-dir : {:}'.format(args.save_dir))
 
   if args.rand_seed < 0:
diff --git a/exps/NATS-algos/regularized_ea.py b/exps/NATS-algos/regularized_ea.py
index d01d7f1..2af14f2 100644
--- a/exps/NATS-algos/regularized_ea.py
+++ b/exps/NATS-algos/regularized_ea.py
@@ -200,7 +200,8 @@ if __name__ == '__main__':
 
   api = create(None, args.search_space, fast_mode=True, verbose=False)
 
-  args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space), args.dataset, 'R-EA-SS{:}'.format(args.ea_sample_size))
+  args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space),
+                               '{:}-T{:}'.format(args.dataset, args.time_budget), 'R-EA-SS{:}'.format(args.ea_sample_size))
   print('save-dir : {:}'.format(args.save_dir))
   print('xargs : {:}'.format(args))
 
diff --git a/exps/NATS-algos/reinforce.py b/exps/NATS-algos/reinforce.py
index da09e99..91778dc 100644
--- a/exps/NATS-algos/reinforce.py
+++ b/exps/NATS-algos/reinforce.py
@@ -194,7 +194,8 @@ if __name__ == '__main__':
 
   api = create(None, args.search_space, fast_mode=True, verbose=False)
 
-  args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space), args.dataset, 'REINFORCE-{:}'.format(args.learning_rate))
+  args.save_dir = os.path.join('{:}-{:}'.format(args.save_dir, args.search_space),
+                               '{:}-T{:}'.format(args.dataset, args.time_budget), 'REINFORCE-{:}'.format(args.learning_rate))
   print('save-dir : {:}'.format(args.save_dir))
 
   if args.rand_seed < 0:
diff --git a/exps/NATS-algos/run-all.sh b/exps/NATS-algos/run-all.sh
index 69b540a..d845cff 100644
--- a/exps/NATS-algos/run-all.sh
+++ b/exps/NATS-algos/run-all.sh
@@ -10,26 +10,61 @@ if [ "$#" -ne 1 ] ;then
   exit 1
 fi
 
-
-datasets="cifar10 cifar100 ImageNet16-120"
 alg_type=$1
 
 if [ "$alg_type" == "mul" ]; then
-  search_spaces="tss sss"
+  # datasets="cifar10 cifar100 ImageNet16-120"
+  # The topology search space
+  dataset="cifar10"
+  search_space="tss"
+  time_budget="20000"
+  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
+  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
+  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
+  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
 
-  for dataset in ${datasets}
-  do
-    for search_space in ${search_spaces}
-    do
-      python ./exps/NATS-algos/reinforce.py --dataset ${dataset} --search_space ${search_space} --learning_rate 0.01
-      python ./exps/NATS-algos/regularized_ea.py --dataset ${dataset} --search_space ${search_space} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
-      python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space}
-      python ./exps/NATS-algos/bohb.py --dataset ${dataset} --search_space ${search_space} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
-    done
-  done
+  dataset="cifar100"
+  search_space="tss"
+  time_budget="40000"
+  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
+  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
+  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
+  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
 
-  python exps/experimental/vis-bench-algos.py --search_space tss
-  python exps/experimental/vis-bench-algos.py --search_space sss
+  dataset="ImageNet16-120"
+  search_space="tss"
+  time_budget="120000"
+  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
+  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
+  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space}
+  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
+
+  # The size search space
+  dataset="cifar10"
+  search_space="sss"
+  time_budget="20000"
+  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
+  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
+  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
+  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
+
+  dataset="cifar100"
+  search_space="sss"
+  time_budget="40000"
+  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
+  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
+  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
+  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
+
+  dataset="ImageNet16-120"
+  search_space="tss"
+  time_budget="60000"
+  python ./exps/NATS-algos/reinforce.py       --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --learning_rate 0.01
+  python ./exps/NATS-algos/regularized_ea.py  --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --ea_cycles 200 --ea_population 10 --ea_sample_size 3
+  python ./exps/NATS-algos/random_wo_share.py --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget}
+  python ./exps/NATS-algos/bohb.py            --dataset ${dataset} --search_space ${search_space} --time_budget ${time_budget} --num_samples 4 --random_fraction 0.0 --bandwidth_factor 3
+  # python exps/experimental/vis-bench-algos.py --search_space tss
+  # python exps/experimental/vis-bench-algos.py --search_space sss
 else
   seeds="777 888 999"
   algos="darts-v1 darts-v2 gdas setn random enas"