From 6facc39a42db38e664a580aa89865ddc65edfcf3 Mon Sep 17 00:00:00 2001 From: D-X-Y <280835372@qq.com> Date: Sun, 5 Jul 2020 22:29:26 +0000 Subject: [PATCH] Update weight watcher codes --- docs/NAS-Bench-201.md | 3 +- exps/NAS-Bench-201/test-nas-api-vis.py | 13 +- exps/NAS-Bench-201/test-weights.py | 113 -------------- exps/experimental/test-api.py | 20 --- exps/experimental/test-ww-bench.py | 151 +++++++++++++++++++ lib/nas_201_api/api_201.py | 1 + lib/nas_201_api/api_301.py | 2 + lib/nas_201_api/api_utils.py | 4 + scripts-search/NAS-Bench-201/test-weights.sh | 21 --- 9 files changed, 167 insertions(+), 161 deletions(-) delete mode 100644 exps/NAS-Bench-201/test-weights.py delete mode 100644 exps/experimental/test-api.py create mode 100644 exps/experimental/test-ww-bench.py delete mode 100644 scripts-search/NAS-Bench-201/test-weights.sh diff --git a/docs/NAS-Bench-201.md b/docs/NAS-Bench-201.md index 13800ba..d4325fc 100644 --- a/docs/NAS-Bench-201.md +++ b/docs/NAS-Bench-201.md @@ -46,7 +46,8 @@ It is recommended to put these data into `$TORCH_HOME` (`~/.torch/` by default). ``` from nas_201_api import NASBench201API as API api = API('$path_to_meta_nas_bench_file') -api = API('NAS-Bench-201-v1_1-096897.pth') +# Create an API without the verbose log +api = API('NAS-Bench-201-v1_1-096897.pth', verbose=False) # The default path for benchmark file is '{:}/{:}'.format(os.environ['TORCH_HOME'], 'NAS-Bench-201-v1_1-096897.pth') api = API(None) ``` diff --git a/exps/NAS-Bench-201/test-nas-api-vis.py b/exps/NAS-Bench-201/test-nas-api-vis.py index 98ef293..02668fe 100644 --- a/exps/NAS-Bench-201/test-nas-api-vis.py +++ b/exps/NAS-Bench-201/test-nas-api-vis.py @@ -90,9 +90,9 @@ def visualize_sss_info(api, dataset, vis_save_dir): print ('Do not find cache file : {:}'.format(cache_file_path)) params, flops, train_accs, valid_accs, test_accs = [], [], [], [], [] for index in range(len(api)): - info = api.get_cost_info(index, dataset) - params.append(info['params']) - flops.append(info['flops']) + cost_info = api.get_cost_info(index, dataset, hp='90') + params.append(cost_info['params']) + flops.append(cost_info['flops']) # accuracy info = api.get_more_info(index, dataset, hp='90', is_random=False) train_accs.append(info['train-accuracy']) @@ -178,9 +178,9 @@ def visualize_tss_info(api, dataset, vis_save_dir): print ('Do not find cache file : {:}'.format(cache_file_path)) params, flops, train_accs, valid_accs, test_accs = [], [], [], [], [] for index in range(len(api)): - info = api.get_cost_info(index, dataset) - params.append(info['params']) - flops.append(info['flops']) + cost_info = api.get_cost_info(index, dataset, hp='12') + params.append(cost_info['params']) + flops.append(cost_info['flops']) # accuracy info = api.get_more_info(index, dataset, hp='200', is_random=False) train_accs.append(info['train-accuracy']) @@ -190,6 +190,7 @@ def visualize_tss_info(api, dataset, vis_save_dir): valid_accs.append(info['valid-accuracy']) else: valid_accs.append(info['valid-accuracy']) + print('') info = {'params': params, 'flops': flops, 'train_accs': train_accs, 'valid_accs': valid_accs, 'test_accs': test_accs} torch.save(info, cache_file_path) else: diff --git a/exps/NAS-Bench-201/test-weights.py b/exps/NAS-Bench-201/test-weights.py deleted file mode 100644 index fdaed79..0000000 --- a/exps/NAS-Bench-201/test-weights.py +++ /dev/null @@ -1,113 +0,0 @@ -##################################################### -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # -############################################################################################### -# Before run these commands, the files must be properly put. -# python exps/NAS-Bench-201/test-weights.py --base_path $HOME/.torch/NAS-Bench-201-v1_0-e61699 -# python exps/NAS-Bench-201/test-weights.py --base_path $HOME/.torch/NAS-Bench-201-v1_1-096897 --dataset cifar10-valid --use_12 1 --use_valid 1 -# bash ./scripts-search/NAS-Bench-201/test-weights.sh cifar10-valid 1 -############################################################################################### -import os, gc, sys, math, argparse, psutil -import numpy as np -import torch -from pathlib import Path -from collections import OrderedDict -lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve() -if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) -from nas_201_api import NASBench201API as API -from log_utils import time_string -from models import get_cell_based_tiny_net -from utils import weight_watcher - - -def get_cor(A, B): - return float(np.corrcoef(A, B)[0,1]) - - -def tostr(accdict, norms): - xstr = [] - for key, accs in accdict.items(): - cor = get_cor(accs, norms) - xstr.append('{:}: {:.3f}'.format(key, cor)) - return ' '.join(xstr) - - -def evaluate(api, weight_dir, data: str, use_12epochs_result: bool): - print('\nEvaluate dataset={:}'.format(data)) - norms, process = [], psutil.Process(os.getpid()) - final_val_accs = OrderedDict({'cifar10': [], 'cifar100': [], 'ImageNet16-120': []}) - final_test_accs = OrderedDict({'cifar10': [], 'cifar100': [], 'ImageNet16-120': []}) - for idx in range(len(api)): - # info = api.get_more_info(idx, data, use_12epochs_result=use_12epochs_result, is_random=False) - for key in ['cifar10-valid', 'cifar10', 'cifar100', 'ImageNet16-120']: - info = api.get_more_info(idx, key, use_12epochs_result=False, is_random=False) - if key == 'cifar10-valid': - final_val_accs['cifar10'].append(info['valid-accuracy']) - elif key == 'cifar10': - final_test_accs['cifar10'].append(info['test-accuracy']) - else: - final_test_accs[key].append(info['test-accuracy']) - final_val_accs[key].append(info['valid-accuracy']) - config = api.get_net_config(idx, data) - net = get_cell_based_tiny_net(config) - api.reload(weight_dir, idx) - params = api.get_net_param(idx, data, None, use_12epochs_result=use_12epochs_result) - cur_norms = [] - for seed, param in params.items(): - with torch.no_grad(): - net.load_state_dict(param) - _, summary = weight_watcher.analyze(net, alphas=False) - cur_norms.append(-summary['lognorm']) - cur_norm = float(np.mean(cur_norms)) - if math.isnan(cur_norm): - print (' IGNORE {:} due to nan.'.format(idx)) - continue - norms.append(cur_norm) - api.clear_params(idx, None) - if idx % 200 == 199 or idx + 1 == len(api): - head = '{:05d}/{:05d}'.format(idx, len(api)) - stem_val = tostr(final_val_accs, norms) - stem_test = tostr(final_test_accs, norms) - print('{:} {:} {:} with {:} epochs ({:.2f} MB memory)'.format(time_string(), head, data, 12 if use_12epochs_result else 200, process.memory_info().rss / 1e6)) - print(' [Valid] -->> {:}'.format(stem_val)) - print(' [Test.] -->> {:}'.format(stem_test)) - gc.collect() - - -def main(meta_file: str, weight_dir, save_dir, xdata, use_12epochs_result): - api = API(meta_file) - datasets = ['cifar10-valid', 'cifar10', 'cifar100', 'ImageNet16-120'] - print(time_string() + ' ' + '='*50) - for data in datasets: - nums = api.statistics(data, True) - total = sum([k*v for k, v in nums.items()]) - print('Using 012 epochs, trained on {:20s} : {:} trials in total ({:}).'.format(data, total, nums)) - print(time_string() + ' ' + '='*50) - for data in datasets: - nums = api.statistics(data, False) - total = sum([k*v for k, v in nums.items()]) - print('Using 200 epochs, trained on {:20s} : {:} trials in total ({:}).'.format(data, total, nums)) - print(time_string() + ' ' + '='*50) - - #evaluate(api, weight_dir, 'cifar10-valid', False, True) - evaluate(api, weight_dir, xdata, use_12epochs_result) - - print('{:} finish this test.'.format(time_string())) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser("Analysis of NAS-Bench-201") - parser.add_argument('--save_dir', type=str, default='./output/search-cell-nas-bench-201/visuals', help='The base-name of folder to save checkpoints and log.') - parser.add_argument('--base_path', type=str, default=None, help='The path to the NAS-Bench-201 benchmark file and weight dir.') - parser.add_argument('--dataset' , type=str, default=None, help='.') - parser.add_argument('--use_12' , type=int, default=None, help='.') - args = parser.parse_args() - - save_dir = Path(args.save_dir) - save_dir.mkdir(parents=True, exist_ok=True) - meta_file = Path(args.base_path + '.pth') - weight_dir = Path(args.base_path + '-archive') - assert meta_file.exists(), 'invalid path for api : {:}'.format(meta_file) - assert weight_dir.exists() and weight_dir.is_dir(), 'invalid path for weight dir : {:}'.format(weight_dir) - - main(str(meta_file), weight_dir, save_dir, args.dataset, bool(args.use_12)) - diff --git a/exps/experimental/test-api.py b/exps/experimental/test-api.py deleted file mode 100644 index e6c25cd..0000000 --- a/exps/experimental/test-api.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# exps/experimental/test-api.py -# -import sys, time, random, argparse -from copy import deepcopy -import torchvision.models as models -from pathlib import Path -lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve() -if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) - -from nas_201_api import NASBench201API as API - - -def main(): - api = API(None) - info = api.get_more_info(100, 'cifar100', 199, False, True) - - -if __name__ == '__main__': - main() diff --git a/exps/experimental/test-ww-bench.py b/exps/experimental/test-ww-bench.py new file mode 100644 index 0000000..570c10c --- /dev/null +++ b/exps/experimental/test-ww-bench.py @@ -0,0 +1,151 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +########################################################################################################################################################### +# Before run these commands, the files must be properly put. +# +# python exps/experimental/test-ww-bench.py --base_path $HOME/.torch/NAS-Bench-201-v1_0-e61699 +# python exps/experimental/test-ww-bench.py --base_path $HOME/.torch/NAS-Bench-201-v1_1-096897 --dataset cifar10-valid --use_12 1 --use_valid 1 +# CUDA_VISIBLE_DEVICES='' OMP_NUM_THREADS=4 python exps/experimental/test-ww-bench.py --base_path $HOME/.torch/NAS-Bench-201-v1_1-096897 --dataset cifar10 +# CUDA_VISIBLE_DEVICES='' OMP_NUM_THREADS=4 python exps/experimental/test-ww-bench.py --search_space sss --base_path $HOME/.torch/NAS-Bench-301-v1_0 --dataset cifar10 +# CUDA_VISIBLE_DEVICES='' OMP_NUM_THREADS=4 python exps/experimental/test-ww-bench.py --search_space sss --base_path $HOME/.torch/NAS-Bench-301-v1_0 --dataset cifar100 +# CUDA_VISIBLE_DEVICES='' OMP_NUM_THREADS=4 python exps/experimental/test-ww-bench.py --search_space sss --base_path $HOME/.torch/NAS-Bench-301-v1_0 --dataset ImageNet16-120 +########################################################################################################################################################### +import os, gc, sys, math, argparse, psutil +import numpy as np +import torch +from pathlib import Path +from collections import OrderedDict +import matplotlib +import seaborn as sns +matplotlib.use('agg') +import matplotlib.pyplot as plt +lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve() +if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) +from nas_201_api import NASBench201API, NASBench301API +from log_utils import time_string +from models import get_cell_based_tiny_net +from utils import weight_watcher + + +""" +def get_cor(A, B): + return float(np.corrcoef(A, B)[0,1]) + + +def tostr(accdict, norms): + xstr = [] + for key, accs in accdict.items(): + cor = get_cor(accs, norms) + xstr.append('{:}: {:.3f}'.format(key, cor)) + return ' '.join(xstr) +""" + +def evaluate(api, weight_dir, data: str): + print('\nEvaluate dataset={:}'.format(data)) + process = psutil.Process(os.getpid()) + norms, accuracies = [], [] + ok, total = 0, 5000 + for idx in range(total): + arch_index = api.random() + api.reload(weight_dir, arch_index) + # compute the weight watcher results + config = api.get_net_config(arch_index, data) + net = get_cell_based_tiny_net(config) + meta_info = api.query_meta_info_by_index(arch_index, hp='200' if isinstance(api, NASBench201API) else '90') + params = meta_info.get_net_param(data, 777) + with torch.no_grad(): + net.load_state_dict(params) + _, summary = weight_watcher.analyze(net, alphas=False) + if 'lognorm' not in summary: + api.clear_params(arch_index, None) + del net ; continue + continue + cur_norm = -summary['lognorm'] + api.clear_params(arch_index, None) + if math.isnan(cur_norm): + del net, meta_info + continue + else: + ok += 1 + norms.append(cur_norm) + # query the accuracy + info = meta_info.get_metrics(data, 'ori-test', iepoch=None, is_random=777) + accuracies.append(info['accuracy']) + del net, meta_info + # print the information + if idx % 20 == 0: + gc.collect() + print('{:} {:04d}_{:04d}/{:04d} ({:.2f} MB memory)'.format(time_string(), ok, idx, total, process.memory_info().rss / 1e6)) + return norms, accuracies + + +def main(search_space, meta_file: str, weight_dir, save_dir, xdata): + API = NASBench201API if search_space == 'tss' else NASBench301API + save_dir.mkdir(parents=True, exist_ok=True) + api = API(meta_file, verbose=False) + datasets = ['cifar10-valid', 'cifar10', 'cifar100', 'ImageNet16-120'] + print(time_string() + ' ' + '='*50) + for data in datasets: + hps = api.avaliable_hps + for hp in hps: + nums = api.statistics(data, hp=hp) + total = sum([k*v for k, v in nums.items()]) + print('Using {:3s} epochs, trained on {:20s} : {:} trials in total ({:}).'.format(hp, data, total, nums)) + print(time_string() + ' ' + '='*50) + + norms, accuracies = evaluate(api, weight_dir, xdata) + + indexes = list(range(len(norms))) + norm_indexes = sorted(indexes, key=lambda i: norms[i]) + accy_indexes = sorted(indexes, key=lambda i: accuracies[i]) + labels = [] + for index in norm_indexes: + labels.append(accy_indexes.index(index)) + + dpi, width, height = 200, 1400, 800 + figsize = width / float(dpi), height / float(dpi) + LabelSize, LegendFontsize = 18, 12 + resnet_scale, resnet_alpha = 120, 0.5 + + fig = plt.figure(figsize=figsize) + ax = fig.add_subplot(111) + plt.xlim(min(indexes), max(indexes)) + plt.ylim(min(indexes), max(indexes)) + # plt.ylabel('y').set_rotation(30) + plt.yticks(np.arange(min(indexes), max(indexes), max(indexes)//3), fontsize=LegendFontsize, rotation='vertical') + plt.xticks(np.arange(min(indexes), max(indexes), max(indexes)//5), fontsize=LegendFontsize) + ax.scatter(indexes, labels , marker='*', s=0.5, c='tab:red' , alpha=0.8) + ax.scatter(indexes, indexes, marker='o', s=0.5, c='tab:blue' , alpha=0.8) + ax.scatter([-1], [-1], marker='o', s=100, c='tab:blue' , label='Test accuracy') + ax.scatter([-1], [-1], marker='*', s=100, c='tab:red' , label='Weight watcher') + plt.grid(zorder=0) + ax.set_axisbelow(True) + plt.legend(loc=0, fontsize=LegendFontsize) + ax.set_xlabel('architecture ranking sorted by the test accuracy ', fontsize=LabelSize) + ax.set_ylabel('architecture ranking computed by weight watcher', fontsize=LabelSize) + save_path = (save_dir / '{:}-{:}-test-ww.pdf'.format(search_space, xdata)).resolve() + fig.savefig(save_path, dpi=dpi, bbox_inches='tight', format='pdf') + save_path = (save_dir / '{:}-{:}-test-ww.png'.format(search_space, xdata)).resolve() + fig.savefig(save_path, dpi=dpi, bbox_inches='tight', format='png') + print ('{:} save into {:}'.format(time_string(), save_path)) + + print('{:} finish this test.'.format(time_string())) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser("Analysis of NAS-Bench-201") + parser.add_argument('--save_dir', type=str, default='./output/vis-nas-bench/', help='The base-name of folder to save checkpoints and log.') + parser.add_argument('--search_space', type=str, default=None, choices=['tss', 'sss'], help='The search space.') + parser.add_argument('--base_path', type=str, default=None, help='The path to the NAS-Bench-201 benchmark file and weight dir.') + parser.add_argument('--dataset' , type=str, default=None, help='.') + args = parser.parse_args() + + save_dir = Path(args.save_dir) + save_dir.mkdir(parents=True, exist_ok=True) + meta_file = Path(args.base_path + '.pth') + weight_dir = Path(args.base_path + '-archive') + assert meta_file.exists(), 'invalid path for api : {:}'.format(meta_file) + assert weight_dir.exists() and weight_dir.is_dir(), 'invalid path for weight dir : {:}'.format(weight_dir) + + main(args.search_space, str(meta_file), weight_dir, save_dir, args.dataset) + diff --git a/lib/nas_201_api/api_201.py b/lib/nas_201_api/api_201.py index 257fc30..02159c0 100644 --- a/lib/nas_201_api/api_201.py +++ b/lib/nas_201_api/api_201.py @@ -77,6 +77,7 @@ class NASBench201API(NASBenchMetaAPI): self.meta_archs = copy.deepcopy( file_path_or_dict['meta_archs'] ) # This is a dict mapping each architecture to a dict, where the key is #epochs and the value is ArchResults self.arch2infos_dict = OrderedDict() + self._avaliable_hps = set(['12', '200']) for xkey in sorted(list(file_path_or_dict['arch2infos'].keys())): all_info = file_path_or_dict['arch2infos'][xkey] hp2archres = OrderedDict() diff --git a/lib/nas_201_api/api_301.py b/lib/nas_201_api/api_301.py index 1d85b6b..b076a3a 100644 --- a/lib/nas_201_api/api_301.py +++ b/lib/nas_201_api/api_301.py @@ -75,11 +75,13 @@ class NASBench301API(NASBenchMetaAPI): self.meta_archs = copy.deepcopy( file_path_or_dict['meta_archs'] ) # This is a dict mapping each architecture to a dict, where the key is #epochs and the value is ArchResults self.arch2infos_dict = OrderedDict() + self._avaliable_hps = set() for xkey in sorted(list(file_path_or_dict['arch2infos'].keys())): all_infos = file_path_or_dict['arch2infos'][xkey] hp2archres = OrderedDict() for hp_key, results in all_infos.items(): hp2archres[hp_key] = ArchResults.create_from_state_dict(results) + self._avaliable_hps.add(hp_key) # save the avaliable hyper-parameter self.arch2infos_dict[xkey] = hp2archres self.evaluated_indexes = sorted(list(file_path_or_dict['evaluated_indexes'])) self.archstr2index = {} diff --git a/lib/nas_201_api/api_utils.py b/lib/nas_201_api/api_utils.py index e9213f4..904b825 100644 --- a/lib/nas_201_api/api_utils.py +++ b/lib/nas_201_api/api_utils.py @@ -57,6 +57,10 @@ class NASBenchMetaAPI(metaclass=abc.ABCMeta): def __repr__(self): return ('{name}({num}/{total} architectures, file={filename})'.format(name=self.__class__.__name__, num=len(self.evaluated_indexes), total=len(self.meta_archs), filename=self.filename)) + @property + def avaliable_hps(self): + return list(copy.deepcopy(self._avaliable_hps)) + def random(self): """Return a random index of all architectures.""" return random.randint(0, len(self.meta_archs)-1) diff --git a/scripts-search/NAS-Bench-201/test-weights.sh b/scripts-search/NAS-Bench-201/test-weights.sh deleted file mode 100644 index 1b5c931..0000000 --- a/scripts-search/NAS-Bench-201/test-weights.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# bash ./scripts-search/NAS-Bench-201/test-weights.sh cifar10-valid 1 -echo script name: $0 -echo $# arguments -if [ "$#" -ne 2 ] ;then - echo "Input illegal number of parameters " $# - echo "Need 2 parameters for dataset and use_12_epoch" - exit 1 -fi - -if [ "$TORCH_HOME" = "" ]; then - echo "Must set TORCH_HOME envoriment variable for data dir saving" - exit 1 -else - echo "TORCH_HOME : $TORCH_HOME" -fi - -CUDA_VISIBLE_DEVICES='' OMP_NUM_THREADS=4 python exps/NAS-Bench-201/test-weights.py \ - --base_path $HOME/.torch/NAS-Bench-201-v1_1-096897 \ - --dataset $1 \ - --use_12 $2