From 4e322bb9c7e06dcf87786e1b589e3c5fc590407e Mon Sep 17 00:00:00 2001 From: D-X-Y <280835372@qq.com> Date: Sun, 23 Feb 2020 10:30:37 +1100 Subject: [PATCH] correct copyright date --- LICENSE.md | 2 +- docs/NAS-Bench-201.md | 7 ++ exps/KD-main.py | 6 +- exps/NAS-Bench-201/check.py | 8 +- exps/NAS-Bench-201/dist-setup.py | 5 +- exps/NAS-Bench-201/functions.py | 6 +- exps/NAS-Bench-201/main.py | 2 +- exps/NAS-Bench-201/statistics.py | 6 +- exps/NAS-Bench-201/test-correlation.py | 4 +- exps/NAS-Bench-201/visualize.py | 8 +- exps/basic-eval.py | 6 +- exps/basic-main.py | 6 +- exps/search-shape.py | 6 +- exps/search-transformable.py | 4 +- lib/log_utils/time_utils.py | 6 +- lib/models/SharedUtils.py | 6 +- lib/models/cell_infers/__init__.py | 6 +- lib/models/cell_infers/cells.py | 6 +- lib/models/cell_infers/tiny_network.py | 6 +- lib/models/cell_searchs/search_model_setn.py | 4 +- .../cell_searchs/search_model_setn_nasnet.py | 113 ++++++++++++++++++ lib/nas_201_api/__init__.py | 6 +- lib/nas_201_api/api.py | 4 +- lib/nas_infer_model/__init__.py | 6 +- lib/nas_infer_model/operations.py | 6 +- lib/procedures/optimizers.py | 6 +- lib/procedures/simple_KD_main.py | 6 +- 27 files changed, 190 insertions(+), 67 deletions(-) create mode 100644 lib/models/cell_searchs/search_model_setn_nasnet.py diff --git a/LICENSE.md b/LICENSE.md index 1cee3ec..ed92c61 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018-2020 Xuanyi Dong (GitHub: https://github.com/D-X-Y) +Copyright (c) since 2019.01.01, author: Xuanyi Dong (GitHub: https://github.com/D-X-Y) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/NAS-Bench-201.md b/docs/NAS-Bench-201.md index 2a3e72d..3a2a810 100644 --- a/docs/NAS-Bench-201.md +++ b/docs/NAS-Bench-201.md @@ -71,6 +71,13 @@ print ('Train Info [10-th epoch] : {:}'.format(results[0].get_train(10))) index = api.query_index_by_arch('|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|skip_connect~2|') api.show(index) ``` +This string `|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|skip_connect~2|` means: +``` +node-0: the input tensor +node-1: conv-3x3( node-0 ) +node-2: conv-3x3( node-0 ) + avg-pool-3x3( node-1 ) +node-3: skip-connect( node-0 ) + conv-3x3( node-1 ) + skip-connect( node-2 ) +``` 5. Create the network from api: ``` diff --git a/exps/KD-main.py b/exps/KD-main.py index 680e3d2..0670222 100644 --- a/exps/KD-main.py +++ b/exps/KD-main.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import sys, time, torch, random, argparse from PIL import ImageFile ImageFile.LOAD_TRUNCATED_IMAGES = True diff --git a/exps/NAS-Bench-201/check.py b/exps/NAS-Bench-201/check.py index 05f7e7d..165f094 100644 --- a/exps/NAS-Bench-201/check.py +++ b/exps/NAS-Bench-201/check.py @@ -1,8 +1,8 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +##################################################### # python exps/NAS-Bench-201/check.py --base_save_dir -################################################## +##################################################### import os, sys, time, argparse, collections from shutil import copyfile import torch diff --git a/exps/NAS-Bench-201/dist-setup.py b/exps/NAS-Bench-201/dist-setup.py index 5410f8c..6071ff2 100644 --- a/exps/NAS-Bench-201/dist-setup.py +++ b/exps/NAS-Bench-201/dist-setup.py @@ -1,3 +1,6 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +##################################################### import os from setuptools import setup @@ -9,7 +12,7 @@ def read(fname='README.md'): setup( name = "nas_bench_201", - version = "1.0", + version = "1.1", author = "Xuanyi Dong", author_email = "dongxuanyi888@gmail.com", description = "API for NAS-Bench-201 (a benchmark for neural architecture search).", diff --git a/exps/NAS-Bench-201/functions.py b/exps/NAS-Bench-201/functions.py index dc1a173..4e48ebb 100644 --- a/exps/NAS-Bench-201/functions.py +++ b/exps/NAS-Bench-201/functions.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +##################################################### import os, sys, time, torch from procedures import prepare_seed, get_optim_scheduler from utils import get_model_infos, obtain_accuracy diff --git a/exps/NAS-Bench-201/main.py b/exps/NAS-Bench-201/main.py index eb600e1..d9ead87 100644 --- a/exps/NAS-Bench-201/main.py +++ b/exps/NAS-Bench-201/main.py @@ -1,7 +1,7 @@ ############################################################### # NAS-Bench-201, ICLR 2020 (https://arxiv.org/abs/2001.00326) # ############################################################### -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019-2020 # +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # ############################################################### import os, sys, time, torch, random, argparse from PIL import ImageFile diff --git a/exps/NAS-Bench-201/statistics.py b/exps/NAS-Bench-201/statistics.py index f745198..a77b204 100644 --- a/exps/NAS-Bench-201/statistics.py +++ b/exps/NAS-Bench-201/statistics.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +##################################################### import os, sys, time, argparse, collections from copy import deepcopy import torch diff --git a/exps/NAS-Bench-201/test-correlation.py b/exps/NAS-Bench-201/test-correlation.py index 4c7c46b..7b3202b 100644 --- a/exps/NAS-Bench-201/test-correlation.py +++ b/exps/NAS-Bench-201/test-correlation.py @@ -1,5 +1,5 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # ######################################################## # python exps/NAS-Bench-201/test-correlation.py --api_path $HOME/.torch/NAS-Bench-201-v1_0-e61699.pth ######################################################## diff --git a/exps/NAS-Bench-201/visualize.py b/exps/NAS-Bench-201/visualize.py index 274171b..ccc85bf 100644 --- a/exps/NAS-Bench-201/visualize.py +++ b/exps/NAS-Bench-201/visualize.py @@ -1,8 +1,8 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +##################################################### # python exps/NAS-Bench-201/visualize.py --api_path $HOME/.torch/NAS-Bench-201-v1_0-e61699.pth -################################################## +##################################################### import os, sys, time, argparse, collections from tqdm import tqdm from collections import OrderedDict diff --git a/exps/basic-eval.py b/exps/basic-eval.py index a822b16..ecf2253 100644 --- a/exps/basic-eval.py +++ b/exps/basic-eval.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import os, sys, time, torch, random, argparse from PIL import ImageFile ImageFile.LOAD_TRUNCATED_IMAGES = True diff --git a/exps/basic-main.py b/exps/basic-main.py index ef4a54d..f3dda20 100644 --- a/exps/basic-main.py +++ b/exps/basic-main.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import sys, time, torch, random, argparse from PIL import ImageFile ImageFile.LOAD_TRUNCATED_IMAGES = True diff --git a/exps/search-shape.py b/exps/search-shape.py index 2a8fc96..8eb5e4c 100644 --- a/exps/search-shape.py +++ b/exps/search-shape.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import sys, time, torch, random, argparse from PIL import ImageFile from os import path as osp diff --git a/exps/search-transformable.py b/exps/search-transformable.py index e677058..3b2a0eb 100644 --- a/exps/search-transformable.py +++ b/exps/search-transformable.py @@ -1,5 +1,5 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # ####################################################################### # Network Pruning via Transformable Architecture Search, NeurIPS 2019 # ####################################################################### diff --git a/lib/log_utils/time_utils.py b/lib/log_utils/time_utils.py index e38461f..4a0f78e 100644 --- a/lib/log_utils/time_utils.py +++ b/lib/log_utils/time_utils.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import time, sys import numpy as np diff --git a/lib/models/SharedUtils.py b/lib/models/SharedUtils.py index 0536ebf..8938752 100644 --- a/lib/models/SharedUtils.py +++ b/lib/models/SharedUtils.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import torch import torch.nn as nn diff --git a/lib/models/cell_infers/__init__.py b/lib/models/cell_infers/__init__.py index 63bd95b..052b477 100644 --- a/lib/models/cell_infers/__init__.py +++ b/lib/models/cell_infers/__init__.py @@ -1,4 +1,4 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### from .tiny_network import TinyNetwork diff --git a/lib/models/cell_infers/cells.py b/lib/models/cell_infers/cells.py index 62bb79e..bd94676 100644 --- a/lib/models/cell_infers/cells.py +++ b/lib/models/cell_infers/cells.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import torch.nn as nn from copy import deepcopy diff --git a/lib/models/cell_infers/tiny_network.py b/lib/models/cell_infers/tiny_network.py index f7994f7..bb5000f 100644 --- a/lib/models/cell_infers/tiny_network.py +++ b/lib/models/cell_infers/tiny_network.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import torch import torch.nn as nn from ..cell_operations import ResNetBasicblock diff --git a/lib/models/cell_searchs/search_model_setn.py b/lib/models/cell_searchs/search_model_setn.py index 3df35b6..83d7659 100644 --- a/lib/models/cell_searchs/search_model_setn.py +++ b/lib/models/cell_searchs/search_model_setn.py @@ -1,5 +1,5 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # ###################################################################################### # One-Shot Neural Architecture Search via Self-Evaluated Template Network, ICCV 2019 # ###################################################################################### diff --git a/lib/models/cell_searchs/search_model_setn_nasnet.py b/lib/models/cell_searchs/search_model_setn_nasnet.py new file mode 100644 index 0000000..7218261 --- /dev/null +++ b/lib/models/cell_searchs/search_model_setn_nasnet.py @@ -0,0 +1,113 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +###################################################################################### +# One-Shot Neural Architecture Search via Self-Evaluated Template Network, ICCV 2019 # +###################################################################################### +import torch +import torch.nn as nn +from copy import deepcopy +from .search_cells import NASNetSearchCell as SearchCell +from .genotypes import Structure + + +# The macro structure is based on NASNet +class NASNetworkSETN(nn.Module): + + def __init__(self, C, N, steps, multiplier, stem_multiplier, num_classes, search_space, affine, track_running_stats): + super(NASNetworkSETN, self).__init__() + self._C = C + self._layerN = N + self._steps = steps + self._multiplier = multiplier + self.stem = nn.Sequential( + nn.Conv2d(3, C*stem_multiplier, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(C*stem_multiplier)) + + # config for each layer + layer_channels = [C ] * N + [C*2 ] + [C*2 ] * (N-1) + [C*4 ] + [C*4 ] * (N-1) + layer_reductions = [False] * N + [True] + [False] * (N-1) + [True] + [False] * (N-1) + + num_edge, edge2index = None, None + C_prev_prev, C_prev, C_curr, reduction_prev = C*stem_multiplier, C*stem_multiplier, C, False + + self.cells = nn.ModuleList() + for index, (C_curr, reduction) in enumerate(zip(layer_channels, layer_reductions)): + cell = SearchCell(search_space, steps, multiplier, C_prev_prev, C_prev, C_curr, reduction, reduction_prev, affine, track_running_stats) + if num_edge is None: num_edge, edge2index = cell.num_edges, cell.edge2index + else: assert num_edge == cell.num_edges and edge2index == cell.edge2index, 'invalid {:} vs. {:}.'.format(num_edge, cell.num_edges) + self.cells.append( cell ) + C_prev_prev, C_prev, reduction_prev = C_prev, multiplier*C_curr, reduction + self.op_names = deepcopy( search_space ) + self._Layer = len(self.cells) + self.edge2index = edge2index + self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True)) + self.global_pooling = nn.AdaptiveAvgPool2d(1) + self.classifier = nn.Linear(C_prev, num_classes) + self.arch_normal_parameters = nn.Parameter( 1e-3*torch.randn(num_edge, len(search_space)) ) + self.arch_reduce_parameters = nn.Parameter( 1e-3*torch.randn(num_edge, len(search_space)) ) + + def get_weights(self): + xlist = list( self.stem.parameters() ) + list( self.cells.parameters() ) + xlist+= list( self.lastact.parameters() ) + list( self.global_pooling.parameters() ) + xlist+= list( self.classifier.parameters() ) + return xlist + + def get_alphas(self): + return [self.arch_normal_parameters, self.arch_reduce_parameters] + + def show_alphas(self): + with torch.no_grad(): + A = 'arch-normal-parameters :\n{:}'.format( nn.functional.softmax(self.arch_normal_parameters, dim=-1).cpu() ) + B = 'arch-reduce-parameters :\n{:}'.format( nn.functional.softmax(self.arch_reduce_parameters, dim=-1).cpu() ) + return '{:}\n{:}'.format(A, B) + + def get_message(self): + string = self.extra_repr() + for i, cell in enumerate(self.cells): + string += '\n {:02d}/{:02d} :: {:}'.format(i, len(self.cells), cell.extra_repr()) + return string + + def extra_repr(self): + return ('{name}(C={_C}, N={_layerN}, steps={_steps}, multiplier={_multiplier}, L={_Layer})'.format(name=self.__class__.__name__, **self.__dict__)) + + def genotype(self): + def _parse(weights): + gene = [] + for i in range(self._steps): + edges = [] + for j in range(2+i): + node_str = '{:}<-{:}'.format(i, j) + ws = weights[ self.edge2index[node_str] ] + for k, op_name in enumerate(self.op_names): + if op_name == 'none': continue + edges.append( (op_name, j, ws[k]) ) + edges = sorted(edges, key=lambda x: -x[-1]) + selected_edges = edges[:2] + gene.append( tuple(selected_edges) ) + return gene + with torch.no_grad(): + gene_normal = _parse(torch.softmax(self.arch_normal_parameters, dim=-1).cpu().numpy()) + gene_reduce = _parse(torch.softmax(self.arch_reduce_parameters, dim=-1).cpu().numpy()) + return {'normal': gene_normal, 'normal_concat': list(range(2+self._steps-self._multiplier, self._steps+2)), + 'reduce': gene_reduce, 'reduce_concat': list(range(2+self._steps-self._multiplier, self._steps+2))} + + def forward(self, inputs): + normal_hardwts = nn.functional.softmax(self.arch_normal_parameters, dim=-1) + reduce_hardwts = nn.functional.softmax(self.arch_reduce_parameters, dim=-1) + with torch.no_grad(): + normal_hardwts_cpu = normal_hardwts.detach().cpu() + reduce_hardwts_cpu = reduce_hardwts.detach().cpu() + + s0 = s1 = self.stem(inputs) + for i, cell in enumerate(self.cells): + # [TODO] + raise NotImplementedError + if cell.reduction: hardwts, index = reduce_hardwts, reduce_index + else : hardwts, index = normal_hardwts, normal_index + s0, s1 = s1, cell.forward_gdas(s0, s1, hardwts, index) + out = self.lastact(s1) + out = self.global_pooling( out ) + out = out.view(out.size(0), -1) + logits = self.classifier(out) + + return out, logits diff --git a/lib/nas_201_api/__init__.py b/lib/nas_201_api/__init__.py index 74d3dda..3eb8b38 100644 --- a/lib/nas_201_api/__init__.py +++ b/lib/nas_201_api/__init__.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +##################################################### from .api import NASBench201API from .api import ArchResults, ResultsCount diff --git a/lib/nas_201_api/api.py b/lib/nas_201_api/api.py index 00ab752..e172386 100644 --- a/lib/nas_201_api/api.py +++ b/lib/nas_201_api/api.py @@ -1,5 +1,5 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # ############################################################################################ # NAS-Bench-201: Extending the Scope of Reproducible Neural Architecture Search, ICLR 2020 # ############################################################################################ diff --git a/lib/nas_infer_model/__init__.py b/lib/nas_infer_model/__init__.py index bb0d0e5..ef20f89 100644 --- a/lib/nas_infer_model/__init__.py +++ b/lib/nas_infer_model/__init__.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### # I write this package to make AutoDL-Projects to be compatible with the old GDAS projects. # Ideally, this package will be merged into lib/models/cell_infers in future. # Currently, this package is used to reproduce the results in GDAS (Searching for A Robust Neural Architecture in Four GPU Hours, CVPR 2019). diff --git a/lib/nas_infer_model/operations.py b/lib/nas_infer_model/operations.py index 1c56905..825f2e3 100644 --- a/lib/nas_infer_model/operations.py +++ b/lib/nas_infer_model/operations.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +############################################################################################## +# This code is copied and modified from Hanxiao Liu's work (https://github.com/quark0/darts) # +############################################################################################## import torch import torch.nn as nn diff --git a/lib/procedures/optimizers.py b/lib/procedures/optimizers.py index a3cb84a..7fe086d 100644 --- a/lib/procedures/optimizers.py +++ b/lib/procedures/optimizers.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import math, torch import torch.nn as nn from bisect import bisect_right diff --git a/lib/procedures/simple_KD_main.py b/lib/procedures/simple_KD_main.py index 007b51c..5ad8c3b 100644 --- a/lib/procedures/simple_KD_main.py +++ b/lib/procedures/simple_KD_main.py @@ -1,6 +1,6 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### import os, sys, time, torch import torch.nn.functional as F # our modules