update scripts

This commit is contained in:
D-X-Y 2019-02-01 03:23:55 +11:00
parent 4eb1a5ccf9
commit 3f9b54d99e
29 changed files with 115 additions and 137 deletions

View File

@ -16,7 +16,20 @@ Searching CNNs
```
```
Train the Searched RNN
Train the searched CNN on CIFAR
```
bash ./scripts-cnn/train-imagenet.sh 0 GDAS_F1 52 14
bash ./scripts-cnn/train-imagenet.sh 0 GDAS_V1 50 14
```
Train the searched CNN on ImageNet
```
bash ./scripts-cnn/train-imagenet.sh 0 GDAS_F1 52 14
bash ./scripts-cnn/train-imagenet.sh 0 GDAS_V1 50 14
```
Train the searched RNN
```
bash ./scripts-rnn/train-PTB.sh 0 DARTS_V1
bash ./scripts-rnn/train-PTB.sh 0 DARTS_V2

View File

@ -1,3 +1,4 @@
# DARTS First Order, Refer to https://github.com/quark0/darts
import os, sys, time, glob, random, argparse
import numpy as np
from copy import deepcopy

View File

@ -13,25 +13,11 @@ if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir))
from utils import AverageMeter, time_string, convert_secs2time
from utils import print_log, obtain_accuracy
from utils import Cutout, count_parameters_in_MB
from nas import DARTS_V1, DARTS_V2, NASNet, PNASNet, AmoebaNet, ENASNet
from nas import DMS_V1, DMS_F1, GDAS_CC
from meta_nas import META_V1, META_V2
from nas import model_types as models
from train_utils import main_procedure
from train_utils_imagenet import main_procedure_imagenet
from scheduler import load_config
models = {'DARTS_V1': DARTS_V1,
'DARTS_V2': DARTS_V2,
'NASNet' : NASNet,
'PNASNet' : PNASNet,
'ENASNet' : ENASNet,
'DMS_V1' : DMS_V1,
'DMS_F1' : DMS_F1,
'GDAS_CC' : GDAS_CC,
'META_V1' : META_V1,
'META_V2' : META_V2,
'AmoebaNet' : AmoebaNet}
parser = argparse.ArgumentParser("cifar")
parser.add_argument('--data_path', type=str, help='Path to dataset')

View File

@ -10,6 +10,7 @@ from utils import time_string, convert_secs2time
from utils import count_parameters_in_MB
from utils import Cutout
from nas import NetworkCIFAR as Network
from datasets import get_datasets
def obtain_best(accuracies):
if len(accuracies) == 0: return (0, 0)
@ -17,38 +18,10 @@ def obtain_best(accuracies):
s2b = sorted( tops )
return s2b[-1]
def main_procedure(config, dataset, data_path, args, genotype, init_channels, layers, log):
# Mean + Std
if dataset == 'cifar10':
mean = [x / 255 for x in [125.3, 123.0, 113.9]]
std = [x / 255 for x in [63.0, 62.1, 66.7]]
elif dataset == 'cifar100':
mean = [x / 255 for x in [129.3, 124.1, 112.4]]
std = [x / 255 for x in [68.2, 65.4, 70.4]]
else:
raise TypeError("Unknow dataset : {:}".format(dataset))
# Dataset Transformation
if dataset == 'cifar10' or dataset == 'cifar100':
lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(),
transforms.Normalize(mean, std)]
if config.cutout > 0 : lists += [Cutout(config.cutout)]
train_transform = transforms.Compose(lists)
test_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)])
else:
raise TypeError("Unknow dataset : {:}".format(dataset))
# Dataset Defination
if dataset == 'cifar10':
train_data = dset.CIFAR10(data_path, train= True, transform=train_transform, download=True)
test_data = dset.CIFAR10(data_path, train=False, transform=test_transform , download=True)
class_num = 10
elif dataset == 'cifar100':
train_data = dset.CIFAR100(data_path, train= True, transform=train_transform, download=True)
test_data = dset.CIFAR100(data_path, train=False, transform=test_transform , download=True)
class_num = 100
else:
raise TypeError("Unknow dataset : {:}".format(dataset))
train_data, test_data, class_num = get_datasets(dataset, data_path, args.cutout)
print_log('-------------------------------------- main-procedure', log)
print_log('config : {:}'.format(config), log)

View File

@ -12,6 +12,7 @@ from utils import count_parameters_in_MB
from utils import print_FLOPs
from utils import Cutout
from nas import NetworkImageNet as Network
from datasets import get_datasets
def obtain_best(accuracies):
@ -40,30 +41,7 @@ class CrossEntropyLabelSmooth(nn.Module):
def main_procedure_imagenet(config, data_path, args, genotype, init_channels, layers, log):
# training data and testing data
traindir = os.path.join(data_path, 'train')
validdir = os.path.join(data_path, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
train_data = dset.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(
brightness=0.4,
contrast=0.4,
saturation=0.4,
hue=0.2),
transforms.ToTensor(),
normalize,
]))
valid_data = dset.ImageFolder(
validdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
train_data, valid_data, class_num = get_datasets('imagenet-1k', data_path, -1)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=config.batch_size, shuffle= True, pin_memory=True, num_workers=args.workers)
@ -73,7 +51,6 @@ def main_procedure_imagenet(config, data_path, args, genotype, init_channels, la
class_num = 1000
print_log('-------------------------------------- main-procedure', log)
print_log('config : {:}'.format(config), log)
print_log('genotype : {:}'.format(genotype), log)
@ -98,8 +75,7 @@ def main_procedure_imagenet(config, data_path, args, genotype, init_channels, la
criterion_smooth = CrossEntropyLabelSmooth(class_num, config.label_smooth).cuda()
optimizer = torch.optim.SGD(model.parameters(), config.LR, momentum=config.momentum, weight_decay=config.decay)
#optimizer = torch.optim.SGD(model.parameters(), config.LR, momentum=config.momentum, weight_decay=config.decay, nestero=True)
optimizer = torch.optim.SGD(model.parameters(), config.LR, momentum=config.momentum, weight_decay=config.decay, nestero=True)
if config.type == 'cosine':
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(config.epochs))
elif config.type == 'steplr':

View File

@ -1,3 +1,4 @@
from .MetaBatchSampler import MetaBatchSampler
from .TieredImageNet import TieredImageNet
from .LanguageDataset import Corpus
from .get_dataset_with_transform import get_datasets

View File

@ -0,0 +1,74 @@
import os, sys, torch
import os.path as osp
import torchvision.datasets as dset
import torch.backends.cudnn as cudnn
import torchvision.transforms as transforms
from utils import Cutout
from .TieredImageNet import TieredImageNet
Dataset2Class = {'cifar10' : 10,
'cifar100': 100,
'tiered' : -1,
'imagnet-1k' : 1000,
'imagenet-100': 100}
def get_datasets(name, root, cutout):
# Mean + Std
if name == 'cifar10':
mean = [x / 255 for x in [125.3, 123.0, 113.9]]
std = [x / 255 for x in [63.0, 62.1, 66.7]]
elif name == 'cifar100':
mean = [x / 255 for x in [129.3, 124.1, 112.4]]
std = [x / 255 for x in [68.2, 65.4, 70.4]]
elif name == 'tiered':
mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
elif name == 'imagnet-1k' or name == 'imagenet-100':
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
else: raise TypeError("Unknow dataset : {:}".format(name))
# Data Argumentation
if name == 'cifar10' or name == 'cifar100':
lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(),
transforms.Normalize(mean, std)]
if cutout > 0 : lists += [Cutout(cutout)]
train_transform = transforms.Compose(lists)
test_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)])
elif name == 'tiered':
lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(80, padding=4), transforms.ToTensor(), transforms.Normalize(mean, std)]
if cutout > 0 : lists += [Cutout(cutout)]
train_transform = transforms.Compose(lists)
test_transform = transforms.Compose([transforms.CenterCrop(80), transforms.ToTensor(), transforms.Normalize(mean, std)])
elif name == 'imagnet-1k' or name == 'imagenet-100':
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
train_transform = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(
brightness=0.4,
contrast=0.4,
saturation=0.4,
hue=0.2),
transforms.ToTensor(),
normalize,
])
test_transform = transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize])
else: raise TypeError("Unknow dataset : {:}".format(name))
train_data = TieredImageNet(root, 'train-val', train_transform)
test_data = None
if name == 'cifar10':
train_data = dset.CIFAR10(root, train=True, transform=train_transform, download=True)
test_data = dset.CIFAR10(root, train=True, transform=test_transform , download=True)
elif name == 'cifar100':
train_data = dset.CIFAR100(root, train=True, transform=train_transform, download=True)
test_data = dset.CIFAR100(root, train=True, transform=test_transform , download=True)
elif name == 'imagnet-1k' or name == 'imagenet-100':
train_data = dset.ImageFolder(osp.join(root, 'train'), train_transform)
test_data = dset.ImageFolder(osp.join(root, 'val'), train_transform)
else: raise TypeError("Unknow dataset : {:}".format(name))
class_num = Dataset2Class[name]
return train_data, test_data, class_num

View File

@ -1,4 +0,0 @@
rm -rf pytorch
git clone https://github.com/pytorch/pytorch.git
cp -r ./pytorch/torch/nn xnn
rm -rf pytorch

View File

@ -11,8 +11,6 @@ from .CifarNet import NetworkCIFAR
from .ImageNet import NetworkImageNet
# genotypes
from .genotypes import DARTS_V1, DARTS_V2
from .genotypes import NASNet, PNASNet, AmoebaNet, ENASNet
from .genotypes import DMS_V1, DMS_F1, GDAS_CC
from .genotypes import model_types
from .construct_utils import return_alphas_str

View File

@ -179,7 +179,7 @@ ENASNet = Genotype(
DARTS = DARTS_V2
# Search by normal and reduce
DMS_V1 = Genotype(
GDAS_V1 = Genotype(
normal=[('skip_connect', 0, 0.13017432391643524), ('skip_connect', 1, 0.12947972118854523), ('skip_connect', 0, 0.13062666356563568), ('sep_conv_5x5', 2, 0.12980839610099792), ('sep_conv_3x3', 3, 0.12923765182495117), ('skip_connect', 0, 0.12901571393013), ('sep_conv_5x5', 4, 0.12938997149467468), ('sep_conv_3x3', 3, 0.1289220005273819)],
normal_concat=range(2, 6),
reduce=[('sep_conv_5x5', 0, 0.12862831354141235), ('sep_conv_3x3', 1, 0.12783904373645782), ('sep_conv_5x5', 2, 0.12725995481014252), ('sep_conv_5x5', 1, 0.12705285847187042), ('dil_conv_5x5', 2, 0.12797553837299347), ('sep_conv_3x3', 1, 0.12737272679805756), ('sep_conv_5x5', 0, 0.12833961844444275), ('sep_conv_5x5', 1, 0.12758426368236542)],
@ -187,7 +187,7 @@ DMS_V1 = Genotype(
)
# Search by normal and fixing reduction
DMS_F1 = Genotype(
GDAS_F1 = Genotype(
normal=[('skip_connect', 0, 0.16), ('skip_connect', 1, 0.13), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.16), ('sep_conv_3x3', 2, 0.15)],
normal_concat=[2, 3, 4, 5],
reduce=None,
@ -201,3 +201,13 @@ GDAS_CC = Genotype(
reduce=None,
reduce_concat=range(2, 6)
)
model_types = {'DARTS_V1': DARTS_V1,
'DARTS_V2': DARTS_V2,
'NASNet' : NASNet,
'PNASNet' : PNASNet,
'AmoebaNet': AmoebaNet,
'ENASNet' : ENASNet,
'GDAS_V1' : GDAS_V1,
'GDAS_F1' : GDAS_F1,
'GDAS_CC' : GDAS_CC}

View File

@ -1,7 +1,8 @@
#!/usr/bin/env sh
if [ "$#" -ne 2 ] ;then
# bash scripts-cnn/train-cifar.sh 0 GDAS cifar10
if [ "$#" -ne 3 ] ;then
echo "Input illegal number of parameters " $#
echo "Need 2 parameters for the GPUs, the architecture"
echo "Need 3 parameters for the GPUs, the architecture, and the dataset-name"
exit 1
fi
if [ "$TORCH_HOME" = "" ]; then
@ -13,7 +14,7 @@ fi
gpus=$1
arch=$2
dataset=cifar100
dataset=$3
SAVED=./snapshots/NAS/${arch}-${dataset}-E600
CUDA_VISIBLE_DEVICES=${gpus} python ./exps-nas/train_base.py \

View File

@ -18,7 +18,7 @@ channels=$3
layers=$4
SAVED=./snapshots/NAS/${arch}-${dataset}-C${channels}-L${layers}-E250
CUDA_VISIBLE_DEVICES=${gpus} python ./exps-nas/train_base.py \
CUDA_VISIBLE_DEVICES=${gpus} python ./exps-cnn/train_base.py \
--data_path $TORCH_HOME/ILSVRC2012 \
--dataset ${dataset} --arch ${arch} \
--save_path ${SAVED} \

View File

@ -1,25 +0,0 @@
#!/usr/bin/env sh
if [ "$#" -ne 2 ] ;then
echo "Input illegal number of parameters " $#
echo "Need 2 parameters for the GPUs and the architecture"
exit 1
fi
if [ "$TORCH_HOME" = "" ]; then
echo "Must set TORCH_HOME envoriment variable for data dir saving"
exit 1
else
echo "TORCH_HOME : $TORCH_HOME"
fi
gpus=$1
arch=$2
dataset=cifar10
SAVED=./snapshots/NAS/${arch}-${dataset}-E100
CUDA_VISIBLE_DEVICES=${gpus} python ./exps-nas/train_base.py \
--data_path $TORCH_HOME/cifar.python \
--dataset ${dataset} --arch ${arch} \
--save_path ${SAVED} \
--grad_clip 5 \
--model_config ./configs/nas-cifar-cos-simple.config \
--print_freq 100 --workers 8

View File

@ -1,26 +0,0 @@
#!/usr/bin/env sh
if [ "$#" -ne 2 ] ;then
echo "Input illegal number of parameters " $#
echo "Need 2 parameters for the GPUs, the architecture"
exit 1
fi
if [ "$TORCH_HOME" = "" ]; then
echo "Must set TORCH_HOME envoriment variable for data dir saving"
exit 1
else
echo "TORCH_HOME : $TORCH_HOME"
fi
gpus=$1
arch=$2
dataset=cifar10
SAVED=./snapshots/NAS/${arch}-${dataset}-E600
CUDA_VISIBLE_DEVICES=${gpus} python ./exps-nas/train_base.py \
--data_path $TORCH_HOME/cifar.python \
--dataset ${dataset} --arch ${arch} \
--save_path ${SAVED} \
--grad_clip 5 \
--init_channels 36 --layers 20 \
--model_config ./configs/nas-cifar-cos.config \
--print_freq 100 --workers 8