update README

This commit is contained in:
D-X-Y 2019-09-28 19:49:13 +10:00
parent cfb462e463
commit c2c1e5100d
7 changed files with 11 additions and 246 deletions

View File

@ -15,7 +15,10 @@ The CIFAR and ImageNet should be downloaded and extracted into `$TORCH_HOME`.
Some methods use knowledge distillation (KD), which require pre-trained models. Please download these models from [Google Driver](https://drive.google.com/open?id=1ANmiYEGX-IQZTfH8w0aSpj-Wypg-0DR-) (or train by yourself) and save into `.latent-data`.
## Network Pruning via Transformable Architecture Search
## [Network Pruning via Transformable Architecture Search](https://arxiv.org/abs/1905.09717)
<img src="https://d-x-y.github.com/resources/paper-icon/NIPS-2019-TAS.png" width="700">
Use `bash ./scripts/prepare.sh` to prepare data splits for `CIFAR-10`, `CIFARR-100`, and `ILSVRC2012`.
If you do not have `ILSVRC2012` data, pleasee comment L12 in `./scripts/prepare.sh`.
@ -50,7 +53,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 bash ./scripts/nas-infer-train.sh imagenet-1k SETN
Searching codes come soon!
## Searching for A Robust Neural Architecture in Four GPU Hours
## [Searching for A Robust Neural Architecture in Four GPU Hours](http://openaccess.thecvf.com/content_CVPR_2019/papers/Dong_Searching_for_a_Robust_Neural_Architecture_in_Four_GPU_Hours_CVPR_2019_paper.pdf)
The old version is located in `others/GDAS`.

View File

@ -1,13 +0,0 @@
{
"scheduler": ["str", "cos"],
"eta_min" : ["float", "0.0"],
"epochs" : ["int", "120"],
"warmup" : ["int", "5"],
"optim" : ["str", "SGD"],
"LR" : ["float", "0.5"],
"decay" : ["float", "0.0001"],
"momentum" : ["float", "0.9"],
"nesterov" : ["bool", "1"],
"criterion": ["str", "Softmax"],
"auxiliary": ["float", "-1"]
}

View File

@ -1,78 +0,0 @@
##################################################
# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 #
##################################################
# python exps/compare.py --checkpoints basic.pth order.pth --names basic order --save ./output/vis/basic-vs-order.pdf
import sys, time, torch, random, argparse
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
from copy import deepcopy
from pathlib import Path
import numpy as np
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
lib_dir = (Path(__file__).parent / '..' / 'lib').resolve()
if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir))
parser = argparse.ArgumentParser(description='Visualize the checkpoint and compare', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--checkpoints', type=str, nargs='+', help='checkpoint paths.')
parser.add_argument('--names', type=str, nargs='+', help='names.')
parser.add_argument('--save', type=str, help='the save path.')
args = parser.parse_args()
def visualize_acc(epochs, accuracies, names, save_path):
LabelSize = 24
LegendFontsize = 22
matplotlib.rcParams['xtick.labelsize'] = LabelSize
matplotlib.rcParams['ytick.labelsize'] = LabelSize
color_set = ['r', 'b', 'g', 'c', 'm', 'y', 'k']
dpi = 300
width, height = 3400, 3600
figsize = width / float(dpi), height / float(dpi)
fig = plt.figure(figsize=figsize)
plt.xlim(0, max(epochs))
plt.ylim(0, 100)
interval_x, interval_y = 20, 10
plt.xticks(np.arange(0, max(epochs) + interval_x, interval_x), fontsize=LegendFontsize)
plt.yticks(np.arange(0, 100 + interval_y, interval_y), fontsize=LegendFontsize)
plt.grid()
plt.xlabel('epoch', fontsize=16)
plt.ylabel('accuracy (%)', fontsize=16)
for idx, tag in enumerate(names):
xaccs = [accuracies[idx][x] for x in epochs]
plt.plot(epochs, xaccs, color=color_set[idx], linestyle='-', label='Test Accuracy : {:}'.format(tag), lw=3)
plt.legend(loc=4, fontsize=LegendFontsize)
if save_path is not None:
fig.savefig(save_path, dpi=dpi, bbox_inches='tight', format='pdf')
print ('---- save figure into {:}.'.format(save_path))
plt.close(fig)
def main():
checkpoints, names = args.checkpoints, args.names
assert len(checkpoints) == len(names), 'invalid length : {:} vs {:}'.format(len(checkpoints), len(names))
for i, checkpoint in enumerate(checkpoints):
assert Path(checkpoint).exists(), 'The {:}-th checkpoint : {:} does not exist'.format( checkpoint )
save_path = Path(args.save)
save_dir = save_path.parent
save_dir.mkdir(parents=True, exist_ok=True)
accuracies = []
for checkpoint in checkpoints:
checkpoint = torch.load( checkpoint )
accuracies.append( checkpoint['valid_accuracies'] )
epochs = [x for x in accuracies[0].keys() if isinstance(x, int)]
epochs = sorted( epochs )
visualize_acc(epochs, accuracies, names, save_path)
if __name__ == '__main__':
main()

View File

@ -1,8 +1,8 @@
## [Searching for A Robust Neural Architecture in Four GPU Hours](http://xuanyidong.com/publication/gradient-based-diff-sampler/)
We propose A Gradient-based neural architecture search approach using Differentiable Architecture Sampler (GDAS). Please find details in [our paper](https://github.com/D-X-Y/GDAS/blob/master/data/GDAS.pdf).
We propose A Gradient-based neural architecture search approach using Differentiable Architecture Sampler (GDAS).
<img src="data/GDAS.png" width="520">
<img src="https://github.com/D-X-Y/NAS-Projects/tree/master/others/GDAS/data/GDAS.png" width="520">
Figure-1. We utilize a DAG to represent the search space of a neural cell. Different operations (colored arrows) transform one node (square) to its intermediate features (little circles). Meanwhile, each node is the sum of the intermediate features transformed from the previous nodes. As indicated by the solid connections, the neural cell in the proposed GDAS is a sampled sub-graph of this DAG. Specifically, among the intermediate features between every two nodes, GDAS samples one feature in a differentiable way.
### Requirements
@ -46,13 +46,15 @@ CUDA_VISIBLE_DEVICES=0 bash ./scripts-rnn/train-WT2.sh GDAS
```
### Training Logs
You can find some training logs in [`./data/logs/`](https://github.com/D-X-Y/GDAS/tree/master/data/logs).
You can find some training logs in [`./data/logs/`](https://github.com/D-X-Y/NAS-Projects/tree/master/others/GDAS/data/logs).
You can also find some pre-trained models in [Google Driver](https://drive.google.com/open?id=1Ofhc49xC1PLIX4O708gJZ1ugzz4td_RJ).
### Experimental Results
<img src="data/imagenet-results.png" width="700">
<img src="https://github.com/D-X-Y/NAS-Projects/tree/master/others/GDAS/data/imagenet-results.png" width="700">
Figure-2. Top-1 and top-5 errors on ImageNet.
### Correction
The Gumbel-softmax tempurature during searching should decrease from 10 to 0.1.

View File

@ -1,74 +0,0 @@
#!/usr/bin/env sh
# bash ./scripts/base-imagenet.sh ResNet110 Step-Soft 256
set -e
if [ "$#" -ne 4 ] ;then
echo "Input illegal number of parameters " $#
echo "Need 4 parameters for the architecture, and softmax/smooth-softmax, and batch-size, and seed"
exit 1
fi
if [ "$TORCH_HOME" = "" ]; then
echo "Must set TORCH_HOME envoriment variable for data dir saving"
exit 1
else
echo "TORCH_HOME : $TORCH_HOME"
fi
arch=$1
opt=$2
batch=$3
rand_seed=$4
dataset=imagenet-1k
PY_C="./env/bin/python"
if [ ! -f ${PY_C} ]; then
echo "Local Run with Python: "`which python`
PY_C="python"
SAVE_ROOT="./output"
else
echo "Cluster Run with Python: "${PY_C}
SAVE_ROOT="./hadoop-data/SearchCheckpoints"
echo "Unzip ILSVRC2012"
tar --version
tar -xf ./hadoop-data/ILSVRC2012.tar -C ${TORCH_HOME}
echo "Unzip ILSVRC2012 done"
fi
if [ ${opt} = "RMSProp" ]; then
epoch=E200
elif [ ${opt} = "Shuffle" ]; then
epoch=E240
dataset=imagenet-1k
elif [ ${opt} = "MobileS" ]; then
epoch=E480
dataset=imagenet-1k-s
elif [ ${opt} = "MobileFast" ] || [ ${opt} = "MobileFastS" ]; then
epoch=E150
dataset=imagenet-1k-s
else
epoch=E120
fi
if [ ${batch} = "256" ]; then
opt_dir=opts
workers=24
elif [ ${batch} = "1024" ]; then
opt_dir=opts-1K
workers=48
else
echo "Invalid batch size : "${batch}
exit 1
fi
save_dir=${SAVE_ROOT}/basic/${dataset}/${arch}-${opt}-${epoch}-${batch}
${PY_C} --version
${PY_C} ./exps/basic-main.py --dataset ${dataset} \
--data_path $TORCH_HOME/ILSVRC2012 \
--model_config ./configs/archs/ImageNet-${arch}.config \
--optim_config ./configs/${opt_dir}/ImageNet-${epoch}-${opt}.config \
--procedure basic \
--save_dir ${save_dir} \
--cutout_length -1 \
--batch_size ${batch} --rand_seed ${rand_seed} --workers ${workers} \
--eval_frequency 1 --print_freq 500 --print_freq_eval 2000

View File

@ -1,6 +0,0 @@
#!/bin/bash
find -name "__pycache__" | xargs rm -rf
find -name "._.DS_Store" | xargs rm -rf
find -name ".DS_Store" | xargs rm -rf
rm -rf output
rm -rf ./scripts-cluster/tmps/job*

View File

@ -1,69 +0,0 @@
#!/bin/bash
# bash ./scripts/com-paddle.sh cifar10
set -e
echo script name: $0
echo $# arguments
if [ "$#" -ne 1 ] ;then
echo "Input illegal number of parameters " $#
echo "Need 1 parameters for the dataset and the-model-name and the-random-seed"
exit 1
fi
if [ "$TORCH_HOME" = "" ]; then
echo "Must set TORCH_HOME envoriment variable for data dir saving"
exit 1
else
echo "TORCH_HOME : $TORCH_HOME"
fi
dataset=$1
PY_C="./env/bin/python"
if [ ! -f ${PY_C} ]; then
echo "Local Run with Python: "`which python`
PY_C="python"
SAVE_ROOT="./output/com-paddle"
else
echo "Cluster Run with Python: "${PY_C}
SAVE_ROOT="./hadoop-data/COM-PADDLE"
fi
basic_func(){
dataset=$1
model=$2
batch=$3
rseed=$4
save_dir=${SAVE_ROOT}/${dataset}-${model}-${batch}
${PY_C} ./exps/basic-main.py --dataset ${dataset} \
--data_path $TORCH_HOME/cifar.python \
--model_config ./configs/archs/CIFAR-${model}.config \
--optim_config ./configs/opts/Com-Paddle-RES.config \
--procedure basic \
--save_dir ${save_dir} --cutout_length -1 --batch_size ${batch} --rand_seed ${rseed} --workers 4 --eval_frequency 1 --print_freq 100 --print_freq_eval 200
}
nas_infer_func(){
dataset=$1
model=$2
batch=$3
rseed=$4
save_dir=${SAVE_ROOT}/${dataset}-${model}-${batch}
${PY_C} ./exps/basic-main.py --dataset ${dataset} \
--data_path $TORCH_HOME/cifar.python --model_source nas \
--model_config ./configs/archs/NAS-CIFAR-${model}.config \
--optim_config ./configs/opts/Com-Paddle-NAS.config \
--procedure basic \
--save_dir ${save_dir} --cutout_length -1 --batch_size ${batch} --rand_seed ${rseed} --workers 4 --eval_frequency 1 --print_freq 100 --print_freq_eval 200
}
#datasets="cifar10 cifar100"
#datasets="cifar10 cifar100"
#for dataset in ${datasets}
#do
#basic_func ${dataset} ResNet20 256 -1
#basic_func ${dataset} ResNet32 256 -1
#basic_func ${dataset} ResNet110 256 -1
#done
nas_infer_func ${dataset} GDAS_V1 96 -1
nas_infer_func ${dataset} SETN 96 -1