diff --git a/README.md b/README.md index c9b0f35..8bfb5fc 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,10 @@ The CIFAR and ImageNet should be downloaded and extracted into `$TORCH_HOME`. Some methods use knowledge distillation (KD), which require pre-trained models. Please download these models from [Google Driver](https://drive.google.com/open?id=1ANmiYEGX-IQZTfH8w0aSpj-Wypg-0DR-) (or train by yourself) and save into `.latent-data`. -## Network Pruning via Transformable Architecture Search +## [Network Pruning via Transformable Architecture Search](https://arxiv.org/abs/1905.09717) + + + Use `bash ./scripts/prepare.sh` to prepare data splits for `CIFAR-10`, `CIFARR-100`, and `ILSVRC2012`. If you do not have `ILSVRC2012` data, pleasee comment L12 in `./scripts/prepare.sh`. @@ -50,7 +53,7 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 bash ./scripts/nas-infer-train.sh imagenet-1k SETN Searching codes come soon! -## Searching for A Robust Neural Architecture in Four GPU Hours +## [Searching for A Robust Neural Architecture in Four GPU Hours](http://openaccess.thecvf.com/content_CVPR_2019/papers/Dong_Searching_for_a_Robust_Neural_Architecture_in_Four_GPU_Hours_CVPR_2019_paper.pdf) The old version is located in `others/GDAS`. diff --git a/configs/opts-1K/ImageNet-E120-Cos-Soft.config b/configs/opts-1K/ImageNet-E120-Cos-Soft.config deleted file mode 100644 index 49e937f..0000000 --- a/configs/opts-1K/ImageNet-E120-Cos-Soft.config +++ /dev/null @@ -1,13 +0,0 @@ -{ - "scheduler": ["str", "cos"], - "eta_min" : ["float", "0.0"], - "epochs" : ["int", "120"], - "warmup" : ["int", "5"], - "optim" : ["str", "SGD"], - "LR" : ["float", "0.5"], - "decay" : ["float", "0.0001"], - "momentum" : ["float", "0.9"], - "nesterov" : ["bool", "1"], - "criterion": ["str", "Softmax"], - "auxiliary": ["float", "-1"] -} diff --git a/exps/compare.py b/exps/compare.py deleted file mode 100644 index eb917bb..0000000 --- a/exps/compare.py +++ /dev/null @@ -1,78 +0,0 @@ -################################################## -# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # -################################################## -# python exps/compare.py --checkpoints basic.pth order.pth --names basic order --save ./output/vis/basic-vs-order.pdf -import sys, time, torch, random, argparse -from PIL import ImageFile -ImageFile.LOAD_TRUNCATED_IMAGES = True -from copy import deepcopy -from pathlib import Path -import numpy as np -import matplotlib -matplotlib.use('agg') -import matplotlib.pyplot as plt - -lib_dir = (Path(__file__).parent / '..' / 'lib').resolve() -if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) - -parser = argparse.ArgumentParser(description='Visualize the checkpoint and compare', formatter_class=argparse.ArgumentDefaultsHelpFormatter) -parser.add_argument('--checkpoints', type=str, nargs='+', help='checkpoint paths.') -parser.add_argument('--names', type=str, nargs='+', help='names.') -parser.add_argument('--save', type=str, help='the save path.') -args = parser.parse_args() - - -def visualize_acc(epochs, accuracies, names, save_path): - - LabelSize = 24 - LegendFontsize = 22 - matplotlib.rcParams['xtick.labelsize'] = LabelSize - matplotlib.rcParams['ytick.labelsize'] = LabelSize - color_set = ['r', 'b', 'g', 'c', 'm', 'y', 'k'] - dpi = 300 - width, height = 3400, 3600 - figsize = width / float(dpi), height / float(dpi) - - fig = plt.figure(figsize=figsize) - plt.xlim(0, max(epochs)) - plt.ylim(0, 100) - interval_x, interval_y = 20, 10 - plt.xticks(np.arange(0, max(epochs) + interval_x, interval_x), fontsize=LegendFontsize) - plt.yticks(np.arange(0, 100 + interval_y, interval_y), fontsize=LegendFontsize) - plt.grid() - - plt.xlabel('epoch', fontsize=16) - plt.ylabel('accuracy (%)', fontsize=16) - - for idx, tag in enumerate(names): - xaccs = [accuracies[idx][x] for x in epochs] - plt.plot(epochs, xaccs, color=color_set[idx], linestyle='-', label='Test Accuracy : {:}'.format(tag), lw=3) - plt.legend(loc=4, fontsize=LegendFontsize) - - if save_path is not None: - fig.savefig(save_path, dpi=dpi, bbox_inches='tight', format='pdf') - print ('---- save figure into {:}.'.format(save_path)) - plt.close(fig) - - -def main(): - checkpoints, names = args.checkpoints, args.names - assert len(checkpoints) == len(names), 'invalid length : {:} vs {:}'.format(len(checkpoints), len(names)) - for i, checkpoint in enumerate(checkpoints): - assert Path(checkpoint).exists(), 'The {:}-th checkpoint : {:} does not exist'.format( checkpoint ) - - save_path = Path(args.save) - save_dir = save_path.parent - save_dir.mkdir(parents=True, exist_ok=True) - accuracies = [] - for checkpoint in checkpoints: - checkpoint = torch.load( checkpoint ) - accuracies.append( checkpoint['valid_accuracies'] ) - epochs = [x for x in accuracies[0].keys() if isinstance(x, int)] - epochs = sorted( epochs ) - - visualize_acc(epochs, accuracies, names, save_path) - - -if __name__ == '__main__': - main() diff --git a/others/GDAS/README.md b/others/GDAS/README.md index 7ddfb73..0739c30 100644 --- a/others/GDAS/README.md +++ b/others/GDAS/README.md @@ -1,8 +1,8 @@ ## [Searching for A Robust Neural Architecture in Four GPU Hours](http://xuanyidong.com/publication/gradient-based-diff-sampler/) -We propose A Gradient-based neural architecture search approach using Differentiable Architecture Sampler (GDAS). Please find details in [our paper](https://github.com/D-X-Y/GDAS/blob/master/data/GDAS.pdf). +We propose A Gradient-based neural architecture search approach using Differentiable Architecture Sampler (GDAS). - + Figure-1. We utilize a DAG to represent the search space of a neural cell. Different operations (colored arrows) transform one node (square) to its intermediate features (little circles). Meanwhile, each node is the sum of the intermediate features transformed from the previous nodes. As indicated by the solid connections, the neural cell in the proposed GDAS is a sampled sub-graph of this DAG. Specifically, among the intermediate features between every two nodes, GDAS samples one feature in a differentiable way. ### Requirements @@ -46,13 +46,15 @@ CUDA_VISIBLE_DEVICES=0 bash ./scripts-rnn/train-WT2.sh GDAS ``` ### Training Logs -You can find some training logs in [`./data/logs/`](https://github.com/D-X-Y/GDAS/tree/master/data/logs). +You can find some training logs in [`./data/logs/`](https://github.com/D-X-Y/NAS-Projects/tree/master/others/GDAS/data/logs). You can also find some pre-trained models in [Google Driver](https://drive.google.com/open?id=1Ofhc49xC1PLIX4O708gJZ1ugzz4td_RJ). + ### Experimental Results - + Figure-2. Top-1 and top-5 errors on ImageNet. + ### Correction The Gumbel-softmax tempurature during searching should decrease from 10 to 0.1. diff --git a/scripts/base-imagenet.sh b/scripts/base-imagenet.sh deleted file mode 100644 index 22e4015..0000000 --- a/scripts/base-imagenet.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env sh -# bash ./scripts/base-imagenet.sh ResNet110 Step-Soft 256 -set -e -if [ "$#" -ne 4 ] ;then - echo "Input illegal number of parameters " $# - echo "Need 4 parameters for the architecture, and softmax/smooth-softmax, and batch-size, and seed" - exit 1 -fi -if [ "$TORCH_HOME" = "" ]; then - echo "Must set TORCH_HOME envoriment variable for data dir saving" - exit 1 -else - echo "TORCH_HOME : $TORCH_HOME" -fi - -arch=$1 -opt=$2 -batch=$3 -rand_seed=$4 -dataset=imagenet-1k - -PY_C="./env/bin/python" -if [ ! -f ${PY_C} ]; then - echo "Local Run with Python: "`which python` - PY_C="python" - SAVE_ROOT="./output" -else - echo "Cluster Run with Python: "${PY_C} - SAVE_ROOT="./hadoop-data/SearchCheckpoints" - echo "Unzip ILSVRC2012" - tar --version - tar -xf ./hadoop-data/ILSVRC2012.tar -C ${TORCH_HOME} - echo "Unzip ILSVRC2012 done" -fi - -if [ ${opt} = "RMSProp" ]; then - epoch=E200 -elif [ ${opt} = "Shuffle" ]; then - epoch=E240 - dataset=imagenet-1k -elif [ ${opt} = "MobileS" ]; then - epoch=E480 - dataset=imagenet-1k-s -elif [ ${opt} = "MobileFast" ] || [ ${opt} = "MobileFastS" ]; then - epoch=E150 - dataset=imagenet-1k-s -else - epoch=E120 -fi - -if [ ${batch} = "256" ]; then - opt_dir=opts - workers=24 -elif [ ${batch} = "1024" ]; then - opt_dir=opts-1K - workers=48 -else - echo "Invalid batch size : "${batch} - exit 1 -fi - -save_dir=${SAVE_ROOT}/basic/${dataset}/${arch}-${opt}-${epoch}-${batch} - -${PY_C} --version - -${PY_C} ./exps/basic-main.py --dataset ${dataset} \ - --data_path $TORCH_HOME/ILSVRC2012 \ - --model_config ./configs/archs/ImageNet-${arch}.config \ - --optim_config ./configs/${opt_dir}/ImageNet-${epoch}-${opt}.config \ - --procedure basic \ - --save_dir ${save_dir} \ - --cutout_length -1 \ - --batch_size ${batch} --rand_seed ${rand_seed} --workers ${workers} \ - --eval_frequency 1 --print_freq 500 --print_freq_eval 2000 diff --git a/scripts/clean.sh b/scripts/clean.sh deleted file mode 100644 index 1fca5ae..0000000 --- a/scripts/clean.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -find -name "__pycache__" | xargs rm -rf -find -name "._.DS_Store" | xargs rm -rf -find -name ".DS_Store" | xargs rm -rf -rm -rf output -rm -rf ./scripts-cluster/tmps/job* diff --git a/scripts/com-paddle.sh b/scripts/com-paddle.sh deleted file mode 100644 index cb3a276..0000000 --- a/scripts/com-paddle.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# bash ./scripts/com-paddle.sh cifar10 -set -e -echo script name: $0 -echo $# arguments -if [ "$#" -ne 1 ] ;then - echo "Input illegal number of parameters " $# - echo "Need 1 parameters for the dataset and the-model-name and the-random-seed" - exit 1 -fi -if [ "$TORCH_HOME" = "" ]; then - echo "Must set TORCH_HOME envoriment variable for data dir saving" - exit 1 -else - echo "TORCH_HOME : $TORCH_HOME" -fi - -dataset=$1 - -PY_C="./env/bin/python" -if [ ! -f ${PY_C} ]; then - echo "Local Run with Python: "`which python` - PY_C="python" - SAVE_ROOT="./output/com-paddle" -else - echo "Cluster Run with Python: "${PY_C} - SAVE_ROOT="./hadoop-data/COM-PADDLE" -fi - -basic_func(){ - dataset=$1 - model=$2 - batch=$3 - rseed=$4 - save_dir=${SAVE_ROOT}/${dataset}-${model}-${batch} - ${PY_C} ./exps/basic-main.py --dataset ${dataset} \ - --data_path $TORCH_HOME/cifar.python \ - --model_config ./configs/archs/CIFAR-${model}.config \ - --optim_config ./configs/opts/Com-Paddle-RES.config \ - --procedure basic \ - --save_dir ${save_dir} --cutout_length -1 --batch_size ${batch} --rand_seed ${rseed} --workers 4 --eval_frequency 1 --print_freq 100 --print_freq_eval 200 -} - -nas_infer_func(){ - dataset=$1 - model=$2 - batch=$3 - rseed=$4 - save_dir=${SAVE_ROOT}/${dataset}-${model}-${batch} - ${PY_C} ./exps/basic-main.py --dataset ${dataset} \ - --data_path $TORCH_HOME/cifar.python --model_source nas \ - --model_config ./configs/archs/NAS-CIFAR-${model}.config \ - --optim_config ./configs/opts/Com-Paddle-NAS.config \ - --procedure basic \ - --save_dir ${save_dir} --cutout_length -1 --batch_size ${batch} --rand_seed ${rseed} --workers 4 --eval_frequency 1 --print_freq 100 --print_freq_eval 200 -} - -#datasets="cifar10 cifar100" - -#datasets="cifar10 cifar100" -#for dataset in ${datasets} -#do -#basic_func ${dataset} ResNet20 256 -1 -#basic_func ${dataset} ResNet32 256 -1 -#basic_func ${dataset} ResNet110 256 -1 -#done - -nas_infer_func ${dataset} GDAS_V1 96 -1 -nas_infer_func ${dataset} SETN 96 -1