From 1ce3249a5a58af3506b8c9af977008ddf8198445 Mon Sep 17 00:00:00 2001 From: D-X-Y <280835372@qq.com> Date: Sat, 18 Jan 2020 22:47:10 +1100 Subject: [PATCH] update DARTS-V1 --- exps/algos/DARTS-V1.py | 7 ++-- scripts-search/algos/DARTS-test-Gradient.sh | 42 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 scripts-search/algos/DARTS-test-Gradient.sh diff --git a/exps/algos/DARTS-V1.py b/exps/algos/DARTS-V1.py index b06f779..e033884 100644 --- a/exps/algos/DARTS-V1.py +++ b/exps/algos/DARTS-V1.py @@ -20,7 +20,7 @@ from models import get_cell_based_tiny_net, get_search_spaces from nas_201_api import NASBench201API as API -def search_func(xloader, network, criterion, scheduler, w_optimizer, a_optimizer, epoch_str, print_freq, logger): +def search_func(xloader, network, criterion, scheduler, w_optimizer, a_optimizer, epoch_str, print_freq, logger, gradient_clip): data_time, batch_time = AverageMeter(), AverageMeter() base_losses, base_top1, base_top5 = AverageMeter(), AverageMeter(), AverageMeter() arch_losses, arch_top1, arch_top5 = AverageMeter(), AverageMeter(), AverageMeter() @@ -38,7 +38,7 @@ def search_func(xloader, network, criterion, scheduler, w_optimizer, a_optimizer _, logits = network(base_inputs) base_loss = criterion(logits, base_targets) base_loss.backward() - torch.nn.utils.clip_grad_norm_(network.parameters(), 5) + if gradient_clip > 0: torch.nn.utils.clip_grad_norm_(network.parameters(), gradient_clip) w_optimizer.step() # record base_prec1, base_prec5 = obtain_accuracy(logits.data, base_targets.data, topk=(1, 5)) @@ -165,7 +165,7 @@ def main(xargs): epoch_str = '{:03d}-{:03d}'.format(epoch, total_epoch) logger.log('\n[Search the {:}-th epoch] {:}, LR={:}'.format(epoch_str, need_time, min(w_scheduler.get_lr()))) - search_w_loss, search_w_top1, search_w_top5 = search_func(search_loader, network, criterion, w_scheduler, w_optimizer, a_optimizer, epoch_str, xargs.print_freq, logger) + search_w_loss, search_w_top1, search_w_top5 = search_func(search_loader, network, criterion, w_scheduler, w_optimizer, a_optimizer, epoch_str, xargs.print_freq, logger, xargs.gradient_clip) search_time.update(time.time() - start_time) logger.log('[{:}] searching : loss={:.2f}, accuracy@1={:.2f}%, accuracy@5={:.2f}%, time-cost={:.1f} s'.format(epoch_str, search_w_loss, search_w_top1, search_w_top5, search_time.sum)) valid_a_loss , valid_a_top1 , valid_a_top5 = valid_func(valid_loader, network, criterion) @@ -225,6 +225,7 @@ if __name__ == '__main__': parser.add_argument('--track_running_stats',type=int, choices=[0,1],help='Whether use track_running_stats or not in the BN layer.') parser.add_argument('--config_path', type=str, help='The config path.') parser.add_argument('--model_config', type=str, help='The path of the model configuration. When this arg is set, it will cover max_nodes / channels / num_cells.') + parser.add_argument('--gradient_clip', type=float, default=5, help='') # architecture leraning rate parser.add_argument('--arch_learning_rate', type=float, default=3e-4, help='learning rate for arch encoding') parser.add_argument('--arch_weight_decay', type=float, default=1e-3, help='weight decay for arch encoding') diff --git a/scripts-search/algos/DARTS-test-Gradient.sh b/scripts-search/algos/DARTS-test-Gradient.sh new file mode 100644 index 0000000..434c6ec --- /dev/null +++ b/scripts-search/algos/DARTS-test-Gradient.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# bash ./scripts-search/algos/DARTS-test-Gradient.sh cifar10 0 5 +echo script name: $0 +echo $# arguments +if [ "$#" -ne 3 ] ;then + echo "Input illegal number of parameters " $# + echo "Need 3 parameters for dataset, tracking_status, and gradient_clip" + exit 1 +fi +if [ "$TORCH_HOME" = "" ]; then + echo "Must set TORCH_HOME envoriment variable for data dir saving" + exit 1 +else + echo "TORCH_HOME : $TORCH_HOME" +fi + +dataset=$1 +BN=$2 +gradient_clip=$3 +seed=-1 +channel=16 +num_cells=5 +max_nodes=4 +space=nas-bench-201 + +if [ "$dataset" == "cifar10" ] || [ "$dataset" == "cifar100" ]; then + data_path="$TORCH_HOME/cifar.python" +else + data_path="$TORCH_HOME/cifar.python/ImageNet16" +fi + +save_dir=./output/search-cell-${space}/DARTS-V1-${dataset}-BN${BN}-Gradient${gradient_clip} + +OMP_NUM_THREADS=4 python ./exps/algos/DARTS-V1.py \ + --save_dir ${save_dir} --max_nodes ${max_nodes} --channel ${channel} --num_cells ${num_cells} \ + --dataset ${dataset} --data_path ${data_path} \ + --search_space_name ${space} \ + --config_path configs/nas-benchmark/algos/DARTS.config \ + --arch_nas_dataset ${TORCH_HOME}/NAS-Bench-201-v1_0-e61699.pth \ + --track_running_stats ${BN} --gradient_clip ${gradient_clip} \ + --arch_learning_rate 0.0003 --arch_weight_decay 0.001 \ + --workers 4 --print_freq 200 --rand_seed ${seed}