{ "cells": [ { "cell_type": "code", "execution_count": 240, "id": "45e05b72", "metadata": {}, "outputs": [], "source": [ "from nasbench201.search_model_darts_proj import TinyNetworkDartsProj\n", "import torch\n", "import torch.nn as nn\n", "from nasbench201.cell_operations import SearchSpaceNames\n", "import nasbench201.utils as ig_utils\n", "import torch.utils\n", "import torchvision.datasets as dset\n", "import numpy as np\n", "import copy" ] }, { "cell_type": "code", "execution_count": 241, "id": "eaa02532", "metadata": {}, "outputs": [], "source": [ "np.random.seed(1)\n", "torch.manual_seed(1)\n", "torch.cuda.manual_seed(1)" ] }, { "cell_type": "code", "execution_count": 242, "id": "29976057", "metadata": {}, "outputs": [], "source": [ "class AGRS():\n", " def __init__(self):\n", " self.data = '../data'\n", " self.dataset = 'cifar10'\n", " self.train_portion = 0.5\n", " self.batch_size = 64\n", " self.init_channels=16\n", " self.layers = 8\n", " self.learning_rate = 0.025\n", " self.learning_rate_min = 0.001\n", " self.momentum = 0.9\n", " self.nesterov = False\n", " self.weight_decay = 3e-4\n", " self.grad_clip = 5\n", " self.cutout = False\n", "args = AGRS()" ] }, { "cell_type": "code", "execution_count": 243, "id": "3725b779", "metadata": {}, "outputs": [], "source": [ "def Jocab_Score(ori_model, input, target, weights=None):\n", " model = copy.deepcopy(ori_model)\n", " model.eval()\n", " model.proj_weights = weights\n", " num_edge, num_op = model.num_edge, model.num_op\n", " for i in range(num_edge):\n", " model.candidate_flags[i] = False\n", " batch_size = input.shape[0]\n", " model.K = torch.zeros(batch_size, batch_size).cuda()\n", " model.K_list = {}\n", " def counting_forward_hook(module, inp, out):\n", " if isinstance(inp, tuple):\n", " inp = inp[0]\n", " inp = inp.view(inp.size(0), -1)\n", " x = (inp > 0).float()\n", " K = x @ x.t()\n", " if x.cpu().numpy().sum() == 0:\n", " model.K = model.K\n", " else:\n", " K2 = (1.-x) @ (1.-x.t())\n", " model.K = model.K + K + K2\n", " model.K_list[module.name]=K\n", " #print(module)\n", " \n", "\n", " for name, module in model.named_modules():\n", " if isinstance(module, nn.ReLU):\n", " module.name = name\n", " module.register_forward_hook(counting_forward_hook)\n", " \n", " input = input.cuda()\n", " model(input)\n", " K = model.K.cpu().numpy()\n", " score = hooklogdet(model.K.cpu().numpy())\n", " #print(model.K_list)\n", " K_list = model.K_list\n", " del model\n", " del input\n", " return score, K,K_list\n", "\n", "def hooklogdet(K, labels=None):\n", " s, ld = np.linalg.slogdet(K)\n", " return ld" ] }, { "cell_type": "code", "execution_count": 244, "id": "ae134d08", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Files already downloaded and verified\n", "Files already downloaded and verified\n" ] } ], "source": [ "train_transform, valid_transform = ig_utils._data_transforms_cifar10(args)\n", "train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)\n", "valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)" ] }, { "cell_type": "code", "execution_count": 245, "id": "cd9923d8", "metadata": {}, "outputs": [], "source": [ "num_train = len(train_data)\n", "indices = list(range(num_train))\n", "split = 64\n", "\n", "train_queue = torch.utils.data.DataLoader(\n", " train_data, batch_size=args.batch_size,\n", " sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),\n", " pin_memory=True)\n", "input, target = next(iter(train_queue))" ] }, { "cell_type": "code", "execution_count": 1529, "id": "e08b4613", "metadata": {}, "outputs": [], "source": [ "np.random.seed(2)\n", "torch.manual_seed(2)\n", "torch.cuda.manual_seed(2)\n", "from scipy.stats import rankdata\n", "input, target = next(iter(train_queue))\n", "LAYER=8\n", "OPN=4" ] }, { "cell_type": "code", "execution_count": 1530, "id": "58c0ad9a", "metadata": {}, "outputs": [], "source": [ "from nasbench201.cell_operations import OPS\n", "class TinyNetwork(nn.Module):\n", " def __init__(self, C, N, num_classes, criterion, affine=False, track_running_stats=True, stem_channels=3):\n", " super(TinyNetwork, self).__init__()\n", " self.stem = nn.Sequential(\n", " nn.Conv2d(stem_channels, C, kernel_size=3, padding=1, bias=False),\n", " nn.BatchNorm2d(C))\n", " op_names=['skip_connect','nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3']\n", " self.N=N\n", " self.edges = nn.ModuleDict()\n", " for i in range(N):\n", " self.edges[str(i)]=nn.ModuleList([OPS[op_name](C, C, 1, affine, track_running_stats) for op_name in op_names])\n", " \n", " self.lastact = nn.Sequential(nn.BatchNorm2d(C), nn.ReLU(inplace=True))\n", " self.global_pooling = nn.AdaptiveAvgPool2d(1)\n", " self.classifier = nn.Linear(C, num_classes)\n", " self.cos = nn.CosineSimilarity(dim=1, eps=1e-6)\n", " self.weights=[[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4]]\n", " self.weights = np.array(self.weights)\n", " self.weights=torch.from_numpy(self.weights)\n", " \n", " def forward(self, inputs):\n", " weights = self.weights\n", " sum_value=[]\n", " feature = self.stem(inputs)\n", " for i in range(self.N):\n", " feature=sum(op(feature, block_input=True)*w if w==0 else op(feature) * w for op, w in zip(self.edges[str(i)], weights[i]))\n", "# with torch.no_grad():\n", "# print(self.calc_k(feature))\n", "# print(torch.mean(torch.abs(feature)))\n", "# print(torch.count_nonzero((feature>0).float()))\n", " \n", " out = self.lastact(feature)\n", " out = self.global_pooling( out )\n", " out = out.view(out.size(0), -1)\n", " logits = self.classifier(out)\n", " #print(sum_value)\n", " #print('model end')\n", " return logits\n", " \n", " def calc_k(self, inp):\n", " inp = inp.view(inp.size(0), -1)\n", " x = (inp > 0).float()\n", " K = x @ x.t()\n", " if x.cpu().numpy().sum() == 0:\n", " return 0\n", " else:\n", " K2 = (1.-x) @ (1.-x.t())\n", " K = K + K2\n", " return hooklogdet(K.cpu().numpy())\n", " \n" ] }, { "cell_type": "code", "execution_count": 1533, "id": "b84ded93", "metadata": {}, "outputs": [], "source": [ "model = TinyNetwork(C=16, N=LAYER, num_classes=10, criterion=nn.CrossEntropyLoss())\n", "#model.cuda()" ] }, { "cell_type": "code", "execution_count": 1534, "id": "baddff20", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/hongkaiw/anaconda2/envs/mct/lib/python3.7/site-packages/torch/tensor.py:593: RuntimeWarning: Iterating over a tensor might cause the trace to be incorrect. Passing a tensor of different shape won't change the number of iterations executed (and might lead to errors or silently give incorrect results).\n", " 'incorrect results).', category=RuntimeWarning)\n", "/home/hongkaiw/.local/lib/python3.7/site-packages/ipykernel_launcher.py:27: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n" ] } ], "source": [ "import torch.onnx \n", "model.eval()\n", " \n", "torch.onnx.export(model, # model being run \n", " input, # model input (or a tuple for multiple inputs) \n", " \"toy.onnx\", # where to save the model \n", " export_params=False, # store the trained parameter weights inside the model file \n", " opset_version=10, # the ONNX version to export the model to \n", " do_constant_folding=True, # whether to execute constant folding for optimization \n", " input_names = ['modelInput'], # the model's input names \n", " output_names = ['modelOutput'], # the model's output names \n", " dynamic_axes={'modelInput' : {1 : 'batch_size'}, # variable length axes \n", " 'modelOutput' : {0 : 'batch_size'}}) " ] }, { "cell_type": "code", "execution_count": 1522, "id": "78f85e24", "metadata": {}, "outputs": [], "source": [ "import torch.nn as nn\n", "def Jocab_Score(ori_model, input, target, weights=None):\n", " model = copy.deepcopy(ori_model)\n", " model.eval()\n", " model.proj_weights = weights\n", " batch_size = input.shape[0]\n", " model.K = torch.zeros(batch_size, batch_size).cuda()\n", " model.K_list = {}\n", " model.count = 0\n", " def counting_forward_hook(module, inp, out):\n", " if isinstance(inp, tuple):\n", " inp = inp[0]\n", " inp = inp.view(inp.size(0), -1)\n", " #with torch.no_grad():\n", " #print(torch.sum((inp > 0).float()), torch.count_nonzero(inp))\n", " x = (inp > 0).float()\n", " K = x @ x.t()\n", " if x.cpu().numpy().sum() == 0:\n", " model.K = model.K\n", " else:\n", " K2 = (1.-x) @ (1.-x.t())\n", " model.K = model.K + K + K2\n", " model.K_list[module.name]=K\n", " #print(module)\n", " \n", "\n", " for name, module in model.named_modules():\n", " if isinstance(module, nn.ReLU):\n", " #if 'ReLU' in str(type(module)):\n", " module.name = name\n", " #print(module)\n", " model.count+=1\n", " module.register_forward_hook(counting_forward_hook)\n", " \n", " input = input.cuda()\n", " model(input, weights)\n", " K = model.K.cpu().numpy()\n", " score = hooklogdet(model.K.cpu().numpy())\n", " #print(model.K_list)\n", " K_list = model.K_list\n", " #print(model.count)\n", " del model\n", " del input\n", " return score, K,K_list\n", "\n", "def hooklogdet(K, labels=None):\n", " s, ld = np.linalg.slogdet(K)\n", " return ld" ] }, { "cell_type": "code", "execution_count": 1523, "id": "42ae5a33", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2\n", "0\n", "1\n", "1\n", "1\n", "1\n", "1\n", "2\n", "[724.5167, 724.62964, 721.42676, 726.57513, 721.924, 724.0039, 724.2308, 724.328, 724.6446, 723.4378, 723.59174, 726.2936, 726.9928, 722.4523, 723.66644, 727.96545, 727.3341, 722.5211, 723.89703, 727.9818, 726.8876, 723.4647, 724.414, 727.43134, 727.0108, 724.00287, 724.3993, 727.34503, 727.5785, 724.18317, 724.01965, 727.66486]\n", "[2.625 1.5 1.875 4. ]\n", "1\n" ] } ], "source": [ "weights=[[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4],[1/4, 1/4, 1/4, 1/4]]\n", "\n", "\n", "pt_score = []\n", "avg_skip_rank=np.array([0.0,0.0,0.0,0.0])\n", "count_skip = 0\n", "crit,K,K_list = Jocab_Score(model, input, target, weights)\n", "for l in range(LAYER):\n", " op_s = []\n", " for o in range(OPN):\n", " w = copy.deepcopy(weights)\n", " w[l][o]=0\n", " crit,K,K_list = Jocab_Score(model, input, target, w)\n", " pt_score.append(crit)\n", " op_s.append(crit)\n", " avg_skip_rank +=(rankdata(op_s))\n", " select=np.argmin(op_s)\n", " print(select)\n", " if select ==0:\n", " count_skip+=1\n", "print(pt_score)\n", "print(avg_skip_rank/LAYER)\n", "print(count_skip)" ] }, { "cell_type": "code", "execution_count": 1467, "id": "3133fd42", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2\n", "[557.9228, 597.0493, 597.9796, 550.9689]\n", "[3. 2. 1. 4.]\n", "0\n" ] } ], "source": [ "disc_score = []\n", "avg_skip_rank=np.array([0.0,0.0,0.0,0.0])\n", "count_skip=0\n", "for l in range(LAYER):\n", " op_s = []\n", " for o in range(OPN):\n", " w = copy.deepcopy(weights)\n", " w[l]=np.zeros_like(w[l])\n", " w[l][o]=1\n", " #w[l][0]=1\n", " crit,K,K_list = Jocab_Score(model, input, target, w)\n", " #print(w)\n", " op_s.append(crit)\n", " disc_score.append(crit)\n", " #print([5-x for x in rankdata(op_s)])\n", " avg_skip_rank +=(5-rankdata(op_s))\n", " select=np.argmax(op_s)\n", " print(select)\n", " if select ==0:\n", " count_skip+=1\n", "print(disc_score)\n", "print(avg_skip_rank/LAYER)\n", "print(count_skip)" ] }, { "cell_type": "code", "execution_count": 1468, "id": "e091015c", "metadata": {}, "outputs": [], "source": [ "w = copy.deepcopy(weights)\n", "arch=[1,1,1,1,1,1,1,1]\n", "for i in range(len(arch)): \n", " w[i]=np.zeros_like(w[i])\n", " w[i][arch[i]]=1\n", "crit,K,K_list = Jocab_Score(model, input, target, w)\n" ] }, { "cell_type": "code", "execution_count": 1469, "id": "05ffd6a0", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7116abcb5edb4d99822e2a5f26b3cfb1", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/4 [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "[557.9228, 597.0493, 597.9796, 550.9689]\n" ] } ], "source": [ "from itertools import combinations_with_replacement,permutations,product\n", "from tqdm.notebook import tqdm\n", "final_score = []\n", "archs=[]\n", "archs =list(product([0,1,2,3], repeat=LAYER)) \n", " \n", "archs = [list(x) for x in archs]\n", "#print(archs)\n", "for i in tqdm(range(len(archs))):\n", " arch = archs[i]\n", " w = copy.deepcopy(weights)\n", " for i in range(len(arch)):\n", " w[i]=np.zeros_like(w[i])\n", " w[i][arch[i]]=1\n", " crit,K,K_list = Jocab_Score(model, input, target, w)\n", " final_score.append(crit)\n", "print(final_score)" ] }, { "cell_type": "code", "execution_count": 1470, "id": "24aab655", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | arch | \n", "naswot | \n", "
---|---|---|
0 | \n", "[0] | \n", "557.922791 | \n", "
1 | \n", "[1] | \n", "597.049316 | \n", "
2 | \n", "[2] | \n", "597.979614 | \n", "
3 | \n", "[3] | \n", "550.968872 | \n", "