diff --git a/configs/config.yaml b/configs/config.yaml index e42aea5..ce858e4 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -2,20 +2,23 @@ general: name: 'graph_dit' wandb: 'disabled' gpus: 1 - gpu_number: 2 + gpu_number: 0 resume: null test_only: null sample_every_val: 2500 - samples_to_generate: 512 + samples_to_generate: 1000 samples_to_save: 3 chains_to_save: 1 log_every_steps: 50 number_chain_steps: 8 - final_model_samples_to_generate: 100 + final_model_samples_to_generate: 1000 final_model_samples_to_save: 20 final_model_chains_to_save: 1 enable_progress_bar: False save_model: True + log_dir: '/nfs/data3/hanzhang/nasbenchDiT' + number_checkpoint_limit: 3 + type: 'Trainer' model: type: 'discrete' transition: 'marginal' @@ -32,7 +35,7 @@ model: ensure_connected: True train: # n_epochs: 5000 - n_epochs: 500 + n_epochs: 10 batch_size: 1200 lr: 0.0002 clip_grad: null @@ -41,8 +44,11 @@ train: seed: 0 val_check_interval: null check_val_every_n_epoch: 1 + gradient_accumulation_steps: 1 dataset: datadir: 'data/' task_name: 'nasbench-201' guidance_target: 'nasbench-201' pin_memory: False +ppo: + clip_param: 1 diff --git a/graph_dit/analysis/rdkit_functions.py b/graph_dit/analysis/rdkit_functions.py index 9a28cee..e23bdb8 100644 --- a/graph_dit/analysis/rdkit_functions.py +++ b/graph_dit/analysis/rdkit_functions.py @@ -54,7 +54,9 @@ class BasicGraphMetrics(object): covered_nodes = set() direct_valid_count = 0 print(f"generated number: {len(generated)}") + print(f"generated: {generated}") for graph in generated: + print(f"graph: {graph}") node_types, edge_types = graph direct_valid_flag = True direct_valid_count += 1 diff --git a/graph_dit/datasets/dataset.py b/graph_dit/datasets/dataset.py index 2ba84b1..5e50bc2 100644 --- a/graph_dit/datasets/dataset.py +++ b/graph_dit/datasets/dataset.py @@ -815,8 +815,8 @@ class Dataset(InMemoryDataset): train_loader = dt.get_data(args.dataset, args.data_loc, args.trainval, args.batch_size, args.augtype, args.repeat, args) self.swap_scores = [] import csv - # with open('/nfs/data3/hanzhang/nasbenchDiT/graph_dit/swap_results.csv', 'r') as f: - with open('/nfs/data3/hanzhang/nasbenchDiT/graph_dit/swap_results_cifar100.csv', 'r') as f: + with open('/nfs/data3/hanzhang/nasbenchDiT/graph_dit/swap_results.csv', 'r') as f: + # with open('/nfs/data3/hanzhang/nasbenchDiT/graph_dit/swap_results_cifar100.csv', 'r') as f: reader = csv.reader(f) header = next(reader) data = [row for row in reader] diff --git a/graph_dit/diffusion_model.py b/graph_dit/diffusion_model.py index a3f0993..b8dfc33 100644 --- a/graph_dit/diffusion_model.py +++ b/graph_dit/diffusion_model.py @@ -23,6 +23,9 @@ class Graph_DiT(pl.LightningModule): self.test_only = cfg.general.test_only self.guidance_target = getattr(cfg.dataset, 'guidance_target', None) + from nas_201_api import NASBench201API as API + self.api = API('/nfs/data3/hanzhang/nasbenchDiT/graph_dit/NAS-Bench-201-v1_1-096897.pth') + input_dims = dataset_infos.input_dims output_dims = dataset_infos.output_dims nodes_dist = dataset_infos.nodes_dist @@ -79,6 +82,7 @@ class Graph_DiT(pl.LightningModule): self.node_dist = nodes_dist self.active_index = active_index self.dataset_info = dataset_infos + self.cur_epoch = 0 self.train_loss = TrainLossDiscrete(self.cfg.model.lambda_train) @@ -162,25 +166,81 @@ class Graph_DiT(pl.LightningModule): return pred def training_step(self, data, i): - data_x = F.one_hot(data.x, num_classes=12).float()[:, self.active_index] - data_edge_attr = F.one_hot(data.edge_attr, num_classes=2).float() + if self.cfg.general.type != 'accelerator' and self.current_epoch > self.cfg.train.n_epochs / 5 * 4: + samples_left_to_generate = self.cfg.general.samples_to_generate + samples_left_to_save = self.cfg.general.samples_to_save + chains_left_to_save = self.cfg.general.chains_to_save - dense_data, node_mask = utils.to_dense(data_x, data.edge_index, data_edge_attr, data.batch, self.max_n_nodes) - dense_data = dense_data.mask(node_mask) - X, E = dense_data.X, dense_data.E - noisy_data = self.apply_noise(X, E, data.y, node_mask) - pred = self.forward(noisy_data) - loss = self.train_loss(masked_pred_X=pred.X, masked_pred_E=pred.E, pred_y=pred.y, - true_X=X, true_E=E, true_y=data.y, node_mask=node_mask, + samples, all_ys, batch_id = [], [], 0 + + def graph_reward_fn(graphs, true_graphs=None, device=None, reward_model='swap'): + rewards = [] + if reward_model == 'swap': + import csv + with open('/nfs/data3/hanzhang/nasbenchDiT/graph_dit/swap_results.csv', 'r') as f: + reader = csv.reader(f) + header = next(reader) + data = [row for row in reader] + swap_scores = [float(row[0]) for row in data] + for graph in graphs: + node_tensor = graph[0] + node = node_tensor.cpu().numpy().tolist() + + def nodes_to_arch_str(nodes): + num_to_op = ['input', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3', 'skip_connect', 'none', 'output'] + nodes_str = [num_to_op[node] for node in nodes] + arch_str = '|' + nodes_str[1] + '~0|+' + \ + '|' + nodes_str[2] + '~0|' + nodes_str[3] + '~1|+' +\ + '|' + nodes_str[4] + '~0|' + nodes_str[5] + '~1|' + nodes_str[6] + '~2|' + return arch_str + + arch_str = nodes_to_arch_str(node) + reward = swap_scores[self.api.query_index_by_arch(arch_str)] + rewards.append(reward) + return torch.tensor(rewards, dtype=torch.float32, requires_grad=True).unsqueeze(0).to(device) + old_log_probs = None + + bs = 1 * self.cfg.train.batch_size + to_generate = min(samples_left_to_generate, bs) + to_save = min(samples_left_to_save, bs) + chains_save = min(chains_left_to_save, bs) + # batch_y = test_y_collection[batch_id : batch_id + to_generate] + batch_y = torch.ones(to_generate, self.ydim_output, device=self.device) + + cur_sample, log_probs = self.sample_batch(batch_id, to_generate, batch_y, save_final=to_save, + keep_chain=chains_save, number_chain_steps=self.number_chain_steps) + # samples = samples + cur_sample + samples.append(cur_sample) + reward = graph_reward_fn(cur_sample, device=self.device) + advantages = (reward - torch.mean(reward)) / (torch.std(reward) + 1e-6) # + if old_log_probs is None: + old_log_probs = log_probs.clone() + ratio = torch.exp(log_probs - old_log_probs) + print(f"ratio: {ratio.shape}, advantages: {advantages.shape}") + unclipped_loss = -advantages * ratio + clipped_loss = -advantages * torch.clamp(ratio, 1.0 - self.cfg.ppo.clip_param, 1.0 + self.cfg.ppo.clip_param) + loss = torch.mean(torch.max(unclipped_loss, clipped_loss)) + return {'loss': loss} + else: + data_x = F.one_hot(data.x, num_classes=12).float()[:, self.active_index] + data_edge_attr = F.one_hot(data.edge_attr, num_classes=2).float() + + dense_data, node_mask = utils.to_dense(data_x, data.edge_index, data_edge_attr, data.batch, self.max_n_nodes) + dense_data = dense_data.mask(node_mask) + X, E = dense_data.X, dense_data.E + noisy_data = self.apply_noise(X, E, data.y, node_mask) + pred = self.forward(noisy_data) + loss = self.train_loss(masked_pred_X=pred.X, masked_pred_E=pred.E, pred_y=pred.y, + true_X=X, true_E=E, true_y=data.y, node_mask=node_mask, + log=i % self.log_every_steps == 0) + # print(f'training loss: {loss}, epoch: {self.current_epoch}, batch: {i}\n, pred type: {type(pred)}, pred.X shape: {type(pred.X)}, {pred.X.shape}, pred.E shape: {type(pred.E)}, {pred.E.shape}') + self.train_metrics(masked_pred_X=pred.X, masked_pred_E=pred.E, true_X=X, true_E=E, log=i % self.log_every_steps == 0) - # print(f'training loss: {loss}, epoch: {self.current_epoch}, batch: {i}\n, pred type: {type(pred)}, pred.X shape: {type(pred.X)}, {pred.X.shape}, pred.E shape: {type(pred.E)}, {pred.E.shape}') - self.train_metrics(masked_pred_X=pred.X, masked_pred_E=pred.E, true_X=X, true_E=E, - log=i % self.log_every_steps == 0) - self.log(f'loss', loss, batch_size=X.size(0), sync_dist=True) - print(f"training loss: {loss}") - with open("training-loss.csv", "a") as f: - f.write(f"{loss}, {i}\n") - return {'loss': loss} + self.log(f'loss', loss, batch_size=X.size(0), sync_dist=True) + print(f"training loss: {loss}") + with open("training-loss.csv", "a") as f: + f.write(f"{loss}, {i}\n") + return {'loss': loss} def configure_optimizers(self): @@ -196,14 +256,15 @@ class Graph_DiT(pl.LightningModule): def on_train_epoch_start(self) -> None: if self.current_epoch / self.trainer.max_epochs in [0.25, 0.5, 0.75, 1.0]: - print("Starting train epoch {}/{}...".format(self.current_epoch, self.trainer.max_epochs)) + # if self.cur_epoch / self.cfg.train.n_epochs in [0.25, 0.5, 0.75, 1.0]: + print("Starting train epoch {}/{}...".format(self.cur_epoch, self.cfg.train.n_epochs)) self.start_epoch_time = time.time() self.train_loss.reset() self.train_metrics.reset() def on_train_epoch_end(self) -> None: - if self.current_epoch / self.trainer.max_epochs in [0.25, 0.5, 0.75, 1.0]: + if self.current_epoch / self.cfg.train.n_epochs in [0.25, 0.5, 0.75, 1.0]: log = True else: log = False @@ -240,6 +301,7 @@ class Graph_DiT(pl.LightningModule): self.val_X_logp.compute(), self.val_E_logp.compute()] if self.current_epoch / self.trainer.max_epochs in [0.25, 0.5, 0.75, 1.0]: + # if self.cur_epoch / self.cfg.train.n_epochs in [0.25, 0.5, 0.75, 1.0]: print(f"Epoch {self.current_epoch}: Val NLL {metrics[0] :.2f} -- Val Atom type KL {metrics[1] :.2f} -- ", f"Val Edge type KL: {metrics[2] :.2f}", 'Val loss: %.2f \t Best : %.2f\n' % (metrics[0], self.best_val_nll)) with open("validation-metrics.csv", "a") as f: @@ -336,7 +398,7 @@ class Graph_DiT(pl.LightningModule): print(f"Epoch {self.current_epoch}: Test NLL {metrics[0] :.2f} -- Test Atom type KL {metrics[1] :.2f} -- ", f"Test Edge type KL: {metrics[2] :.2f}") - ## final epcoh + ## final epoch samples_left_to_generate = self.cfg.general.final_model_samples_to_generate samples_left_to_save = self.cfg.general.final_model_samples_to_save chains_left_to_save = self.cfg.general.final_model_chains_to_save @@ -359,9 +421,9 @@ class Graph_DiT(pl.LightningModule): # batch_y = test_y_collection[batch_id : batch_id + to_generate] batch_y = torch.ones(to_generate, self.ydim_output, device=self.device) - cur_sample = self.sample_batch(batch_id, to_generate, batch_y, save_final=to_save, + cur_sample, log_probs = self.sample_batch(batch_id, to_generate, batch_y, save_final=to_save, keep_chain=chains_save, number_chain_steps=self.number_chain_steps) - samples = samples + cur_sample + samples.append(cur_sample) all_ys.append(batch_y) batch_id += to_generate @@ -601,6 +663,12 @@ class Graph_DiT(pl.LightningModule): assert (E == torch.transpose(E, 1, 2)).all() + if self.cfg.general.type != 'accelerator': + if self.trainer.training or self.trainer.validating: + total_log_probs = torch.zeros([self.cfg.general.samples_to_generate, 10], device=self.device) + elif self.trainer.testing: + total_log_probs = torch.zeros([self.cfg.general.final_model_samples_to_generate, 10], device=self.device) + # Iteratively sample p(z_s | z_t) for t = 1, ..., T, with s = t - 1. for s_int in reversed(range(0, self.T)): s_array = s_int * torch.ones((batch_size, 1)).type_as(y) @@ -609,21 +677,24 @@ class Graph_DiT(pl.LightningModule): t_norm = t_array / self.T # Sample z_s - sampled_s, discrete_sampled_s = self.sample_p_zs_given_zt(s_norm, t_norm, X, E, y, node_mask) + sampled_s, discrete_sampled_s, log_probs = self.sample_p_zs_given_zt(s_norm, t_norm, X, E, y, node_mask) X, E, y = sampled_s.X, sampled_s.E, sampled_s.y + total_log_probs += log_probs # Sample sampled_s = sampled_s.mask(node_mask, collapse=True) X, E, y = sampled_s.X, sampled_s.E, sampled_s.y - molecule_list = [] + graph_list = [] for i in range(batch_size): n = n_nodes[i] - atom_types = X[i, :n].cpu() + node_types = X[i, :n].cpu() edge_types = E[i, :n, :n].cpu() - molecule_list.append([atom_types, edge_types]) + graph_list.append((node_types , edge_types)) - return molecule_list + total_log_probs = torch.sum(total_log_probs, dim=-1) + + return graph_list, total_log_probs def sample_p_zs_given_zt(self, s, t, X_t, E_t, y_t, node_mask): """Samples from zs ~ p(zs | zt). Only used during sampling. @@ -675,6 +746,14 @@ class Graph_DiT(pl.LightningModule): # with condition = P_t(A_{t-1} |A_t, y) prob_X, prob_E, pred = get_prob(noisy_data) + log_prob_X = torch.log(torch.gather(prob_X, -1, X_t.long()).squeeze(-1)) # bs, n + log_prob_E = torch.log(torch.gather(prob_E, -1, E_t.long()).squeeze(-1)) # bs, n, n + + # Sum the log_prob across dimensions for total log_prob + log_prob_X = log_prob_X.sum(dim=-1) + log_prob_E = log_prob_E.sum(dim=(1, 2)) + + log_probs = torch.cat([log_prob_X, log_prob_E], dim=-1) ### Guidance if self.guidance_target is not None and self.guide_scale is not None and self.guide_scale != 1: uncon_prob_X, uncon_prob_E, pred = get_prob(noisy_data, unconditioned=True) @@ -810,4 +889,4 @@ class Graph_DiT(pl.LightningModule): out_one_hot = utils.PlaceHolder(X=X_s, E=E_s, y=y_t) out_discrete = utils.PlaceHolder(X=X_s, E=E_s, y=y_t) - return out_one_hot.mask(node_mask).type_as(y_t), out_discrete.mask(node_mask, collapse=True).type_as(y_t) + return out_one_hot.mask(node_mask).type_as(y_t), out_discrete.mask(node_mask, collapse=True).type_as(y_t), log_probs diff --git a/graph_dit/main.py b/graph_dit/main.py index f3d89e5..bb92456 100644 --- a/graph_dit/main.py +++ b/graph_dit/main.py @@ -177,32 +177,92 @@ def test(cfg: DictConfig): os.chdir(cfg.general.resume.split("checkpoints")[0]) # os.environ["CUDA_VISIBLE_DEVICES"] = cfg.general.gpu_number model = Graph_DiT(cfg=cfg, **model_kwargs) - trainer = Trainer( - gradient_clip_val=cfg.train.clip_grad, - # accelerator="cpu", - accelerator="gpu" - if torch.cuda.is_available() and cfg.general.gpus > 0 - else "cpu", - devices=[cfg.general.gpu_number] - if torch.cuda.is_available() and cfg.general.gpus > 0 - else None, - max_epochs=cfg.train.n_epochs, - enable_checkpointing=False, - check_val_every_n_epoch=cfg.train.check_val_every_n_epoch, - val_check_interval=cfg.train.val_check_interval, - strategy="ddp" if cfg.general.gpus > 1 else "auto", - enable_progress_bar=cfg.general.enable_progress_bar, - callbacks=[], - reload_dataloaders_every_n_epochs=0, - logger=[], - ) - if not cfg.general.test_only: - print("start testing fit method") - trainer.fit(model, datamodule=datamodule, ckpt_path=cfg.general.resume) - if cfg.general.save_model: - trainer.save_checkpoint(f"checkpoints/{cfg.general.name}/last.ckpt") - trainer.test(model, datamodule=datamodule) + if cfg.general.type == "accelerator": + graph_dit_model = model + + from accelerate import Accelerator + from accelerate.utils import set_seed, ProjectConfiguration + + accelerator_config = ProjectConfiguration( + project_dir=os.path.join(cfg.general.log_dir, cfg.general.name), + automatic_checkpoint_naming=True, + total_limit=cfg.general.number_checkpoint_limit, + ) + accelerator = Accelerator( + mixed_precision='no', + project_config=accelerator_config, + # gradient_accumulation_steps=cfg.train.gradient_accumulation_steps * cfg.train.n_epochs, + gradient_accumulation_steps=cfg.train.gradient_accumulation_steps, + ) + + optimizer = graph_dit_model.configure_optimizers() + + train_dataloader = datamodule.train_dataloader() + train_dataloader = accelerator.prepare(train_dataloader) + val_dataloader = datamodule.val_dataloader() + val_dataloader = accelerator.prepare(val_dataloader) + test_dataloader = datamodule.test_dataloader() + test_dataloader = accelerator.prepare(test_dataloader) + + optimizer, graph_dit_model = accelerator.prepare(optimizer, graph_dit_model) + + # train_epoch + from pytorch_lightning import seed_everything + seed_everything(cfg.train.seed) + for epoch in range(cfg.train.n_epochs): + print(f"Epoch {epoch}") + graph_dit_model.train() + graph_dit_model.cur_epoch = epoch + graph_dit_model.on_train_epoch_start() + for batch in train_dataloader: + optimizer.zero_grad() + loss = graph_dit_model.training_step(batch, epoch)['loss'] + accelerator.backward(loss) + optimizer.step() + graph_dit_model.on_train_epoch_end() + for batch in val_dataloader: + if epoch % cfg.train.check_val_every_n_epoch == 0: + graph_dit_model.eval() + graph_dit_model.on_validation_epoch_start() + graph_dit_model.validation_step(batch, epoch) + graph_dit_model.on_validation_epoch_end() + + # test_epoch + + graph_dit_model.test() + graph_dit_model.on_test_epoch_start() + for batch in test_dataloader: + graph_dit_model.test_step(batch, epoch) + graph_dit_model.on_test_epoch_end() + + elif cfg.general.type == "Trainer": + trainer = Trainer( + gradient_clip_val=cfg.train.clip_grad, + # accelerator="cpu", + accelerator="gpu" + if torch.cuda.is_available() and cfg.general.gpus > 0 + else "cpu", + devices=[cfg.general.gpu_number] + if torch.cuda.is_available() and cfg.general.gpus > 0 + else None, + max_epochs=cfg.train.n_epochs, + enable_checkpointing=False, + check_val_every_n_epoch=cfg.train.check_val_every_n_epoch, + val_check_interval=cfg.train.val_check_interval, + strategy="ddp" if cfg.general.gpus > 1 else "auto", + enable_progress_bar=cfg.general.enable_progress_bar, + callbacks=[], + reload_dataloaders_every_n_epochs=0, + logger=[], + ) + + if not cfg.general.test_only: + print("start testing fit method") + trainer.fit(model, datamodule=datamodule, ckpt_path=cfg.general.resume) + if cfg.general.save_model: + trainer.save_checkpoint(f"checkpoints/{cfg.general.name}/last.ckpt") + trainer.test(model, datamodule=datamodule) if __name__ == "__main__": test() diff --git a/graph_dit/test_nasbench.ipynb b/graph_dit/test_nasbench.ipynb index f34e5bf..3598de5 100644 --- a/graph_dit/test_nasbench.ipynb +++ b/graph_dit/test_nasbench.ipynb @@ -11,9 +11,18 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/stud/hanzhang/anaconda3/envs/graphdit/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "import sys\n", "sys.path.append('../') \n", @@ -34,6 +43,89 @@ "from sklearn.model_selection import train_test_split\n" ] }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([100, 1])\n" + ] + } + ], + "source": [ + "tensor1 = torch.randn(100,10)\n", + "sums_tensor1 = torch.sum(tensor1, dim=-1).unsqueeze(1)\n", + "print(sums_tensor1.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[ 7.1564e-01, -3.5812e-02, -2.5924e-01],\n", + " [ 1.5302e-03, 1.0105e+00, 2.3484e+00]])\n", + "tensor([[-0.3566, -0.9514, -0.3267],\n", + " [-1.0915, -0.2063, 0.1615]])\n", + "torch.Size([4, 3])\n", + "tensor([[ 7.1564e-01, -3.5812e-02, -2.5924e-01],\n", + " [ 1.5302e-03, 1.0105e+00, 2.3484e+00],\n", + " [-3.5660e-01, -9.5144e-01, -3.2673e-01],\n", + " [-1.0915e+00, -2.0631e-01, 1.6153e-01]])\n", + "torch.Size([2, 6])\n", + "tensor([[ 7.1564e-01, -3.5812e-02, -2.5924e-01, -3.5660e-01, -9.5144e-01,\n", + " -3.2673e-01],\n", + " [ 1.5302e-03, 1.0105e+00, 2.3484e+00, -1.0915e+00, -2.0631e-01,\n", + " 1.6153e-01]])\n" + ] + } + ], + "source": [ + "tensor1 = torch.randn(2, 3)\n", + "tensor2 = torch.randn(2, 3)\n", + "print(tensor1)\n", + "print(tensor2)\n", + "result1 = torch.cat([tensor1, tensor2], dim=0)\n", + "print(result1.shape)\n", + "print(result1)\n", + "\n", + "result2 = torch.cat([tensor1, tensor2], dim=1)\n", + "print(result2.shape)\n", + "print(result2)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[tensor([[0.4206, 3.3604]]), tensor([[-1.6348, -1.1363]])]\n" + ] + } + ], + "source": [ + "tensors = [tensor1, tensor2]\n", + "for i in range(len(tensors)):\n", + " tensors[i] = torch.sum(tensors[i], dim=-1).unsqueeze(0)\n", + "print(tensors)\n", + "# for tensor in tensors:\n", + "# tensor = torch.sum(tensor).unsqueeze(0)\n", + "# print(tensor)\n", + "# print(tensors)" + ] + }, { "cell_type": "code", "execution_count": 2, @@ -15709,23 +15801,13 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Call the get_more_info function with index=5374, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", - "Call query_index_by_arch with arch=5374\n", - "train-loss 0.0018764479948580265\n", - "train-accuracy 99.983\n", - "train-per-time 24.43959446748098\n", - "train-all-time 4887.918893496196\n", - "test-loss 0.29164503993988034\n", - "test-accuracy 93.925\n", - "test-per-time 1.5713793436686203\n", - "test-all-time 314.27586873372405\n", "Call the get_more_info function with index=0, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", "Call query_index_by_arch with arch=0\n", "Call the get_more_info function with index=1, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", @@ -46976,6 +47058,7 @@ "Call query_index_by_arch with arch=15623\n", "Call the get_more_info function with index=15624, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", "Call query_index_by_arch with arch=15624\n", + "42 370 588\n", "Best architecture for cifar10: 6111 with accuracy 94.37333333333333\n" ] } @@ -46997,15 +47080,49 @@ "args = Args()\n", "best_idx = None\n", "best_value = None\n", + "scores_200 = []\n", + "\n", "for i in range(15625):\n", " res = api.get_more_info(i, args.dataset, None, hp=args.hp, is_random=args.is_random)\n", + " scores_200.append(res['test-accuracy'])\n", " if best_value is None or res['test-accuracy'] > best_value:\n", " best_value = res['test-accuracy']\n", " best_idx = i\n", + "scores_top1000 = sorted(scores_200, reverse=True)[:1000]\n", + "over94 = [x for x in scores_top1000 if x > 94]\n", + "over935 = [x for x in scores_top1000 if x > 93.5]\n", + "over93 = [x for x in scores_top1000 if x > 93]\n", + "print(len(over94), len(over935) - len(over94), len(over93)-len(over935))\n", "print(f'Best architecture for {args.dataset}: {best_idx} with accuracy {best_value}')\n", + "# get mean, max, min, std of the top 1000 scores\n", + "min_score = min(scores_top1000)\n", + "max_score = max(scores_top1000)\n", + "mean_score = np.mean(scores_top1000)\n", + "\n", "\n" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Min score: 93.18333333333334\n", + "Max score: 94.37333333333333\n", + "Mean score: 93.49814333333333\n" + ] + } + ], + "source": [ + "print(f'Min score: {min_score}') \n", + "print(f'Max score: {max_score}')\n", + "print(f'Mean score: {mean_score}')" + ] + }, { "cell_type": "code", "execution_count": 6, @@ -47469,519 +47586,422 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'nodes': [[0, 3, 5, 5, 1, 2, 2, 6], [0, 4, 2, 5, 2, 5, 2, 6], [0, 4, 1, 2, 2, 3, 1, 6], [0, 1, 1, 1, 2, 1, 3, 6], [0, 2, 1, 5, 1, 3, 1, 6], [0, 1, 3, 2, 5, 1, 4, 6], [0, 2, 2, 1, 5, 4, 2, 6], [0, 2, 2, 4, 3, 2, 3, 6], [0, 1, 2, 5, 4, 2, 2, 6], [0, 3, 4, 2, 1, 1, 1, 6], [0, 1, 5, 2, 2, 2, 3, 6], [0, 2, 5, 4, 2, 2, 2, 6], [0, 1, 5, 4, 5, 2, 2, 6], [0, 2, 3, 2, 1, 1, 2, 6], [0, 2, 1, 1, 5, 2, 3, 6], [0, 1, 2, 1, 3, 3, 1, 6], [0, 4, 2, 3, 4, 2, 2, 6], [0, 2, 2, 2, 4, 1, 3, 6], [0, 1, 2, 2, 1, 1, 3, 6], [0, 2, 1, 2, 2, 2, 1, 6], [0, 1, 3, 2, 1, 2, 1, 6], [0, 2, 2, 4, 1, 3, 2, 6], [0, 1, 2, 5, 1, 2, 1, 6], [0, 2, 1, 2, 1, 4, 4, 6], [0, 1, 1, 1, 2, 3, 5, 6], [0, 3, 5, 1, 1, 2, 2, 6], [0, 2, 1, 3, 3, 5, 2, 6], [0, 2, 1, 3, 2, 2, 2, 6], [0, 1, 4, 2, 2, 2, 3, 6], [0, 3, 1, 1, 2, 1, 5, 6], [0, 1, 1, 5, 3, 2, 2, 6], [0, 1, 2, 3, 4, 2, 5, 6], [0, 2, 5, 1, 2, 2, 1, 6], [0, 2, 1, 2, 1, 5, 4, 6], [0, 2, 2, 4, 2, 2, 3, 6], [0, 1, 1, 5, 2, 2, 3, 6], [0, 2, 2, 1, 2, 2, 3, 6], [0, 2, 5, 2, 4, 2, 2, 6], [0, 3, 1, 1, 3, 2, 2, 6], [0, 2, 1, 2, 2, 2, 3, 6], [0, 2, 2, 3, 3, 1, 1, 6], [0, 2, 2, 3, 2, 2, 1, 6], [0, 4, 4, 2, 1, 4, 1, 6], [0, 2, 1, 1, 5, 2, 4, 6], [0, 2, 2, 5, 3, 3, 2, 6], [0, 2, 1, 2, 1, 3, 1, 6], [0, 2, 3, 1, 1, 4, 5, 6], [0, 2, 2, 2, 2, 1, 3, 6], [0, 3, 2, 2, 2, 2, 1, 6], [0, 1, 2, 2, 2, 2, 2, 6], [0, 3, 1, 5, 2, 2, 2, 6], [0, 4, 2, 1, 2, 2, 1, 6], [0, 1, 2, 2, 2, 4, 3, 6], [0, 2, 1, 1, 2, 3, 3, 6], [0, 1, 2, 4, 5, 1, 5, 6], [0, 1, 2, 1, 1, 2, 4, 6], [0, 2, 3, 1, 1, 2, 1, 6], [0, 4, 1, 2, 1, 4, 1, 6], [0, 4, 2, 1, 2, 1, 5, 6], [0, 5, 2, 1, 2, 5, 2, 6], [0, 1, 2, 2, 3, 1, 3, 6], [0, 4, 1, 1, 2, 2, 2, 6], [0, 5, 2, 1, 5, 1, 1, 6], [0, 1, 2, 3, 3, 2, 2, 6], [0, 1, 1, 3, 1, 2, 5, 6], [0, 1, 4, 1, 4, 2, 1, 6], [0, 3, 1, 1, 3, 1, 2, 6], [0, 1, 1, 1, 5, 2, 2, 6], [0, 2, 1, 2, 5, 1, 5, 6], [0, 2, 1, 3, 3, 2, 1, 6], [0, 2, 5, 4, 2, 1, 2, 6], [0, 3, 2, 5, 1, 1, 2, 6], [0, 3, 1, 4, 3, 2, 1, 6], [0, 1, 2, 5, 2, 1, 2, 6], [0, 2, 2, 3, 1, 2, 1, 6], [0, 2, 2, 5, 1, 1, 2, 6], [0, 5, 1, 1, 2, 4, 2, 6], [0, 2, 1, 4, 2, 3, 1, 6], [0, 2, 1, 3, 5, 1, 1, 6], [0, 2, 4, 1, 1, 3, 1, 6], [0, 5, 2, 1, 2, 1, 2, 6], [0, 1, 1, 4, 1, 2, 3, 6], [0, 1, 4, 2, 2, 2, 5, 6], [0, 1, 1, 2, 3, 2, 1, 6], [0, 3, 2, 2, 1, 2, 1, 6], [0, 2, 5, 2, 4, 2, 1, 6], [0, 2, 2, 1, 1, 5, 1, 6], [0, 2, 1, 3, 3, 4, 2, 6], [0, 3, 2, 1, 2, 2, 3, 6], [0, 2, 2, 3, 2, 2, 5, 6], [0, 1, 2, 1, 1, 1, 3, 6], [0, 2, 1, 4, 1, 1, 2, 6], [0, 2, 2, 4, 2, 2, 3, 6], [0, 2, 3, 2, 1, 2, 1, 6], [0, 1, 2, 1, 1, 2, 4, 6], [0, 1, 4, 5, 5, 2, 2, 6], [0, 2, 4, 4, 1, 2, 2, 6], [0, 3, 3, 2, 2, 2, 1, 6], [0, 2, 5, 1, 2, 4, 1, 6], [0, 2, 2, 5, 2, 1, 3, 6]], 'edges': [[[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]]]}\n", + "{'nodes': [[0, 2, 2, 4, 4, 1, 2, 6], [0, 2, 2, 2, 4, 2, 4, 6], [0, 2, 1, 1, 3, 5, 3, 6], [0, 4, 3, 1, 1, 1, 1, 6], [0, 3, 2, 1, 1, 1, 3, 6], [0, 4, 1, 5, 5, 2, 2, 6], [0, 2, 4, 2, 2, 4, 4, 6], [0, 2, 5, 4, 5, 2, 5, 6], [0, 2, 1, 2, 1, 5, 1, 6], [0, 1, 5, 2, 1, 3, 1, 6], [0, 4, 2, 3, 4, 2, 2, 6], [0, 1, 3, 4, 4, 1, 2, 6], [0, 1, 2, 5, 2, 1, 2, 6], [0, 1, 4, 2, 1, 1, 4, 6], [0, 2, 1, 5, 2, 3, 1, 6], [0, 2, 1, 5, 2, 2, 5, 6], [0, 4, 1, 2, 1, 1, 1, 6], [0, 4, 2, 4, 1, 1, 1, 6], [0, 2, 4, 2, 1, 1, 4, 6], [0, 3, 2, 1, 1, 3, 1, 6], [0, 4, 3, 1, 1, 1, 5, 6], [0, 3, 2, 2, 2, 1, 1, 6], [0, 1, 2, 4, 1, 4, 5, 6], [0, 2, 5, 5, 2, 2, 4, 6], [0, 1, 1, 1, 1, 4, 5, 6], [0, 1, 2, 1, 3, 4, 2, 6], [0, 1, 1, 5, 1, 1, 1, 6], [0, 1, 1, 4, 1, 2, 5, 6], [0, 5, 1, 3, 1, 4, 1, 6], [0, 4, 2, 3, 2, 5, 1, 6], [0, 1, 4, 4, 1, 2, 5, 6], [0, 1, 3, 2, 2, 3, 5, 6], [0, 2, 1, 5, 1, 2, 4, 6], [0, 3, 5, 2, 2, 1, 1, 6], [0, 3, 2, 1, 1, 2, 5, 6], [0, 1, 4, 1, 5, 3, 1, 6], [0, 2, 3, 1, 3, 2, 4, 6], [0, 3, 2, 5, 5, 2, 2, 6], [0, 3, 2, 2, 4, 4, 1, 6], [0, 1, 1, 5, 1, 3, 4, 6], [0, 1, 5, 4, 1, 2, 1, 6], [0, 2, 1, 2, 1, 3, 4, 6], [0, 2, 1, 3, 2, 1, 2, 6], [0, 1, 2, 4, 5, 1, 3, 6], [0, 1, 4, 2, 2, 5, 2, 6], [0, 2, 5, 1, 4, 3, 1, 6], [0, 1, 4, 5, 1, 3, 2, 6], [0, 4, 4, 1, 1, 2, 5, 6], [0, 1, 4, 1, 5, 4, 2, 6], [0, 1, 4, 1, 2, 4, 1, 6], [0, 3, 2, 2, 3, 1, 3, 6], [0, 1, 1, 1, 4, 3, 2, 6], [0, 4, 4, 3, 5, 2, 2, 6], [0, 1, 4, 2, 3, 3, 2, 6], [0, 2, 2, 4, 2, 2, 4, 6], [0, 2, 4, 1, 5, 3, 1, 6], [0, 1, 4, 4, 5, 2, 1, 6], [0, 4, 1, 2, 1, 3, 1, 6], [0, 3, 2, 5, 1, 5, 5, 6], [0, 2, 2, 2, 5, 3, 4, 6], [0, 2, 1, 4, 2, 5, 3, 6], [0, 3, 5, 1, 3, 1, 1, 6], [0, 1, 2, 4, 5, 2, 3, 6], [0, 2, 4, 4, 1, 5, 2, 6], [0, 2, 4, 2, 1, 2, 3, 6], [0, 1, 2, 5, 5, 4, 4, 6], [0, 1, 2, 1, 4, 2, 5, 6], [0, 2, 3, 5, 2, 1, 5, 6], [0, 2, 5, 2, 2, 3, 4, 6], [0, 1, 3, 4, 1, 5, 2, 6], [0, 1, 1, 5, 2, 2, 2, 6], [0, 2, 1, 1, 3, 5, 2, 6], [0, 4, 4, 1, 2, 1, 1, 6], [0, 1, 1, 2, 5, 1, 2, 6], [0, 4, 2, 3, 1, 5, 2, 6], [0, 4, 2, 2, 2, 5, 1, 6], [0, 2, 5, 1, 1, 4, 5, 6], [0, 1, 3, 1, 2, 1, 3, 6], [0, 2, 5, 2, 3, 2, 2, 6], [0, 2, 2, 4, 1, 3, 1, 6], [0, 3, 2, 1, 4, 2, 2, 6], [0, 3, 3, 2, 1, 1, 5, 6], [0, 2, 2, 2, 5, 1, 3, 6], [0, 2, 4, 1, 1, 1, 1, 6], [0, 4, 1, 2, 1, 2, 4, 6], [0, 2, 2, 2, 4, 2, 3, 6], [0, 1, 4, 2, 4, 1, 2, 6], [0, 4, 2, 1, 5, 2, 1, 6], [0, 2, 2, 1, 1, 3, 3, 6], [0, 2, 2, 5, 4, 1, 2, 6], [0, 2, 4, 3, 5, 1, 5, 6], [0, 2, 1, 1, 2, 1, 5, 6], [0, 2, 4, 5, 1, 1, 2, 6], [0, 5, 1, 1, 4, 1, 2, 6], [0, 5, 1, 1, 1, 4, 3, 6], [0, 2, 4, 2, 2, 2, 1, 6], [0, 3, 3, 5, 2, 2, 2, 6], [0, 2, 1, 4, 3, 2, 3, 6], [0, 2, 2, 3, 1, 1, 2, 6], [0, 1, 3, 2, 5, 1, 5, 6]], 'edges': [[[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]], [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]]]}\n", "df\n", - "[0, 3, 5, 5, 1, 2, 2, 6] [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]]\n", - "[0, 3, 5, 5, 1, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|none~0|none~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=10117, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=10117, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|none~0|none~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|none~0|none~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 4, 2, 5, 2, 5, 2, 6]\n", - "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|none~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=4797, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=4797, hp=200\n", - "{777: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|none~1|nor_conv_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|none~1|nor_conv_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|none~1|nor_conv_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 4, 1, 2, 2, 3, 1, 6]\n", - "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=4218, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=4218, hp=200\n", - "{777: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 1, 1, 2, 1, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=3844, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=3844, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=58.90M, Param=0.428MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 5, 1, 3, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=5182, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=5182, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 3, 2, 5, 1, 4, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|skip_connect~2|\n", - "Call query_by_index with arch_index=2087, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=2087, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|skip_connect~2|, FLOP=51.04M, Param=0.372MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|skip_connect~2|, FLOP=51.04M, Param=0.372MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|skip_connect~2|, FLOP=51.04M, Param=0.372MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 1, 5, 4, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|none~0|skip_connect~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=14919, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=14919, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|none~0|skip_connect~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|none~0|skip_connect~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|none~0|skip_connect~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 4, 3, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|avg_pool_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=443, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=443, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|avg_pool_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|avg_pool_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|avg_pool_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 5, 4, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=4747, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=4747, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 3, 4, 2, 1, 1, 1, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=11473, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=11473, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 5, 2, 2, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|none~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=2819, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=2819, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|none~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|none~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|none~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 5, 4, 2, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=12184, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=12184, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 5, 4, 5, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|none~0|skip_connect~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=14194, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=14194, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|none~0|skip_connect~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|none~0|skip_connect~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|none~0|skip_connect~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 3, 2, 1, 1, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=11915, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=11915, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 1, 5, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=6096, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=6096, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 1, 3, 3, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|avg_pool_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=6465, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=6465, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|avg_pool_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|avg_pool_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test])}\n", + "[0, 2, 2, 4, 4, 1, 2, 6] [[0, 1, 1, 0, 1, 0, 0, 0], [1, 0, 0, 1, 0, 1, 0, 0], [1, 0, 0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0, 1, 0], [1, 0, 0, 0, 0, 0, 0, 1], [0, 1, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1, 0]]\n", + "[0, 2, 2, 4, 4, 1, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|skip_connect~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=8621, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=8621\n", + "[0, 2, 2, 2, 4, 2, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=13981, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=13981\n", + "[0, 2, 1, 1, 3, 5, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|avg_pool_3x3~0|none~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=11890, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=11890\n", + "[0, 4, 3, 1, 1, 1, 1, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=2289, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=2289\n", + "[0, 3, 2, 1, 1, 1, 3, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=11341, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=11341\n", + "[0, 4, 1, 5, 5, 2, 2, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_1x1~0|none~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=11828, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=11828\n", + "[0, 2, 4, 2, 2, 4, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|skip_connect~1|skip_connect~2|\n", + "Call the get_more_info function with index=1177, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1177\n", + "[0, 2, 5, 4, 5, 2, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|skip_connect~1|+|none~0|nor_conv_3x3~1|none~2|\n", + "Call the get_more_info function with index=11135, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=11135\n", + "[0, 2, 1, 2, 1, 5, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|none~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=1742, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1742\n", + "[0, 1, 5, 2, 1, 3, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|none~0|nor_conv_3x3~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=10425, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10425\n", "[0, 4, 2, 3, 4, 2, 2, 6]\n", "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=137, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=137, hp=200\n", - "{777: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 2, 4, 1, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=14152, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=14152, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 2, 1, 1, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=4825, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=4825, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 2, 2, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=12692, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=12692, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=157.21M, Param=1.101MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 3, 2, 1, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=2758, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=2758, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 4, 1, 3, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=14039, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=14039, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 5, 1, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=10324, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=10324, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 2, 1, 4, 4, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|skip_connect~2|\n", - "Call query_by_index with arch_index=551, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=551, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|skip_connect~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|skip_connect~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|skip_connect~2|, FLOP=86.43M, Param=0.615MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 1, 1, 2, 3, 5, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|avg_pool_3x3~1|none~2|\n", - "Call query_by_index with arch_index=12334, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=12334, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|avg_pool_3x3~1|none~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|avg_pool_3x3~1|none~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 3, 5, 1, 1, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=12926, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=12926, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 3, 3, 5, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|avg_pool_3x3~0|none~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=7162, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=7162, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|avg_pool_3x3~0|none~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|avg_pool_3x3~0|none~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 3, 2, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=5315, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=5315, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=153.27M, Param=1.073MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=153.27M, Param=1.073MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 4, 2, 2, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=1577, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=1577, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 3, 1, 1, 2, 1, 5, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|none~2|\n", - "Call query_by_index with arch_index=3871, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=3871, hp=200\n", - "{888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|none~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 1, 5, 3, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|none~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=5447, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=5447, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|none~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 3, 4, 2, 5, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|none~2|\n", - "Call query_by_index with arch_index=5202, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=5202, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|none~2|, FLOP=82.49M, Param=0.587MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|none~2|, FLOP=82.49M, Param=0.587MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|skip_connect~0|nor_conv_3x3~1|none~2|, FLOP=82.49M, Param=0.587MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 5, 1, 2, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=2401, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=2401, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 2, 1, 5, 4, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|none~1|skip_connect~2|\n", - "Call query_by_index with arch_index=12249, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=12249, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|none~1|skip_connect~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|none~1|skip_connect~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 4, 2, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=7064, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=7064, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 1, 5, 2, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|none~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=8464, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=8464, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|none~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 1, 2, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=15041, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=15041, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=153.27M, Param=1.073MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=153.27M, Param=1.073MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 5, 2, 4, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=3456, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=3456, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 3, 1, 1, 3, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=13232, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=13232, hp=200\n", - "{888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 2, 2, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=10739, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=10739, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=153.27M, Param=1.073MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=153.27M, Param=1.073MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 3, 3, 1, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|avg_pool_3x3~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=7726, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=7726, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|avg_pool_3x3~0|nor_conv_1x1~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 3, 2, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=1062, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=1062, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=153.27M, Param=1.073MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=153.27M, Param=1.073MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=153.27M, Param=1.073MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 4, 4, 2, 1, 4, 1, 6]\n", - "Call query_index_by_arch with arch=|skip_connect~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=1626, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=1626, hp=200\n", - "{777: ResultsCount(cifar10, arch=|skip_connect~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|nor_conv_1x1~2|, FLOP=51.04M, Param=0.372MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|skip_connect~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|nor_conv_1x1~2|, FLOP=51.04M, Param=0.372MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|skip_connect~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|nor_conv_1x1~2|, FLOP=51.04M, Param=0.372MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 1, 5, 2, 4, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|skip_connect~2|\n", - "Call query_by_index with arch_index=7615, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=7615, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|skip_connect~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|skip_connect~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 5, 3, 3, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|avg_pool_3x3~0|avg_pool_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=9263, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=9263, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|avg_pool_3x3~0|avg_pool_3x3~1|nor_conv_3x3~2|, FLOP=113.95M, Param=0.802MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 2, 1, 3, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=14224, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=14224, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 3, 1, 1, 4, 5, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|skip_connect~1|none~2|\n", - "Call query_by_index with arch_index=15274, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=15274, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|skip_connect~1|none~2|, FLOP=51.04M, Param=0.372MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|skip_connect~1|none~2|, FLOP=51.04M, Param=0.372MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|skip_connect~1|none~2|, FLOP=51.04M, Param=0.372MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 2, 2, 1, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=3849, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=3849, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=153.27M, Param=1.073MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 3, 2, 2, 2, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=14493, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=14493, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=153.27M, Param=1.073MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=153.27M, Param=1.073MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=153.27M, Param=1.073MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 2, 2, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=15341, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=15341, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=188.66M, Param=1.317MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=188.66M, Param=1.317MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 3, 1, 5, 2, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=11401, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=11401, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 4, 2, 1, 2, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=4441, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=4441, hp=200\n", - "{777: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 2, 2, 4, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|skip_connect~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=15573, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=15573, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|skip_connect~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 1, 2, 3, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|avg_pool_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=15598, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=15598, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|avg_pool_3x3~1|avg_pool_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 4, 5, 1, 5, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|skip_connect~1|+|none~0|nor_conv_1x1~1|none~2|\n", - "Call query_by_index with arch_index=3925, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=3925, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|skip_connect~1|+|none~0|nor_conv_1x1~1|none~2|, FLOP=51.04M, Param=0.372MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|skip_connect~1|+|none~0|nor_conv_1x1~1|none~2|, FLOP=51.04M, Param=0.372MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|skip_connect~1|+|none~0|nor_conv_1x1~1|none~2|, FLOP=51.04M, Param=0.372MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 1, 1, 2, 4, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|skip_connect~2|\n", - "Call query_by_index with arch_index=8085, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=8085, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|skip_connect~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|skip_connect~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 3, 1, 1, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=13572, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=13572, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 4, 1, 2, 1, 4, 1, 6]\n", - "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=5878, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=5878, hp=200\n", - "{777: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|skip_connect~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 4, 2, 1, 2, 1, 5, 6]\n", - "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|none~2|\n", - "Call query_by_index with arch_index=12332, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=12332, hp=200\n", - "{777: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|none~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|none~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|none~2|, FLOP=86.43M, Param=0.615MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 5, 2, 1, 2, 5, 2, 6]\n", - "Call query_index_by_arch with arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|none~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=13672, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=13672, hp=200\n", - "{777: ResultsCount(cifar10, arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|none~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|none~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|none~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 2, 3, 1, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|avg_pool_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=15096, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=15096, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|avg_pool_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 4, 1, 1, 2, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=5978, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=5978, hp=200\n", - "{777: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 5, 2, 1, 5, 1, 1, 6]\n", - "Call query_index_by_arch with arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|none~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=9873, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=9873, hp=200\n", - "{777: ResultsCount(cifar10, arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|none~0|nor_conv_1x1~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|none~0|nor_conv_1x1~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 3, 3, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=14775, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=14775, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 1, 3, 1, 2, 5, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|none~2|\n", - "Call query_by_index with arch_index=5815, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=5815, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|none~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|none~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 4, 1, 4, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_1x1~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=10635, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=10635, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_1x1~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_1x1~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 3, 1, 1, 3, 1, 2, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|avg_pool_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=9410, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=9410, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|avg_pool_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|avg_pool_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 1, 1, 5, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=10265, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=10265, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 2, 5, 1, 5, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|none~2|\n", - "Call query_by_index with arch_index=7547, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=7547, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|none~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|none~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|none~2|, FLOP=86.43M, Param=0.615MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 3, 3, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=1173, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=1173, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 5, 4, 2, 1, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=9094, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=9094, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 3, 2, 5, 1, 1, 2, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=13039, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=13039, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 3, 1, 4, 3, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=2045, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=2045, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=51.04M, Param=0.372MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=51.04M, Param=0.372MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=51.04M, Param=0.372MB, seed=0999, 1 eval-sets: [ori-test])}\n", + "Call the get_more_info function with index=137, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=137\n", + "[0, 1, 3, 4, 4, 1, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|skip_connect~1|+|skip_connect~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=12383, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=12383\n", "[0, 1, 2, 5, 2, 1, 2, 6]\n", "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=12848, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=12848, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 3, 1, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=65, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=65, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 5, 1, 1, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=14118, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=14118, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 5, 1, 1, 2, 4, 2, 6]\n", - "Call query_index_by_arch with arch=|none~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|skip_connect~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=165, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=165, hp=200\n", - "{777: ResultsCount(cifar10, arch=|none~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|skip_connect~1|nor_conv_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|none~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|skip_connect~1|nor_conv_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|none~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|skip_connect~1|nor_conv_3x3~2|, FLOP=86.43M, Param=0.615MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 4, 2, 3, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=6711, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=6711, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 3, 5, 1, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|none~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=310, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=310, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|none~0|nor_conv_1x1~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|none~0|nor_conv_1x1~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 4, 1, 1, 3, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_1x1~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=4770, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=4770, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_1x1~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_1x1~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_1x1~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|, FLOP=54.97M, Param=0.400MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 5, 2, 1, 2, 1, 2, 6]\n", - "Call query_index_by_arch with arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=15312, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=15312, hp=200\n", - "{777: ResultsCount(cifar10, arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|none~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=121.82M, Param=0.858MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 1, 4, 1, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=1251, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=1251, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=54.97M, Param=0.400MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=54.97M, Param=0.400MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=54.97M, Param=0.400MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 4, 2, 2, 2, 5, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|none~2|\n", - "Call query_by_index with arch_index=1010, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=1010, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|none~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|none~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|none~2|, FLOP=117.88M, Param=0.830MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 1, 1, 2, 3, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=7524, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=7524, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 3, 2, 2, 1, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=10321, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=10321, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 5, 2, 4, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=8449, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=8449, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 1, 1, 5, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|none~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=1013, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=1013, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|none~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|none~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|none~1|nor_conv_1x1~2|, FLOP=90.36M, Param=0.643MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 3, 3, 4, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|avg_pool_3x3~0|skip_connect~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=8471, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=8471, hp=200\n", - "{888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|avg_pool_3x3~0|skip_connect~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 3, 2, 1, 2, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=9803, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=9803, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 3, 2, 2, 5, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|none~2|\n", - "Call query_by_index with arch_index=6593, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=6593, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|none~2|, FLOP=149.34M, Param=1.045MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|none~2|, FLOP=149.34M, Param=1.045MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 1, 1, 1, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=15463, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=15463, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=58.90M, Param=0.428MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=58.90M, Param=0.428MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 1, 4, 1, 1, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=3424, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=3424, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 4, 2, 2, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=7064, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=7064, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|, FLOP=149.34M, Param=1.045MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 3, 2, 1, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=3771, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=3771, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=121.82M, Param=0.858MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 2, 1, 1, 2, 4, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|skip_connect~2|\n", - "Call query_by_index with arch_index=8085, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=8085, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|skip_connect~2|, FLOP=90.36M, Param=0.643MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|skip_connect~2|, FLOP=90.36M, Param=0.643MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 1, 4, 5, 5, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|none~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=2657, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=2657, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|none~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|none~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_1x1~0|+|skip_connect~0|none~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=82.49M, Param=0.587MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[0, 2, 4, 4, 1, 2, 2, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", - "Call query_by_index with arch_index=13431, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=13431, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|skip_connect~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|skip_connect~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 3, 3, 2, 2, 2, 1, 6]\n", - "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=12182, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=12182, hp=200\n", - "{777: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|avg_pool_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 5, 1, 2, 4, 1, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_3x3~0|skip_connect~1|nor_conv_1x1~2|\n", - "Call query_by_index with arch_index=9098, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=9098, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_3x3~0|skip_connect~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_3x3~0|skip_connect~1|nor_conv_1x1~2|, FLOP=86.43M, Param=0.615MB, seed=0888, 1 eval-sets: [ori-test])}\n", - "[0, 2, 2, 5, 2, 1, 3, 6]\n", - "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", - "Call query_by_index with arch_index=14456, dataname=cifar10, hp=200\n", - "Call query_meta_info_by_index with arch_index=14456, hp=200\n", - "{777: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0777, 1 eval-sets: [ori-test]), 888: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0888, 1 eval-sets: [ori-test]), 999: ResultsCount(cifar10, arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|, FLOP=117.88M, Param=0.830MB, seed=0999, 1 eval-sets: [ori-test])}\n", - "[(10117, 91.09), (4797, 93.08), (4218, 92.8), (3844, 92.0), (5182, 91.89), (2087, 91.51), (14919, 93.45), (443, 92.46), (4747, 93.57), (11473, 91.77), (2819, 92.28), (12184, 93.14), (14194, 91.21), (11915, 93.23), (6096, 92.0), (6465, 92.03), (137, 93.25), (14152, 93.68), (4825, 92.13), (12692, 93.61), (2758, 92.43), (14039, 92.87), (10324, 93.24), (551, 92.93), (12334, 91.64), (12926, 91.73), (7162, 91.31), (5315, 93.26), (1577, 91.79), (3871, 92.06), (5447, 92.56), (5202, 92.85), (2401, 92.85), (12249, 93.13), (7064, 92.91), (8464, 92.73), (15041, 93.09), (3456, 94.13), (13232, 91.16), (10739, 93.36), (7726, 92.92), (1062, 93.43), (1626, 92.61), (7615, 92.91), (9263, 92.42), (14224, 93.33), (15274, 92.1), (3849, 93.56), (14493, 93.14), (15341, 93.78), (11401, 92.53), (4441, 93.11), (15573, 93.0), (15598, 92.61), (3925, 85.31), (8085, 92.95), (13572, 93.15), (5878, 93.46), (12332, 91.99), (13672, 92.99), (15096, 90.75), (5978, 92.77), (9873, 90.62), (14775, 92.65), (5815, 92.19), (10635, 92.64), (9410, 91.29), (10265, 91.94), (7547, 91.26), (1173, 93.21), (9094, 93.15), (13039, 93.08), (2045, 89.72), (12848, 93.29), (65, 93.36), (14118, 93.95), (165, 92.7), (6711, 92.68), (310, 91.99), (4770, 92.72), (15312, 92.88), (1251, 91.87), (1010, 92.91), (7524, 92.87), (10321, 93.19), (8449, 93.97), (1013, 93.21), (8471, 91.97), (9803, 91.44), (6593, 93.19), (15463, 91.41), (3424, 93.5), (7064, 92.91), (3771, 93.19), (8085, 92.95), (2657, 92.98), (13431, 93.55), (12182, 92.18), (9098, 92.53), (14456, 92.79)]\n", + "Call the get_more_info function with index=12848, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=12848\n", + "[0, 1, 4, 2, 1, 1, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|skip_connect~2|\n", + "Call the get_more_info function with index=8209, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=8209\n", + "[0, 2, 1, 5, 2, 3, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_3x3~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=10859, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10859\n", + "[0, 2, 1, 5, 2, 2, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_3x3~0|nor_conv_3x3~1|none~2|\n", + "Call the get_more_info function with index=15123, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=15123\n", + "[0, 4, 1, 2, 1, 1, 1, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=1828, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1828\n", + "[0, 4, 2, 4, 1, 1, 1, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=1892, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1892\n", + "[0, 2, 4, 2, 1, 1, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|skip_connect~2|\n", + "Call the get_more_info function with index=111, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=111\n", + "[0, 3, 2, 1, 1, 3, 1, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=10418, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10418\n", + "[0, 4, 3, 1, 1, 1, 5, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_1x1~1|none~2|\n", + "Call the get_more_info function with index=12502, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=12502\n", + "[0, 3, 2, 2, 2, 1, 1, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=13123, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=13123\n", + "[0, 1, 2, 4, 1, 4, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_1x1~0|skip_connect~1|none~2|\n", + "Call the get_more_info function with index=14267, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14267\n", + "[0, 2, 5, 5, 2, 2, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|none~1|+|nor_conv_3x3~0|nor_conv_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=13859, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=13859\n", + "[0, 1, 1, 1, 1, 4, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_1x1~0|skip_connect~1|none~2|\n", + "Call the get_more_info function with index=5485, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=5485\n", + "[0, 1, 2, 1, 3, 4, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|avg_pool_3x3~0|skip_connect~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=5028, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=5028\n", + "[0, 1, 1, 5, 1, 1, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|none~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=11187, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=11187\n", + "[0, 1, 1, 4, 1, 2, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|none~2|\n", + "Call the get_more_info function with index=10640, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10640\n", + "[0, 5, 1, 3, 1, 4, 1, 6]\n", + "Call query_index_by_arch with arch=|none~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|nor_conv_1x1~0|skip_connect~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=9644, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=9644\n", + "[0, 4, 2, 3, 2, 5, 1, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_3x3~0|none~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=10414, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10414\n", + "[0, 1, 4, 4, 1, 2, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|none~2|\n", + "Call the get_more_info function with index=7613, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=7613\n", + "[0, 1, 3, 2, 2, 3, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|avg_pool_3x3~1|none~2|\n", + "Call the get_more_info function with index=10225, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10225\n", + "[0, 2, 1, 5, 1, 2, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|none~1|+|nor_conv_1x1~0|nor_conv_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=14294, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14294\n", + "[0, 3, 5, 2, 2, 1, 1, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|none~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=14886, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14886\n", + "[0, 3, 2, 1, 1, 2, 5, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|none~2|\n", + "Call the get_more_info function with index=10839, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10839\n", + "[0, 1, 4, 1, 5, 3, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_1x1~1|+|none~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=1060, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1060\n", + "[0, 2, 3, 1, 3, 2, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|avg_pool_3x3~0|nor_conv_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=1850, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1850\n", + "[0, 3, 2, 5, 5, 2, 2, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|none~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=10928, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10928\n", + "[0, 3, 2, 2, 4, 4, 1, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|skip_connect~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=1811, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1811\n", + "[0, 1, 1, 5, 1, 3, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|none~1|+|nor_conv_1x1~0|avg_pool_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=3756, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=3756\n", + "[0, 1, 5, 4, 1, 2, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|none~0|skip_connect~1|+|nor_conv_1x1~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=2541, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=2541\n", + "[0, 2, 1, 2, 1, 3, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|avg_pool_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=15348, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=15348\n", + "[0, 2, 1, 3, 2, 1, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|avg_pool_3x3~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=10073, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10073\n", + "[0, 1, 2, 4, 5, 1, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|skip_connect~1|+|none~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=3819, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=3819\n", + "[0, 1, 4, 2, 2, 5, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|none~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=13997, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=13997\n", + "[0, 2, 5, 1, 4, 3, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|nor_conv_1x1~1|+|skip_connect~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=14616, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14616\n", + "[0, 1, 4, 5, 1, 3, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|none~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=787, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=787\n", + "[0, 4, 4, 1, 1, 2, 5, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|skip_connect~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_3x3~1|none~2|\n", + "Call the get_more_info function with index=4874, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=4874\n", + "[0, 1, 4, 1, 5, 4, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_1x1~1|+|none~0|skip_connect~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=12709, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=12709\n", + "[0, 1, 4, 1, 2, 4, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_1x1~1|+|nor_conv_3x3~0|skip_connect~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=1363, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1363\n", + "[0, 3, 2, 2, 3, 1, 3, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|avg_pool_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=11846, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=11846\n", + "[0, 1, 1, 1, 4, 3, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|skip_connect~0|avg_pool_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=903, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=903\n", + "[0, 4, 4, 3, 5, 2, 2, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|skip_connect~0|avg_pool_3x3~1|+|none~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=8025, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=8025\n", + "[0, 1, 4, 2, 3, 3, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|avg_pool_3x3~0|avg_pool_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=7119, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=7119\n", + "[0, 2, 2, 4, 2, 2, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_3x3~0|nor_conv_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=688, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=688\n", + "[0, 2, 4, 1, 5, 3, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_1x1~1|+|none~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=13273, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=13273\n", + "[0, 1, 4, 4, 5, 2, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|skip_connect~1|+|none~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=4593, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=4593\n", + "[0, 4, 1, 2, 1, 3, 1, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=14791, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14791\n", + "[0, 3, 2, 5, 1, 5, 5, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|none~1|+|nor_conv_1x1~0|none~1|none~2|\n", + "Call the get_more_info function with index=7339, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=7339\n", + "[0, 2, 2, 2, 5, 3, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|none~0|avg_pool_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=5436, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=5436\n", + "[0, 2, 1, 4, 2, 5, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_3x3~0|none~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=15113, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=15113\n", + "[0, 3, 5, 1, 3, 1, 1, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|none~0|nor_conv_1x1~1|+|avg_pool_3x3~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=5965, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=5965\n", + "[0, 1, 2, 4, 5, 2, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|skip_connect~1|+|none~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=2012, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=2012\n", + "[0, 2, 4, 4, 1, 5, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|skip_connect~1|+|nor_conv_1x1~0|none~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=14377, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14377\n", + "[0, 2, 4, 2, 1, 2, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=202, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=202\n", + "[0, 1, 2, 5, 5, 4, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|none~1|+|none~0|skip_connect~1|skip_connect~2|\n", + "Call the get_more_info function with index=5319, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=5319\n", + "[0, 1, 2, 1, 4, 2, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|skip_connect~0|nor_conv_3x3~1|none~2|\n", + "Call the get_more_info function with index=1413, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1413\n", + "[0, 2, 3, 5, 2, 1, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|avg_pool_3x3~0|none~1|+|nor_conv_3x3~0|nor_conv_1x1~1|none~2|\n", + "Call the get_more_info function with index=14891, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14891\n", + "[0, 2, 5, 2, 2, 3, 4, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|nor_conv_3x3~1|+|nor_conv_3x3~0|avg_pool_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=1316, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1316\n", + "[0, 1, 3, 4, 1, 5, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|skip_connect~1|+|nor_conv_1x1~0|none~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=4162, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=4162\n", + "[0, 1, 1, 5, 2, 2, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|none~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=3147, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=3147\n", + "[0, 2, 1, 1, 3, 5, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|avg_pool_3x3~0|none~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=14312, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14312\n", + "[0, 4, 4, 1, 2, 1, 1, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|skip_connect~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=1758, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=1758\n", + "[0, 1, 1, 2, 5, 1, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=6546, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=6546\n", + "[0, 4, 2, 3, 1, 5, 2, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_1x1~0|none~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=13398, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=13398\n", + "[0, 4, 2, 2, 2, 5, 1, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|nor_conv_3x3~0|none~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=15622, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=15622\n", + "[0, 2, 5, 1, 1, 4, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|nor_conv_1x1~1|+|nor_conv_1x1~0|skip_connect~1|none~2|\n", + "Call the get_more_info function with index=6979, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=6979\n", + "[0, 1, 3, 1, 2, 1, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=14397, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14397\n", + "[0, 2, 5, 2, 3, 2, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|none~0|nor_conv_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=14917, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=14917\n", + "[0, 2, 2, 4, 1, 3, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|skip_connect~1|+|nor_conv_1x1~0|avg_pool_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=4063, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=4063\n", + "[0, 3, 2, 1, 4, 2, 2, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|skip_connect~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=10476, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10476\n", + "[0, 3, 3, 2, 1, 1, 5, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|none~2|\n", + "Call the get_more_info function with index=10289, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10289\n", + "[0, 2, 2, 2, 5, 1, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=12866, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=12866\n", + "[0, 2, 4, 1, 1, 1, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_1x1~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=9385, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=9385\n", + "[0, 4, 1, 2, 1, 2, 4, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_1x1~0|nor_conv_3x3~1|+|nor_conv_1x1~0|nor_conv_3x3~1|skip_connect~2|\n", + "Call the get_more_info function with index=11050, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=11050\n", + "[0, 2, 2, 2, 4, 2, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=12608, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=12608\n", + "[0, 1, 4, 2, 4, 1, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|skip_connect~0|nor_conv_3x3~1|+|skip_connect~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=5204, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=5204\n", + "[0, 4, 2, 1, 5, 2, 1, 6]\n", + "Call query_index_by_arch with arch=|skip_connect~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|none~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=10703, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=10703\n", + "[0, 2, 2, 1, 1, 3, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|nor_conv_1x1~1|+|nor_conv_1x1~0|avg_pool_3x3~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=7059, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=7059\n", + "[0, 2, 2, 5, 4, 1, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|none~1|+|skip_connect~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=2053, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=2053\n", + "[0, 2, 4, 3, 5, 1, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|avg_pool_3x3~1|+|none~0|nor_conv_1x1~1|none~2|\n", + "Call the get_more_info function with index=13888, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=13888\n", + "[0, 2, 1, 1, 2, 1, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_3x3~0|nor_conv_1x1~1|none~2|\n", + "Call the get_more_info function with index=4601, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=4601\n", + "[0, 2, 4, 5, 1, 1, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|none~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=2552, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=2552\n", + "[0, 5, 1, 1, 4, 1, 2, 6]\n", + "Call query_index_by_arch with arch=|none~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|skip_connect~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=7579, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=7579\n", + "[0, 5, 1, 1, 1, 4, 3, 6]\n", + "Call query_index_by_arch with arch=|none~0|+|nor_conv_1x1~0|nor_conv_1x1~1|+|nor_conv_1x1~0|skip_connect~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=12393, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=12393\n", + "[0, 2, 4, 2, 2, 2, 1, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|skip_connect~0|nor_conv_3x3~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_1x1~2|\n", + "Call the get_more_info function with index=571, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=571\n", + "[0, 3, 3, 5, 2, 2, 2, 6]\n", + "Call query_index_by_arch with arch=|avg_pool_3x3~0|+|avg_pool_3x3~0|none~1|+|nor_conv_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=11568, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=11568\n", + "[0, 2, 1, 4, 3, 2, 3, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|avg_pool_3x3~0|nor_conv_3x3~1|avg_pool_3x3~2|\n", + "Call the get_more_info function with index=13626, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=13626\n", + "[0, 2, 2, 3, 1, 1, 2, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_3x3~0|+|nor_conv_3x3~0|avg_pool_3x3~1|+|nor_conv_1x1~0|nor_conv_1x1~1|nor_conv_3x3~2|\n", + "Call the get_more_info function with index=12411, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=12411\n", + "[0, 1, 3, 2, 5, 1, 5, 6]\n", + "Call query_index_by_arch with arch=|nor_conv_1x1~0|+|avg_pool_3x3~0|nor_conv_3x3~1|+|none~0|nor_conv_1x1~1|none~2|\n", + "Call the get_more_info function with index=4673, dataset=cifar10, iepoch=None, hp=200, and is_random=False.\n", + "Call query_index_by_arch with arch=4673\n", + "[(8621, 93.795), (13981, 93.695), (11890, 90.685), (2289, 90.33), (11341, 91.44333333333333), (11828, 92.52666666666669), (1177, 93.61333333333333), (11135, 90.975), (1742, 93.43333333333334), (10425, 92.59), (137, 93.18333333333334), (12383, 92.48), (12848, 93.29), (8209, 92.52333333333333), (10859, 92.45), (15123, 93.09), (1828, 92.65666666666665), (1892, 92.47000000000001), (111, 93.68666666666667), (10418, 91.38), (12502, 89.28000000000002), (13123, 92.95), (14267, 89.02), (13859, 93.14333333333333), (5485, 89.11333333333334), (5028, 93.135), (11187, 89.5), (10640, 92.45), (9644, 89.02000000000001), (10414, 92.62), (7613, 92.37), (10225, 91.83), (14294, 93.195), (14886, 92.51666666666667), (10839, 91.31), (1060, 90.3), (1850, 89.58), (10928, 92.08999999999999), (1811, 90.60000000000001), (3756, 90.075), (2541, 92.13333333333333), (15348, 93.15), (10073, 93.66499999999999), (3819, 91.19), (13997, 93.045), (14616, 92.97), (787, 92.12333333333333), (4874, 92.54666666666667), (12709, 92.74), (1363, 92.40333333333332), (11846, 89.08500000000001), (903, 92.63666666666667), (8025, 92.26333333333332), (7119, 92.03999999999999), (688, 93.08333333333333), (13273, 92.40333333333335), (4593, 91.97), (14791, 92.28666666666668), (7339, 88.63), (5436, 92.98), (15113, 91.8), (5965, 83.22), (2012, 91.36666666666667), (14377, 92.74), (202, 92.77999999999999), (5319, 91.95), (1413, 92.71), (14891, 93.18666666666667), (1316, 92.80333333333334), (4162, 92.19), (3147, 92.88333333333334), (14312, 92.66), (1758, 92.11333333333333), (6546, 92.345), (13398, 93.07), (15622, 92.96666666666665), (6979, 92.01), (14397, 90.98), (14917, 93.02), (4063, 93.21000000000001), (10476, 93.83), (10289, 89.945), (12866, 92.18), (9385, 92.71000000000001), (11050, 92.28666666666668), (12608, 93.765), (5204, 93.34333333333332), (10703, 93.14333333333333), (7059, 92.37), (2053, 94.20666666666666), (13888, 91.20666666666666), (4601, 92.81), (2552, 93.39333333333333), (7579, 92.735), (12393, 89.775), (571, 93.69333333333334), (11568, 91.4), (13626, 92.24), (12411, 93.57), (4673, 88.14)]\n", "100 0\n", - "{'<90': 2, '<91': 2, '<92': 19, '<93': 41, '<94': 35, '>94': 1}\n" + "{'<90': 12, '<91': 7, '<92': 11, '<93': 43, '<94': 26, '>94': 1}\n", + "mean: 92.04488333333333\n", + "max: 94.20666666666666\n", + "min: 83.22\n" ] } ], @@ -48066,7 +48086,7 @@ " '|' + nodes_str[4] + '~0|' + nodes_str[5] + '~1|' + nodes_str[6] + '~2|' \n", " return arch_str\n", "\n", - "filename = '210943.txt'\n", + "filename = '211035.txt'\n", "with open('./output_graphs/' + filename, 'r') as f:\n", " texts = f.read()\n", " df = process_graph_data(texts)\n", @@ -48090,10 +48110,11 @@ " valid += 1\n", " arch_str = nodes_to_arch_str(nodes)\n", " index = api.query_index_by_arch(arch_str)\n", - " results = api.query_by_index(index, 'cifar10', hp='200')\n", - " print(results)\n", - " result = results[888].get_eval('ori-test')\n", - " acc = result['accuracy']\n", + " # results = api.query_by_index(index, 'cifar10', hp='200')\n", + " # print(results)\n", + " # result = results[888].get_eval('ori-test')\n", + " res = api.get_more_info(index, 'cifar10', None, hp=200, is_random=False)\n", + " acc = res['test-accuracy']\n", " scores.append((index, acc))\n", " if acc < 90:\n", " dist['<90'] += 1\n", @@ -48107,8 +48128,6 @@ " dist['<94'] += 1\n", " else: \n", " dist['>94'] += 1\n", - "\n", - "\n", " else:\n", " not_valid += 1\n", " with open('./output_graphs/' + filename + '.json', 'w') as f:\n", @@ -48116,6 +48135,10 @@ " print(scores)\n", " print(valid, not_valid)\n", " print(dist)\n", + " print(\"mean: \", np.mean([x[1] for x in scores]))\n", + " print(\"max: \", np.max([x[1] for x in scores]))\n", + " print(\"min: \", np.min([x[1] for x in scores]))\n", + " \n", "\n" ] },