diff --git a/.gitmodules b/.gitmodules index 3027337..e693015 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "qlib-git"] path = .latent-data/qlib url = git@github.com:microsoft/qlib.git +[submodule ".latent-data/qlib"] + path = .latent-data/qlib + url = git@github.com:microsoft/qlib.git diff --git a/.latent-data/qlib b/.latent-data/qlib new file mode 160000 index 0000000..a96f0c2 --- /dev/null +++ b/.latent-data/qlib @@ -0,0 +1 @@ +Subproject commit a96f0c2e5fa427ce33660c8791fa2057c6f423fc diff --git a/README.md b/README.md index 6cbf758..934b184 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ If you find that this project helps your research, please consider citing the re note = {\mbox{doi}:\url{10.1109/TPAMI.2021.3054824}} } @inproceedings{dong2020nasbench201, - title = {NAS-Bench-201: Extending the Scope of Reproducible Neural Architecture Search}, + title = {{NAS-Bench-201}: Extending the Scope of Reproducible Neural Architecture Search}, author = {Dong, Xuanyi and Yang, Yi}, booktitle = {International Conference on Learning Representations (ICLR)}, url = {https://openreview.net/forum?id=HJxyZkBKDr}, diff --git a/README_CN.md b/README_CN.md index 3b6c10e..7101a80 100644 --- a/README_CN.md +++ b/README_CN.md @@ -107,7 +107,7 @@ Some methods use knowledge distillation (KD), which require pre-trained models. note = {\mbox{doi}:\url{10.1109/TPAMI.2021.3054824}} } @inproceedings{dong2020nasbench201, - title = {NAS-Bench-201: Extending the Scope of Reproducible Neural Architecture Search}, + title = {{NAS-Bench-201}: Extending the Scope of Reproducible Neural Architecture Search}, author = {Dong, Xuanyi and Yang, Yi}, booktitle = {International Conference on Learning Representations (ICLR)}, url = {https://openreview.net/forum?id=HJxyZkBKDr}, diff --git a/notebooks/NATS-Bench/issue-96.ipynb b/notebooks/NATS-Bench/issue-96.ipynb new file mode 100644 index 0000000..70ff0e6 --- /dev/null +++ b/notebooks/NATS-Bench/issue-96.ipynb @@ -0,0 +1,91 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-03-01 12:28:12] Try to use the default NATS-Bench (topology) path from fast_mode=True and path=None.\n" + ] + } + ], + "source": [ + "from nats_bench import create\n", + "import numpy as np\n", + "\n", + "def get_correlation(A, B):\n", + " return float(np.corrcoef(A, B)[0,1])\n", + "\n", + "# Create the API for tologoy search space\n", + "api = create(None, 'tss', fast_mode=True, verbose=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "There are 15625 architectures on the topology search space\n" + ] + } + ], + "source": [ + "print('There are {:} architectures on the topology search space'.format(len(api)))\n", + "accuracies_12, accuracies_200 = [], []\n", + "for i, arch in enumerate(api):\n", + " info_a = api.get_more_info(i, dataset='cifar10-valid', hp='12', is_random=False)\n", + " accuracies_12.append(info_a['valid-accuracy'])\n", + "\n", + " info_b = api.get_more_info(i, dataset='cifar10-valid', hp='200', is_random=False)\n", + " accuracies_200.append(info_b['test-accuracy'])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[CIFAR-10] The correlation between 12-epoch validation accuracy and 200-epoch test accuracy is: 91.18%\n" + ] + } + ], + "source": [ + "correlation = get_correlation(accuracies_12, accuracies_200)\n", + "print('[CIFAR-10] The correlation between 12-epoch validation accuracy and 200-epoch test accuracy is: {:.2f}%'.format(correlation * 100))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}