92 lines
2.3 KiB
Plaintext
92 lines
2.3 KiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"[2021-03-01 12:28:12] Try to use the default NATS-Bench (topology) path from fast_mode=True and path=None.\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"from nats_bench import create\n",
|
||
|
|
"import numpy as np\n",
|
||
|
|
"\n",
|
||
|
|
"def get_correlation(A, B):\n",
|
||
|
|
" return float(np.corrcoef(A, B)[0,1])\n",
|
||
|
|
"\n",
|
||
|
|
"# Create the API for tologoy search space\n",
|
||
|
|
"api = create(None, 'tss', fast_mode=True, verbose=False)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 2,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"There are 15625 architectures on the topology search space\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"print('There are {:} architectures on the topology search space'.format(len(api)))\n",
|
||
|
|
"accuracies_12, accuracies_200 = [], []\n",
|
||
|
|
"for i, arch in enumerate(api):\n",
|
||
|
|
" info_a = api.get_more_info(i, dataset='cifar10-valid', hp='12', is_random=False)\n",
|
||
|
|
" accuracies_12.append(info_a['valid-accuracy'])\n",
|
||
|
|
"\n",
|
||
|
|
" info_b = api.get_more_info(i, dataset='cifar10-valid', hp='200', is_random=False)\n",
|
||
|
|
" accuracies_200.append(info_b['test-accuracy'])"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 4,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"[CIFAR-10] The correlation between 12-epoch validation accuracy and 200-epoch test accuracy is: 91.18%\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"correlation = get_correlation(accuracies_12, accuracies_200)\n",
|
||
|
|
"print('[CIFAR-10] The correlation between 12-epoch validation accuracy and 200-epoch test accuracy is: {:.2f}%'.format(correlation * 100))"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.8.3"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 4
|
||
|
|
}
|