363 lines
94 KiB
Plaintext
363 lines
94 KiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"import os, pickle, sys\n",
|
||
|
|
"import matplotlib.pyplot as plt\n",
|
||
|
|
"from scipy import stats\n",
|
||
|
|
"import numpy as np\n",
|
||
|
|
"import glob\n",
|
||
|
|
"from prettytable import PrettyTable"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 2,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"ptcv_seed0\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| exp | grad_norm | snip | grasp | fisher | synflow | jacob_cov | samples |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| pred_ptcv_svhn_pretrain.p | 0.707 | 0.576 | 0.382 | 0.168 | 0.747 | 0.34 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100.p | 0.385 | 0.509 | 0.105 | 0.469 | 0.428 | 0.145 | 54 |\n",
|
||
|
|
"| pred_ptcv_svhn.p | 0.668 | 0.695 | 0.165 | 0.675 | 0.821 | 0.344 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100_pretrain.p | 0.763 | 0.813 | 0.832 | 0.595 | 0.424 | 0.595 | 54 |\n",
|
||
|
|
"| pred_ptcv_cifar10.p | 0.409 | 0.521 | 0.127 | 0.471 | 0.456 | 0.046 | 56 |\n",
|
||
|
|
"| pred_ptcv_cifar10_pretrain.p | 0.639 | 0.71 | 0.434 | 0.464 | 0.416 | 0.646 | 56 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k.p | 0.563 | 0.644 | 0.025 | 0.675 | 0.652 | 0.343 | 191 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k_pretrain.p | 0.692 | 0.67 | 0.493 | 0.725 | 0.691 | 0.141 | 191 |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"ptcv_seed1\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| exp | grad_norm | snip | grasp | fisher | synflow | jacob_cov | samples |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| pred_ptcv_svhn_pretrain.p | 0.681 | 0.541 | 0.449 | 0.148 | 0.747 | 0.324 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100.p | 0.384 | 0.501 | 0.051 | 0.483 | 0.429 | 0.059 | 54 |\n",
|
||
|
|
"| pred_ptcv_svhn.p | 0.642 | 0.666 | 0.077 | 0.633 | 0.83 | 0.224 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100_pretrain.p | 0.618 | 0.646 | 0.793 | 0.543 | 0.424 | 0.62 | 54 |\n",
|
||
|
|
"| pred_ptcv_cifar10.p | 0.387 | 0.505 | 0.111 | 0.476 | 0.455 | 0.101 | 56 |\n",
|
||
|
|
"| pred_ptcv_cifar10_pretrain.p | 0.689 | 0.733 | 0.376 | 0.476 | 0.416 | 0.646 | 56 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k.p | 0.569 | 0.64 | 0.165 | 0.668 | 0.651 | 0.292 | 191 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k_pretrain.p | 0.69 | 0.671 | 0.502 | 0.73 | 0.691 | 0.14 | 191 |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"ptcv_seed2\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| exp | grad_norm | snip | grasp | fisher | synflow | jacob_cov | samples |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| pred_ptcv_svhn_pretrain.p | 0.661 | 0.526 | 0.346 | 0.115 | 0.747 | 0.357 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100.p | 0.366 | 0.474 | 0.045 | 0.48 | 0.428 | 0.093 | 54 |\n",
|
||
|
|
"| pred_ptcv_svhn.p | 0.668 | 0.685 | 0.244 | 0.661 | 0.823 | 0.28 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100_pretrain.p | 0.462 | 0.608 | 0.829 | 0.465 | 0.424 | 0.636 | 54 |\n",
|
||
|
|
"| pred_ptcv_cifar10.p | 0.411 | 0.511 | 0.085 | 0.496 | 0.454 | 0.056 | 56 |\n",
|
||
|
|
"| pred_ptcv_cifar10_pretrain.p | 0.715 | 0.761 | 0.478 | 0.515 | 0.416 | 0.641 | 56 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k.p | 0.564 | 0.633 | 0.096 | 0.669 | 0.652 | 0.327 | 191 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k_pretrain.p | 0.692 | 0.671 | 0.507 | 0.732 | 0.691 | 0.177 | 191 |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"ptcv_seed3\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| exp | grad_norm | snip | grasp | fisher | synflow | jacob_cov | samples |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| pred_ptcv_svhn_pretrain.p | 0.653 | 0.529 | 0.309 | 0.102 | 0.747 | 0.349 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100.p | 0.366 | 0.48 | 0.058 | 0.474 | 0.428 | 0.204 | 54 |\n",
|
||
|
|
"| pred_ptcv_svhn.p | 0.661 | 0.678 | 0.128 | 0.661 | 0.833 | 0.256 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100_pretrain.p | 0.682 | 0.783 | 0.8 | 0.664 | 0.424 | 0.621 | 54 |\n",
|
||
|
|
"| pred_ptcv_cifar10.p | 0.388 | 0.495 | 0.014 | 0.479 | 0.454 | 0.036 | 56 |\n",
|
||
|
|
"| pred_ptcv_cifar10_pretrain.p | 0.63 | 0.718 | 0.222 | 0.478 | 0.416 | 0.666 | 56 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k.p | 0.575 | 0.647 | 0.081 | 0.669 | 0.651 | 0.301 | 191 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k_pretrain.p | 0.691 | 0.668 | 0.493 | 0.725 | 0.691 | 0.171 | 191 |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"ptcv_seed4\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| exp | grad_norm | snip | grasp | fisher | synflow | jacob_cov | samples |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| pred_ptcv_svhn_pretrain.p | 0.684 | 0.549 | 0.33 | 0.17 | 0.747 | 0.334 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100.p | 0.368 | 0.484 | 0.085 | 0.492 | 0.429 | 0.207 | 54 |\n",
|
||
|
|
"| pred_ptcv_svhn.p | 0.659 | 0.671 | 0.082 | 0.641 | 0.824 | 0.252 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100_pretrain.p | 0.766 | 0.831 | 0.793 | 0.755 | 0.424 | 0.62 | 54 |\n",
|
||
|
|
"| pred_ptcv_cifar10.p | 0.401 | 0.533 | 0.086 | 0.473 | 0.454 | 0.06 | 56 |\n",
|
||
|
|
"| pred_ptcv_cifar10_pretrain.p | 0.536 | 0.614 | 0.273 | 0.412 | 0.416 | 0.657 | 56 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k.p | 0.561 | 0.627 | 0.092 | 0.659 | 0.651 | 0.272 | 191 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k_pretrain.p | 0.689 | 0.67 | 0.498 | 0.73 | 0.691 | 0.164 | 191 |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"ptcv_seed5\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| exp | grad_norm | snip | grasp | fisher | synflow | jacob_cov | samples |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n",
|
||
|
|
"| pred_ptcv_svhn_pretrain.p | 0.672 | 0.557 | 0.384 | 0.186 | 0.747 | 0.271 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100.p | 0.393 | 0.503 | 0.056 | 0.493 | 0.429 | 0.05 | 54 |\n",
|
||
|
|
"| pred_ptcv_svhn.p | 0.643 | 0.675 | 0.237 | 0.641 | 0.826 | 0.25 | 49 |\n",
|
||
|
|
"| pred_ptcv_cifar100_pretrain.p | 0.741 | 0.776 | 0.833 | 0.676 | 0.424 | 0.615 | 54 |\n",
|
||
|
|
"| pred_ptcv_cifar10.p | 0.384 | 0.484 | 0.087 | 0.468 | 0.457 | 0.004 | 56 |\n",
|
||
|
|
"| pred_ptcv_cifar10_pretrain.p | 0.692 | 0.767 | 0.303 | 0.533 | 0.416 | 0.66 | 56 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k.p | 0.57 | 0.638 | 0.141 | 0.67 | 0.651 | 0.337 | 191 |\n",
|
||
|
|
"| pred_ptcv_ImageNet1k_pretrain.p | 0.689 | 0.671 | 0.51 | 0.723 | 0.691 | 0.178 | 191 |\n",
|
||
|
|
"+---------------------------------+-----------+-------+-------+--------+---------+-----------+---------+\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"root='../results_release/ptcv'\n",
|
||
|
|
"\n",
|
||
|
|
"alld = [f'ptcv_seed{i}' for i in range(0,6)]\n",
|
||
|
|
"\n",
|
||
|
|
"allm = []\n",
|
||
|
|
"for dirs in alld:\n",
|
||
|
|
" res = {}\n",
|
||
|
|
" print(dirs)\n",
|
||
|
|
" dirs = os.path.join(root,dirs)\n",
|
||
|
|
" hl = ['exp']\n",
|
||
|
|
" added_hl = False\n",
|
||
|
|
" for fn in os.listdir(dirs):\n",
|
||
|
|
" res[fn] = {}\n",
|
||
|
|
" with open(os.path.join(dirs,fn),'rb') as f:\n",
|
||
|
|
" ptcv=pickle.load(f)\n",
|
||
|
|
" acc = []\n",
|
||
|
|
" metrics = {}\n",
|
||
|
|
" for d in ptcv:\n",
|
||
|
|
" acc.append(d['valacc'])\n",
|
||
|
|
" if len(hl) == 1:\n",
|
||
|
|
" hl.extend(d['logmeasures'].keys())\n",
|
||
|
|
" for m in d['logmeasures'].keys():\n",
|
||
|
|
" if not m in metrics:\n",
|
||
|
|
" metrics[m] = []\n",
|
||
|
|
" metrics[m].append(d['logmeasures'][m])\n",
|
||
|
|
" if not added_hl:\n",
|
||
|
|
" added_hl = True\n",
|
||
|
|
" hl.append('samples')\n",
|
||
|
|
" t = PrettyTable(hl)\n",
|
||
|
|
" row=[fn]\n",
|
||
|
|
" for m,v in metrics.items():\n",
|
||
|
|
" cr = abs(stats.spearmanr(acc,v,nan_policy='omit').correlation)\n",
|
||
|
|
" res[fn][m] = cr\n",
|
||
|
|
" cr=round(cr,3)\n",
|
||
|
|
" row.append(cr)\n",
|
||
|
|
" row.append(len(acc))\n",
|
||
|
|
" t.add_row(row)\n",
|
||
|
|
" allm.append(res)\n",
|
||
|
|
" print(t)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 3,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"parsed = {}\n",
|
||
|
|
"for e in allm:\n",
|
||
|
|
" for d,t in e.items():\n",
|
||
|
|
" if not d in parsed.keys():\n",
|
||
|
|
" parsed[d] = {}\n",
|
||
|
|
" for m,c in t.items():\n",
|
||
|
|
" if not m in parsed[d].keys():\n",
|
||
|
|
" parsed[d][m] = []\n",
|
||
|
|
" parsed[d][m].append(c)\n",
|
||
|
|
"labels = {\n",
|
||
|
|
" 'pred_ptcv_ImageNet1k.p': 'ImageNet1k',\n",
|
||
|
|
" 'pred_ptcv_cifar10.p': 'CIFAR-10',\n",
|
||
|
|
" 'pred_ptcv_cifar100.p': 'CIFAR100',\n",
|
||
|
|
" 'pred_ptcv_svhn.p': 'SVHN', \n",
|
||
|
|
" 'pred_ptcv_ImageNet1k_pretrain.p': 'ImageNet1k',\n",
|
||
|
|
" 'pred_ptcv_cifar10_pretrain.p': 'CIFAR-10',\n",
|
||
|
|
" 'pred_ptcv_cifar100_pretrain.p': 'CIFAR100',\n",
|
||
|
|
" 'pred_ptcv_svhn_pretrain.p': 'SVHN', \n",
|
||
|
|
"}\n",
|
||
|
|
"pattern = {\n",
|
||
|
|
" 'pred_ptcv_ImageNet1k.p': '*',\n",
|
||
|
|
" 'pred_ptcv_cifar10.p': '\\\\',\n",
|
||
|
|
" 'pred_ptcv_cifar100.p': '/',\n",
|
||
|
|
" 'pred_ptcv_svhn.p': 'o', \n",
|
||
|
|
" 'pred_ptcv_ImageNet1k_pretrain.p': '*',\n",
|
||
|
|
" 'pred_ptcv_cifar10_pretrain.p': '\\\\',\n",
|
||
|
|
" 'pred_ptcv_cifar100_pretrain.p': '/',\n",
|
||
|
|
" 'pred_ptcv_svhn_pretrain.p': 'o', \n",
|
||
|
|
"}\n",
|
||
|
|
"ps=[ \"|\" , \"\\\\\" , \"/\" , \"+\" , \"-\", \".\", \"*\",\"x\", \"o\", \"O\" ]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 4,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAEYCAYAAABRMYxdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOy9e3xU1b33/165TG4DhCGSCIMk1LQKKPAkgLUeRasFWou1N8WnF+utPk8vSrU9tuepor+jVn+eI1XseYq21aNWRMUbKtYLtNUKgXC/aagkJOgACQmQyyQkWc8fkxkmk7nsPTN7bvm+Xy9eJHt/98za36z1ne/stdb3o7TWCIIgCIIgZBJZyW6AIAiCIAhCvJEERxAEQRCEjEMSHEEQBEEQMg5JcARBEARByDgkwREEQRAEIePISXYDYqWkpESXl5cnuxmCIAiCICSB2traZq31KYHH0z7BKS8vZ+PGjcluhiAIgiAISUAp1RDsuExRCYIgCIKQcUiCIwiCIAhCxiEJjiAIgiAIGUfar8ERBEEQhERw4sQJmpqacLvdyW7KsCQ/Px+n00lubq4he0lwBEEQBMEATU1NjBgxgvLycpRSyW7OsEJrTUtLC01NTVRUVBi6RqaoBEEQBMEAbrebMWPGSHKTBJRSjBkzxtTTM0lwBEEQBMEgktwkD7O+lwRHEARBEISMQ9bgCIIgCEIU3HHP/TS5moOe6+nupvVIM6MdJdjy8gy9nrOshDt/9YuwNi6Xi5tvvpkNGzZQXFxMaWkpS5Ys4etf/zo7duxg7dq1XHbZZb51KiUlJbz99tsAfO1rX8PlcrFu3Trf6y1evJhHH32UU045hZ6eHn7961+zcOHCIe+7Z88efvCDH7Bp0ybuvvtubr31Vt+51atXc9NNN9HX18d1113HbbfdZuh+rUYSHEEQBCFt8c5aaJ34925yNTP1q9eFPH/E1cSWv73B9JnzcJQ5I77ejlcfC3tea83ll1/O97//fZYvXw7A1q1bOXjw4CC7f/mXf2HVqlWDjrW1tVFbW4vdbufjjz9m0qRJvnOLFi3i1ltvpa6ujqqqKr75zW8O2ankcDh46KGHeOmllwYd7+vr40c/+hFvvfUWTqeTmTNnsmDBAiZPnhzxfq1GpqgEQRAEwQIcZU6mnz+fLX97gyOupphfb82aNeTm5nLjjTf6jk2bNo0JEyZEvHblypV89atf5corr/QlR4FUVlZSWFhIa2vrkHNjx45l5syZQxKfmpoaTj/9dCZNmoTNZuPKK6/k5ZdfNnln1iBPcARBEISU58777qTpcLAk4VEArr/1+pDXOk9xcse/3mFRy8Ljn+RMP3++oSc5odixYwdVVVUR7f7+978zffp0AL71rW/xb//2bzzzzDPcfvvtlJaW8o1vfINf/epXQ67btGkTlZWVjB071nCbDhw4MCjBcjqdrF+/3vD1ViIJjiAIgpDyNB1uYsZ3Zgw98R+e/4KeG2DzU5stapUx4pnkGCFwiurgwYPU1dVx3nnnoZQiNzeXHTt2MHXqVAAefPBB/vSnP/HRRx/x6quvWtq2RCJTVIIgCELaoLVm4xsbeeGBF3j+gecN2R85fCQBLQtPPKarpkyZQm1trenrVqxYQWtrKxUVFZSXl1NfX88zzzzjO79o0SJ27tzJCy+8wLXXXovb7eaRRx5h+vTpTJ8+nU8++STka48fP57Gxkbf701NTYwfP950G61AEhxBEAQhbah5rYb6HfWcXnU6lVWVXHjVRfzgnmvC2ne2dyawhaGJNcm56KKL6O7uZtmyZb5j27ZtG5RgBOOZZ55h9erV1NfXU19fT21tbdB1OAsWLKC6uponnniCH/3oR2zZsoUtW7Ywbty4kK89c+ZM6urq2LdvHz09PSxfvpwFCxaYvjcrkCkqQRAEIW1o3NNIZXUl0y6c5ju2t3Yv1fOrQ9qPGTnGkrY4y0oi7nwKRnFWD5tf+cOQLeTOspKw1ymlePHFF7n55pu57777yM/Pp7y8nCVLloS8pr6+noaGBs455xzfsYqKCkaNGhV0rcztt9/OVVddxfXXX09W1slnIC6Xi+rqao4dO0ZWVhZLlixh165djBw5kqVLlzJ37lz6+vq45pprmDJlihl3WEZCExyl1Dzgt0A28JjW+jcB508DngCKB2xu01q/nsg2CoIgCKmLYmg1W03oPeLB7ONFpJo1VjBu3DhWrFgx5PiOHTsAmDNnDnPmzPEdLy8v58CBA0PsN23aBMDs2bMHHa+qquLDDz8cYl9WVkZTU/CnTl/+8pf58pe/bPgeEkXCEhylVDbwCHAJ0ARsUEq9orXe5Wf2f4AVWuv/UkpNBl4HyhPVRkEQBCG1mThlInUb63y/122sY8KZobdJT5wykU83fpqIpgkpRiKf4MwC9mqtPwZQSi0HLgP8ExwNjBz4eRQQemWTIAiCMOyomldFX18fdbV1KBTlU8upmhd663TVvCre2f1OAlsopAqJTHDGA/4roZqA2QE2i4G/KKV+AhQBFwd7IaXUDcANAKWlpaxduzbebRUEQRBSCJfLRX19PQClU0spnVrqO9fQ0BD22hP9J+LyOTFq1CiOHz8e8+sI0eN2uw3/LVNtkfFC4HGt9X8opT4PPKmUmqq17vc30lovA5YBVFdXa//5RkEQBCHzeHrV05SXl0d1bWtZK/H4nNi9ezcjRoyI+XWE6MnPz2fGjNA1j/xJ5DbxA4D/RKlz4Jg/1wIrALTWHwD5QPhl5YIgCIIgCAEkMsHZAFQqpSqUUjbgSuCVAJv9wBcBlFJn4klwDiewjYIgCIIgZAAJm6LSWvcqpX4MvIlnC/gftdY7lVJ3ARu11q8AtwCPKqUW4VlwfLXWydCIFQRBEITwLLnndtoOBi+y5+7u4VBLG2PHFJOfZzP0esWlE7j5V3eFtXG5XNx8881s2LCB4uJiSktLWbJkCV//+tfZsWMHa9eu5bLLLqOiogKAkpIS3n77bQC+9rWv4XK5WLdune/1Fi9ezKOPPsopp5xCT08Pv/71r1m4cCEAzz33HIsXL2b37t3U1NRQXX2y1tC9997LH/7wB7Kzs3nooYeYO3cuAKtXr+amm26ir6+P6667jttuu83QvVtBQtfgDNS0eT3g2O1+P+8CvpDINgmCIAhCNLQdbGTx1z475Hi9q5UVa7dz49fPorxstOHXW/zSR2HPa625/PLL+f73v++rRLx161YOHjw4yC5Qiwqgra2N2tpa7HY7H3/8MZMmTfKdW7RoEbfeeit1dXVUVVXxzW9+k9zcXKZOncrKlSv54Q9/OOi1du3axfLly9m5cyeffPIJF198MR995Gn7j370I9566y2cTiczZ85kwYIFTJ482bAP4kmqLTIWBEEQhCG0HGpBa41SwQv3aa2pXV1Lw84GNJoJZ0xg1ldmhbS3Cm9y8+055pIbI6xZs4bc3FxuvPFG37Fp06b5dpeFY+XKlXz1q1+ltLSU5cuXB1UTr6yspLCwkNbWVsaOHcuZZ54Z9LVefvllrrzySvLy8qioqOD000+npqYGgNNPP92XPF155ZW8/PLLSUtwRItKEARBSHm6OrqoXR1aaDJQo6pxd2NYeyuwMrkBT7XiqqrQNX+8/P3vf/cJZd59992AR49q4cKFLFy4cJDQpj+bNm2isrKSsWPHhn39AwcOMGHCyT1DTqeTAwcOhDyeLOQJjiAIgpDy2EfaadjZEFZzyoxGVbyxOrkxQ+AU1cGDB6mrq+O8885DKUVubi47duxg6tSpADz44IP86U9/4qOPPuLVV19NVrPjjjzBEQRBENICs5pT4ezjSaKSmylTplBba/6p1IoVK2htbaWiooLy8nLq6+sHPcVZtGgRO3fu5IUXXuDaa6/F7XaHfb3x48cPUjBvampi/PjxIY8nC0lwBEEQhJSn/Vg7E84IrzlVt7GOrWu2snXNVo9GVRj7eJHIJzcXXXQR3d3dLFu2zHds27Ztg5KKYDzzzDOsXr2a+vp66uvrqa2t9S1S9mfBggVUV1fzxBNPhH29BQsWsHz5crq7u9m
|
||
|
|
"text/plain": [
|
||
|
|
"<Figure size 576x288 with 1 Axes>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"needs_background": "light"
|
||
|
|
},
|
||
|
|
"output_type": "display_data"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"width = 0.75\n",
|
||
|
|
"fig, ax = plt.subplots(figsize=(8,4))\n",
|
||
|
|
"i=0\n",
|
||
|
|
"for exp in ['pred_ptcv_cifar10.p', 'pred_ptcv_cifar100.p', 'pred_ptcv_svhn.p', 'pred_ptcv_ImageNet1k.p']:\n",
|
||
|
|
" #exp=exp.replace('.p', '_pretrain.p')\n",
|
||
|
|
" bars = []\n",
|
||
|
|
" for k,v in parsed[exp].items():\n",
|
||
|
|
" v = np.array(v)\n",
|
||
|
|
" m = np.mean(v)\n",
|
||
|
|
" s = np.std(v)\n",
|
||
|
|
" #print(k,m,s,v)\n",
|
||
|
|
" bars.append((k,m,s))\n",
|
||
|
|
" \n",
|
||
|
|
" ys = [s[1] for s in bars]\n",
|
||
|
|
" yerrs = [s[2] for s in bars]\n",
|
||
|
|
" lbs = [s[0] for s in bars]\n",
|
||
|
|
" xpos = np.arange(len(ys))\n",
|
||
|
|
" i += 1\n",
|
||
|
|
" \n",
|
||
|
|
" ax.bar(xpos+(i-2.5)*width/4, ys, yerr=yerrs, width=width/4, align='center', alpha=0.5, label=labels[exp],\n",
|
||
|
|
" hatch=pattern[exp], edgecolor='black', lw=1.,\n",
|
||
|
|
" error_kw=dict(ecolor='blue', lw=2, capsize=1, capthick=2))\n",
|
||
|
|
" ax.set_ylabel('Spearman $\\\\rho$')\n",
|
||
|
|
" ax.set_xticks(xpos)\n",
|
||
|
|
" ax.set_xticklabels(lbs)\n",
|
||
|
|
" ax.yaxis.grid(True)\n",
|
||
|
|
" ax.set_axisbelow(True)\n",
|
||
|
|
"\n",
|
||
|
|
"# Save the figure and show\n",
|
||
|
|
"plt.legend(prop={'size': 10})\n",
|
||
|
|
"plt.tight_layout()\n",
|
||
|
|
"plt.savefig('ptcv.pdf')\n",
|
||
|
|
"plt.show()"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 5,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"pattern = {\n",
|
||
|
|
" 'grad_norm': '\\\\',\n",
|
||
|
|
" 'snip': '/',\n",
|
||
|
|
" 'grasp': '-',\n",
|
||
|
|
" 'fisher': '*',\n",
|
||
|
|
" 'synflow': 'o', \n",
|
||
|
|
" 'jacob_cov': 'x' \n",
|
||
|
|
"}\n",
|
||
|
|
"ps=[ \"|\" , \"\\\\\" , \"/\" , \"+\" , \"-\", \".\", \"*\",\"x\", \"o\", \"O\" ]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 6,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAncAAAEYCAYAAAA+gNBwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeXhTZdo/8O+TpFm6EwqUbrSFFlqoKJTdtxSVgoijOAqKOjruziAjzs91FIF53WdwQ1HcX8ZRR0cdZFABBZxBkVLWLnRBShdIF7q3SdMkz++PNCW0J01Ompwm6f25Li7a5OTkSe7m6d1znnPfjHMOQgghhBASGGSDPQBCCCGEEOI5lNwRQgghhAQQSu4IIYQQQgIIJXeEEEIIIQGEkjtCCCGEkACiGOwBDFRUVBRPTEwc7GEQQgghfikvL6+ecz5isMdBPMfvk7vExEQcOHBgsIdBCCGE+CXG2KnBHgPxLDotSwghhBASQCi5I4QQQggJIJTcEUIIIYQEEEruCCGEEEICCCV3hBBCCCEBhJI7QgghhJAAQskdIYQQQkgAoeSOEEIIISSAUHJHCCGEEBJAKLkjhBAfwpj1HyGEuIuSO0IIIYSQAOL3vWUJIcQfPfu/L6BWVy9wz3MAgAdWPCz4uJHRUXjk8Qe9ODJCiL+j5I4QQgZBra4e1y+4u8/tL75m/V/oPgD4+Ns3vTksQkgAoOSOEEJ80EvvPAXOOcaPnYjLs68Go4V4hBAXUXJHCCE+5MlVf8XxsnxcNHEGACDv2M+Qy+RYMPdXgzwyQoi/oOSOEEJ8SPGJAkzNmIG5M3N6bjtUkEvJHSHEZZJeLcsYW8gYK2aMlTHGHhG4P4ExtosxdogxdpQxtkjK8RFCyGATOv3KOR+EkRBC/JVkR+4YY3IArwGYD6AKQC5jbAvnvNBus8cB/INzvpExlg5gG4BEqcZICCGDLW1cBvKO/dzzfd6xnzFh3KRBHBEhxN9IeVp2OoAyzvkvAMAY+xjAVQDskzsOILz76wgApyUcHyGEDLqcrCthtphxMH8/GGOYmDoZOVlXDvawCCF+RMrkLhZApd33VQBm9NpmDYDtjLH7AIQAuExoR4yxuwDcBQCjRo3C7t27PT1WQgjxKp1Oh/LycsH70pMuQnrSRT3fnzp16rzH0ZxHCOmPr11QcQOA9znnf2WMzQKwmTE2iXNusd+Ic74JwCYAyMzM5NnZ2dKPlBBCBmDLZ18jMTFR9OOii6NBcx4hpD9SXlBRDSDe7vu47tvs3Q7gHwDAOf8JgBpAlCSjI4QQQggJAFImd7kAUhhjSYwxJYDrAWzptU0FgEsBgDGWBmtyVyfhGAkhhBBC/JpkyR3n3ARgBYBvARTBelVsAWNsHWPMVsDpjwDuZIwdAfARgFs51QAghBBCCHGZpGvuOOfbYC1vYn/baruvCwHMkXJMhBBCCCGBxNcuqCCEkCHj2z1bUFR2zKUespxzbP/hK5RXnJR4lIQQf0PJHSGEDIK6+lqc7qjF1AzXesh+vftLHC/LR3hYuOD9hBBiQ8kdIYQMgvaONszKuNTlHrK2nrNnmunIHSGkf5L2liWEEGIjroeso9O1hBDSGx25I4SQQRAaEiqqh6yt56w6TC7F8AghfoySO0IIGQRRw0cgfPRIl3vI2nrOHi3bJ+EoCSH+iJI7QggZJIvmLcGieUtc2pYxhkXzlqDFWOvlURFC/B2tuSOEEEIICSCU3BFCCCGEBBBK7gghhBBCAggld4QQQgghAYSSO0IIIYSQAEJXyxJCiA+x9ZB1tecsIYT0RskdIYT4EFsPWVd7zhJCSG+U3BFCiA+x9ZB1tecsIYT0RmvuCCHEhwidfu2v5ywhhPRGR+4IIcSH2HrI2jjrOUsIIb1RckcIIT7E1kPW1Z6zhBDSGyV3hBDiQ2w9ZF3tOUsIIb1JuuaOMbaQMVbMGCtjjD0icP+LjLHD3f9KGGNNUo6PEEIIIcTfSXbkjjEmB/AagPkAqgDkMsa2cM4LbdtwzlfZbX8fgIukGh8hhBBCSCCQ8sjddABlnPNfOOdGAB8DuKqf7W8A8JEkIyOEEEIICRBSJnexACrtvq/qvq0PxtgYAEkAvpdgXIQQQgghAcNXL6i4HsBnnHOz0J2MsbsA3AUAo0aNwu7duyUcGiGEDJxOp0N5eblbj6M5jxDSHymTu2oA8Xbfx3XfJuR6AL93tCPO+SYAmwAgMzOTZ2dne2iIhBAijS2ffY3iU0dd7iFr6zlrMOpBcx4hpD9SJne5AFIYY0mwJnXXA1jeeyPG2AQAwwD8JOHYCCEBzpYz+Uqzh7r6WpzuqHW5h6yt52x4WLiUwySE+CHJkjvOuYkxtgLAtwDkAN7lnBcwxtYBOMA539K96fUAPubUb4cQEsDaO9owK+NSl3vI2nrOnmk+KdUQCSF+StI1d5zzbQC29bptda/v10g5JkIIGRziesg6Ol1LCCG9+eoFFYQQEtBCQ0JF9ZC19ZxVh8mlGB4hxI9RckcICSgvPb0aTTWVAve8BwBY84ff9txi6DSi9mwTRg6PRHTCWNz/2DqJRglEDR+B8NEjXe4ha+s5e7Rsn2RjHOp8bZ0mIa6i5I4QElCaaiqx5urUPrevfcX6v+2+cl0j/rH7GO65JgOJ0cOw5ssSKYcJAKJ6yNp6zrYYa708KkKIv5O0tywhhAwUY+eOqLjLltgtzbYmdt7giXESQog76MgdIWRI4N8/A0CaxI74l7+sWYuGqiqBe94CADx2x52Cj9PGxeH/rXnSiyMjxD2U3BFChgxK7IiQhqoqrJo8uc/tz3T/L3QfALx45IgXR0WI+yi5I4QMCZTYETH2lpbii/v+gDkpKYM9FEJEo+SOEOKTnnz6eVTp6gXueR4AcPvKh8671djZicaGesjqioFeF1RQYkfE4Jzj66NHAQCzx40TXWOQrrIlg42SO0KIT6rS1WPSlXf0veNV639C9zXoqrDrL+evj/J2YvfMn/6E1poa9C1KbFuvdQc6DAZU6moAAPHRoxCsVqPgeBWwoO/+bD1kXe05SzyLc47Smho06TsAAKU1NUgZNUrw/T+QX4IHVjyM+rN1aGtvA8AREhwKW9mdB1Y8LPgcwWEq/O8z0pXdIUMPJXeEkIChjY6DUqXu+V6KI3Z5+/cj3NCJ+OFa/E9KKiYnJAA4t15LV1yCDmMnIlUq6/elpQhWqtBwtkNwf7Yesq72nB1KvHlE7EhFBf5TWoLKsw3oMHYitDter+7cgWClqk98j1RUoLG5HeHKkTjdXotZGZcAwHmFqa9fcHef59m26wscLaBahcS7KLkjhAQUudzawUGqU7EqpRItTU0IUaqQHhvb5/4Wgx6JUVFg3Uf2hoWEoLy+HjIHR+JsPWRd7TlLPCM9NhZ5p8odxqt3fNNjYyFjTDBeMtntuP/2Pwk+T/GJAoSHRXj3xZAhj5I7QohfWb/debFhKdfYdRqNmJ6UjJtnz4ZC3rc12PSkZByprEBMpHUcpxubMD0pGfVlpwX3J3T6r7+es8QzguRy/Gb2HDAwwXj1jm+QXA6NWi06XnR6nUiBkjtCSEDpMpklv3giJjJSMLGz3XeksgKNHe0ArKcWYyIjAQgnd7YesjbOes4GImct5G677gqMHB4JtUp53r2Ro+IH1EJOIZc7jJej+IqNV9q4DBwq2ev2GAlxBSV3hJCA0t5hkDSxM5stKK8XuqrXqry+Hq0GA+K1wwEAhaer+93e1kPW1Z6zgchZC7nV10wUjK8nWsh5O145WVeipMp/6+Pl5eWNVCgUbwOYBOpyNVgsAPJNJtMdU6dOFexHSMkdISSghASrJS13kjA6GsEWS5/ba1/ZAADY/KMSf1x4OdJGjwbnHMd1OuSVn3S4P1sPWVd7zg4V5brGnq+9Gd9glXfjxRjDiKiRnhqu5BQKxdvR0dFpI0aMaJTJZLReYBBYLBZWV1eXrtPp3gYguBiXkjtCSEAJUgifPvOWEE0wlmdkgHPeZz0V5xzLZ86CXGY9wMEYQ3pMDMZHR2NL8aeSjtOf2dZQnvz7Q15N7Pq
|
||
|
|
"text/plain": [
|
||
|
|
"<Figure size 648x288 with 1 Axes>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {
|
||
|
|
"needs_background": "light"
|
||
|
|
},
|
||
|
|
"output_type": "display_data"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"width = 0.75\n",
|
||
|
|
"fig, ax = plt.subplots(figsize=(9,4))\n",
|
||
|
|
"i=0\n",
|
||
|
|
"all_b = {}\n",
|
||
|
|
"for exp in ['pred_ptcv_cifar10.p', 'pred_ptcv_cifar100.p', 'pred_ptcv_svhn.p', 'pred_ptcv_ImageNet1k.p']:\n",
|
||
|
|
" #exp=exp.replace('.p', '_pretrain.p')\n",
|
||
|
|
" bars = []\n",
|
||
|
|
" for k,v in parsed[exp].items():\n",
|
||
|
|
" v = np.array(v)\n",
|
||
|
|
" m = np.mean(v)\n",
|
||
|
|
" s = np.std(v)\n",
|
||
|
|
" #print(k,m,s,v)\n",
|
||
|
|
" bars.append((k,m,s))\n",
|
||
|
|
" if k not in all_b:\n",
|
||
|
|
" all_b[k] = []\n",
|
||
|
|
" all_b[k].append((labels[exp],m,s))\n",
|
||
|
|
" print()\n",
|
||
|
|
" \n",
|
||
|
|
"\n",
|
||
|
|
"i=0\n",
|
||
|
|
"for k,v in all_b.items():\n",
|
||
|
|
" \n",
|
||
|
|
" ys = [s[1] for s in v]\n",
|
||
|
|
" yerrs = [s[2] for s in v]\n",
|
||
|
|
" lbs = [s[0] for s in v]\n",
|
||
|
|
" xpos = np.arange(len(ys))\n",
|
||
|
|
" \n",
|
||
|
|
" ax.bar(xpos+(i-2.5)*width/6, ys, yerr=yerrs, width=width/6, align='center', alpha=0.5, label=k,\n",
|
||
|
|
" hatch=pattern[k], edgecolor='black', lw=1.,\n",
|
||
|
|
" error_kw=dict(ecolor='blue', lw=2, capsize=1, capthick=2))\n",
|
||
|
|
" ax.set_ylabel('Spearman $\\\\rho$')\n",
|
||
|
|
" ax.yaxis.grid(True)\n",
|
||
|
|
" ax.set_axisbelow(True)\n",
|
||
|
|
" i += 1\n",
|
||
|
|
"\n",
|
||
|
|
" \n",
|
||
|
|
"ax.set_xticks([0, 1, 2, 3])\n",
|
||
|
|
"ax.set_xticklabels(['CIFAR-10', 'CIFAR-100', 'SVHN', 'ImageNet1k'])\n",
|
||
|
|
"# Save the figure and show\n",
|
||
|
|
"#plt.legend(loc='upper center', prop={'size': 11}, ncol=6, bbox_to_anchor=(0.5,1.15))\n",
|
||
|
|
"plt.legend(prop={'size': 11}, bbox_to_anchor=(1.25,0.75))\n",
|
||
|
|
"plt.tight_layout()\n",
|
||
|
|
"plt.savefig('ptcv_flip.pdf')\n",
|
||
|
|
"plt.show()"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": []
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.7.6"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 4
|
||
|
|
}
|